Merge pull request #8502 from nextcloud/feature/notify-file-ids

feat: support `notify_file_id` push notifications

Author: mgallien

src/common/preparedsqlquerymanager.h

@@ -98,6 +98,7 @@ class OCSYNC_EXPORT PreparedSqlQueryManager
         ListAllTopLevelE2eeFoldersStatusLessThanQuery,
         FolderUpdateInvalidEncryptionStatus,
         FileUpdateInvalidEncryptionStatus,
+        HasFileIdQuery,
 
         PreparedQueryCount
     };

src/common/syncjournaldb.cpp

@@ -6,6 +6,8 @@
 
 #include <QCryptographicHash>
 #include <QFile>
+#include <QJsonArray>
+#include <QJsonDocument>
 #include <QLoggingCategory>
 #include <QStringList>
 #include <QElapsedTimer>
@@ -33,6 +35,8 @@
 static constexpr auto MAJOR_VERSION_3 = 3;
 static constexpr auto MINOR_VERSION_16 = 16;
 
+using namespace Qt::StringLiterals;
+
 namespace OCC {
 
 Q_LOGGING_CATEGORY(lcDb, "nextcloud.sync.database", QtInfoMsg)
@@ -1769,6 +1773,58 @@ bool SyncJournalDb::updateLocalMetadata(const QString &filename,
     return true;
 }
 
+bool SyncJournalDb::hasFileIds(const QList<qint64> &fileIds)
+{
+    if (fileIds.isEmpty()) {
+        // no need to check the db if no file id matches
+        return false;
+    }
+
+    QMutexLocker locker(&_mutex);
+
+    if (!checkConnect()) {
+        return false;
+    }
+
+    // quick workaround for looking up pure numeric file IDs: cast it to integer
+    //
+    // using `IN` with a list of IDs does not allow for a prepared query: the execution plan
+    // would be different depending on the amount of elements as each element is added to a
+    // temporary table one-by-one.  with `json_each()`, all that changes is one string
+    // parameter which is perfect for creating a prepared query :)
+    const auto query = _queryManager.get(
+        PreparedSqlQueryManager::HasFileIdQuery,
+        "SELECT 1 FROM metadata, json_each(?1) file_ids WHERE CAST(metadata.fileid AS INTEGER) = CAST(file_ids.value AS INTEGER) LIMIT 1;"_ba,
+        _db
+    );
+    if (!query) {
+        qCWarning(lcDb) << "database error:" << query->error();
+        return false;
+    }
+
+    // we have a prepared query, so let's build up a JSON array of file IDs to check for.
+    // Strings are used here to avoid any surprises during serialisation: JSON only really
+    // has a double type, and who knows when the representation changes to the +e* variant.
+    QJsonArray fileIdStrings = {};
+    for (const auto &fileId : fileIds) {
+        fileIdStrings.append(QString::number(fileId));
+    }
+    const auto fileIdsParameter = QJsonDocument(fileIdStrings).toJson(QJsonDocument::Compact);
+    query->bindValue(1, fileIdsParameter);
+
+    if (!query->exec()) {
+        qCWarning(lcDb) << "file id query failed:" << query->error();
+        return false;
+    }
+
+    if (query->next().hasData && query->intValue(0) == 1) {
+        // at least one file ID from the passed list is present
+        return true;
+    }
+
+    return false;
+}
+
 Optional<SyncJournalDb::HasHydratedDehydrated> SyncJournalDb::hasHydratedOrDehydratedFiles(const QByteArray &filename)
 {
     QMutexLocker locker(&_mutex);

src/common/syncjournaldb.h

@@ -76,6 +76,8 @@ class OCSYNC_EXPORT SyncJournalDb : public QObject
     [[nodiscard]] bool updateLocalMetadata(const QString &filename,
         qint64 modtime, qint64 size, quint64 inode, const SyncJournalFileLockInfo &lockInfo);
 
+    [[nodiscard]] bool hasFileIds(const QList<qint64> &fileIds);
+
     /// Return value for hasHydratedOrDehydratedFiles()
     struct HasHydratedDehydrated
     {

src/gui/folder.cpp

@@ -92,6 +92,7 @@ Folder::Folder(const FolderDefinition &definition,
 
     connect(_accountState.data(), &AccountState::isConnectedChanged, this, &Folder::canSyncChanged);
     connect(_engine.data(), &SyncEngine::rootEtag, this, &Folder::etagRetrievedFromSyncEngine);
+    connect(_engine.data(), &SyncEngine::rootFileIdReceived, this, &Folder::rootFileIdReceivedFromSyncEngine);
 
     connect(_engine.data(), &SyncEngine::started, this, &Folder::slotSyncStarted, Qt::QueuedConnection);
     connect(_engine.data(), &SyncEngine::finished, this, &Folder::slotSyncFinished, Qt::QueuedConnection);
@@ -396,6 +397,12 @@ void Folder::etagRetrievedFromSyncEngine(const QByteArray &etag, const QDateTime
     _lastEtag = etag;
 }
 
+void Folder::rootFileIdReceivedFromSyncEngine(const qint64 fileId)
+{
+    qCDebug(lcFolder).nospace() << "retrieved root fileId=" << fileId;
+    _rootFileId = fileId;
+}
+
 void Folder::showSyncResultPopup()
 {
     if (_syncResult.firstItemNew()) {
@@ -1002,6 +1009,11 @@ void Folder::migrateBlackListPath(const QString &legacyPath)
     }
 }
 
+bool Folder::hasFileIds(const QList<qint64>& fileIds) const
+{
+    return fileIds.contains(_rootFileId) || journalDb()->hasFileIds(fileIds);
+}
+
 QString Folder::filePath(const QString& fileName)
 {
     const auto folderDir = QDir(_canonicalLocalPath);

src/gui/folder.h

@@ -316,6 +316,9 @@ class Folder : public QObject
     void blacklistPath(const QString &path);
     void migrateBlackListPath(const QString &legacyPath);
 
+    /// whether the current folder contains any of the passed fileIds
+    [[nodiscard]] bool hasFileIds(const QList<qint64>& fileIds) const;
+
 signals:
     void syncStateChange();
     void syncStarted();
@@ -429,6 +432,8 @@ private slots:
     void etagRetrieved(const QByteArray &, const QDateTime &tp);
     void etagRetrievedFromSyncEngine(const QByteArray &, const QDateTime &time);
 
+    void rootFileIdReceivedFromSyncEngine(qint64 fileId);
+
     void slotEmitFinishedDelayed();
 
     void slotNewBigFolderDiscovered(const QString &, bool isExternal);
@@ -585,6 +590,9 @@ private slots:
     QMetaObject::Connection _officeFileLockReleaseUnlockFailure;
     QMetaObject::Connection _fileLockSuccess;
     QMetaObject::Connection _fileLockFailure;
+
+    /// The remote file ID of the current folder.
+    qint64 _rootFileId = 0;
 };
 }
 

src/gui/folderman.cpp

@@ -2084,15 +2084,35 @@ void FolderMan::slotSetupPushNotifications(const Folder::Map &folderMap)
 
 void FolderMan::slotProcessFilesPushNotification(Account *account)
 {
-    qCInfo(lcFolderMan) << "Got files push notification for account" << account;
+    qCDebug(lcFolderMan) << "received notify_file push notification account=" << account->displayName();
 
     for (auto folder : std::as_const(_folderMap)) {
         // Just run on the folders that belong to this account
         if (folder->accountState()->account() != account) {
             continue;
         }
 
-        qCInfo(lcFolderMan) << "Schedule folder" << folder << "for sync";
+        qCInfo(lcFolderMan).nospace() << "scheduling sync folder=" << folder->alias() << " account=" << account->displayName() << " reason=notify_file";
+        scheduleFolder(folder);
+    }
+}
+
+void FolderMan::slotProcessFileIdsPushNotification(Account *account, const QList<qint64> &fileIds)
+{
+    qCDebug(lcFolderMan).nospace() << "received notify_file_id push notification account=" << account->displayName() << " fileIds=" << fileIds;
+
+    for (auto folder : std::as_const(_folderMap)) {
+        // Just run on the folders that belong to this account
+        if (folder->accountState()->account() != account) {
+            continue;
+        }
+
+        if (!folder->hasFileIds(fileIds)) {
+            qCDebug(lcFolderMan).nospace() << "no matching file ids, ignoring folder=" << folder->alias() << " account=" << account->displayName();
+            continue;
+        }
+
+        qCInfo(lcFolderMan).nospace() << "scheduling sync folder=" << folder->alias() << " account=" << account->displayName() << " reason=notify_file_id";
         scheduleFolder(folder);
     }
 }
@@ -2104,6 +2124,7 @@ void FolderMan::slotConnectToPushNotifications(const AccountPtr &account)
     if (pushNotificationsFilesReady(account)) {
         qCInfo(lcFolderMan) << "Push notifications ready";
         connect(pushNotifications, &PushNotifications::filesChanged, this, &FolderMan::slotProcessFilesPushNotification, Qt::UniqueConnection);
+        connect(pushNotifications, &PushNotifications::fileIdsChanged, this, &FolderMan::slotProcessFileIdsPushNotification, Qt::UniqueConnection);
     }
 }
 

src/gui/folderman.h

@@ -330,6 +330,7 @@ private slots:
 
     void slotSetupPushNotifications(const OCC::Folder::Map &);
     void slotProcessFilesPushNotification(OCC::Account *account);
+    void slotProcessFileIdsPushNotification(OCC::Account *account, const QList<qint64> &fileIds);
     void slotConnectToPushNotifications(const OCC::AccountPtr &account);
 
     void slotLeaveShare(const QString &localFile, const QByteArray &folderToken = {});

src/libsync/discovery.cpp

@@ -2271,6 +2271,7 @@ DiscoverySingleDirectoryJob *ProcessDirectoryJob::startAsyncServerQuery()
     }
 
     connect(serverJob, &DiscoverySingleDirectoryJob::etag, this, &ProcessDirectoryJob::etag);
+    connect(serverJob, &DiscoverySingleDirectoryJob::firstDirectoryFileId, this, &ProcessDirectoryJob::rootFileIdReceived);
     connect(serverJob, &DiscoverySingleDirectoryJob::setfolderQuota, this, &ProcessDirectoryJob::setFolderQuota);
     _discoveryData->_currentlyActiveJobs++;
     _pendingAsyncJobs++;

src/libsync/discovery.h

@@ -308,6 +308,7 @@ class ProcessDirectoryJob : public QObject
     // The root etag of this directory was fetched
     void etag(const QByteArray &, const QDateTime &time);
     void updatedRootFolderQuota(const int64_t &bytesUsed, const int64_t &bytesAvailable);
+    void rootFileIdReceived(qint64 fileId);
 
 private slots:
     void setFolderQuota(const OCC::FolderQuota &folderQuota);

src/libsync/discoveryphase.cpp

@@ -503,9 +503,19 @@ void DiscoverySingleDirectoryJob::directoryListingIteratedSlot(const QString &fi
             }
         }
         if (map.contains("fileid"_L1)) {
+            // this is from the "oc:fileid" property, this is the plain ID without any special format (e.g. "2")
             _localFileId = map.value("fileid"_L1).toUtf8();
+
+            bool ok = false;
+            if (qint64 numericFileId = _localFileId.toLongLong(&ok, 10); ok) {
+                qCDebug(lcDiscovery).nospace() << "received numericFileId=" << numericFileId;
+                emit firstDirectoryFileId(numericFileId);
+            } else {
+                qCWarning(lcDiscovery).nospace() << "conversion to qint64 failed _localFileId=" << _localFileId;
+            }
         }
         if (map.contains("id"_L1)) {
+            // this is from the "oc:id" property, the format is e.g. "00000002oc123xyz987e"
             _fileId = map.value("id"_L1).toUtf8();
         }
         if (map.contains("is-encrypted"_L1) && map.value("is-encrypted"_L1) == "1"_L1) {