Skip to content

Commit 7e434f5

Browse files
doxthreedoxthree
committed
296 files in groupfolders are not found when they are moved or deleted (#297)
* add setup for Groupfolders app and wait for HTTP readiness * enhance file handling in TagUnscannedService and ScanService * added FileService dependency to TagUnscannedService for improved file ID validation. * implemented removal of stale VaaS tags in ScanService when files are not found. * updated DbFileMapper to use distinct file IDs for better query accuracy. * feat: add benchmarking scripts for DbFileMapper queries and file population * inject TagUnscannedService directly instead other deps * enhance MariaDB configuration and add slow query log support
1 parent 0efb198 commit 7e434f5

8 files changed

Lines changed: 848 additions & 26 deletions

‎lib/Command/TagUnscannedCommand.php‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,24 +7,19 @@
77
namespace OCA\GDataVaas\Command;
88

99
use OCA\GDataVaas\Logging\ConsoleCommandLogger;
10-
use OCA\GDataVaas\Service\TagService;
1110
use OCA\GDataVaas\Service\TagUnscannedService;
1211
use OCP\DB\Exception;
13-
use OCP\IAppConfig;
1412
use Psr\Log\LoggerInterface;
1513
use Symfony\Component\Console\Command\Command;
1614
use Symfony\Component\Console\Input\InputInterface;
1715
use Symfony\Component\Console\Output\OutputInterface;
1816

1917
class TagUnscannedCommand extends Command {
20-
private TagUnscannedService $tagUnscannedService;
21-
private LoggerInterface $logger;
22-
23-
public function __construct(IAppConfig $appConfig, TagService $tagService, LoggerInterface $logger) {
18+
public function __construct(
19+
private TagUnscannedService $tagUnscannedService,
20+
private LoggerInterface $logger,
21+
) {
2422
parent::__construct();
25-
$this->logger = $logger;
26-
27-
$this->tagUnscannedService = new TagUnscannedService($logger, $tagService, $appConfig);
2823
}
2924

3025
/**

‎lib/Db/DbFileMapper.php‎

Lines changed: 27 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -69,11 +69,16 @@ public function getFileIdsWithoutTags(array $excludedTagIds, int $limit, int $of
6969
->from('filecache', 'fc')
7070
->leftJoin('fc', 'storages', 's', $query->expr()->eq('fc.storage', 's.numeric_id'))
7171
->leftJoin(
72-
'fc', 'systemtag_object_mapping', 'o', $query->expr()->eq(
73-
'o.objectid', $query->createFunction(sprintf('CAST(fc.fileid AS %s)', $this->stringType))))
74-
->where($query->expr()->notIn(
75-
'o.systemtagid', $query->createNamedParameter($excludedTagIds, IQueryBuilder::PARAM_INT_ARRAY)))
76-
->orWhere($query->expr()->isNull('o.systemtagid'))
72+
'fc',
73+
'systemtag_object_mapping',
74+
'o',
75+
$query->expr()->andX(
76+
$query->expr()->eq('o.objectid', $query->createFunction(sprintf('CAST(fc.fileid AS %s)', $this->stringType))),
77+
$query->expr()->eq('o.objecttype', $query->createNamedParameter('files')),
78+
$query->expr()->in('o.systemtagid', $query->createNamedParameter($excludedTagIds, IQueryBuilder::PARAM_INT_ARRAY))
79+
)
80+
)
81+
->where($query->expr()->isNull('o.objectid'))
7782
->andWhere($query->expr()->neq('fc.mimetype', $query->createNamedParameter($dirMimeTypeId)))
7883
->andWhere($query->expr()->orX(
7984
$query->expr()->like('fc.path', $query->createNamedParameter('files/%')),
@@ -107,16 +112,25 @@ public function getFileIdsWithTags(array $includedTagIds, int $limit, int $offse
107112
$instanceId = $this->config->getSystemValue('instanceid', '');
108113

109114
$query = $this->db->getQueryBuilder();
110-
$query->select('fc.fileid')
115+
if (count($includedTagIds) > 1) {
116+
$query->selectDistinct('fc.fileid');
117+
} else {
118+
$query->select('fc.fileid');
119+
}
120+
$query
111121
->from('filecache', 'fc')
112122
->leftJoin('fc', 'storages', 's', $query->expr()->eq('fc.storage', 's.numeric_id'))
113-
->leftJoin(
114-
'fc', 'systemtag_object_mapping', 'o', $query->expr()->eq(
115-
'o.objectid', $query->createFunction(sprintf('CAST(fc.fileid AS %s)', $this->stringType))))
116-
->where($query->expr()->in(
117-
'o.systemtagid', $query->createNamedParameter($includedTagIds, IQueryBuilder::PARAM_INT_ARRAY)))
118-
->orWhere($query->expr()->isNull('o.systemtagid'))
119-
->andWhere($query->expr()->neq('fc.mimetype', $query->createNamedParameter($dirMimeTypeId)))
123+
->innerJoin(
124+
'fc',
125+
'systemtag_object_mapping',
126+
'o',
127+
$query->expr()->andX(
128+
$query->expr()->eq('o.objectid', $query->createFunction(sprintf('CAST(fc.fileid AS %s)', $this->stringType))),
129+
$query->expr()->eq('o.objecttype', $query->createNamedParameter('files')),
130+
$query->expr()->in('o.systemtagid', $query->createNamedParameter($includedTagIds, IQueryBuilder::PARAM_INT_ARRAY))
131+
)
132+
)
133+
->where($query->expr()->neq('fc.mimetype', $query->createNamedParameter($dirMimeTypeId)))
120134
->andWhere($query->expr()->orX(
121135
$query->expr()->like('fc.path', $query->createNamedParameter('files/%')),
122136
$query->expr()->notLike('s.id', $query->createNamedParameter('home::%'))

‎lib/Service/ScanService.php‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,8 @@ public function run(): int {
7575
" . $this->appConfig->getValueInt(Application::APP_ID, 'maxScanSizeInMB', 256) . 'MB.'
7676
);
7777
} catch (NotFoundException) {
78-
$this->logger->error("File $fileId not found");
78+
$this->tagService->removeVaasTagsFromFile($fileId);
79+
$this->logger->debug("File $fileId not found during scan, likely moved or deleted concurrently — skipping");
7980
} catch (NotPermittedException) {
8081
$this->logger->error("Current settings do not permit scanning file wit ID $fileId.");
8182
} catch (VaasAuthenticationException) {
@@ -131,8 +132,9 @@ private function getFileIdsToScan(): array {
131132
} else {
132133
$this->logger->debug("File with ID $fileId is not readable or not allowed to scan, skipping.");
133134
}
134-
} catch (NotFoundException $e) {
135-
$this->logger->error("File with ID $fileId not found, skipping: " . $e->getMessage(), ['exception' => $e]);
135+
} catch (NotFoundException) {
136+
$this->tagService->removeVaasTagsFromFile($fileId);
137+
$this->logger->debug("File with ID $fileId not found, likely moved or trashed concurrently — skipping");
136138
} catch (NotPermittedException $e) {
137139
$this->logger->error(
138140
"Current settings do not permit scanning file with ID $fileId, skipping: " . $e->getMessage()

‎lib/Service/TagService.php‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,26 @@ public function removeTag(string $tagName): void {
196196
$this->logger->debug('Tag removed: ' . $tagName);
197197
}
198198

199+
/**
200+
* Removes all VaaS tags from a file id if they are present.
201+
* Useful to clean up stale tag mappings that point to missing files.
202+
* @param int $fileId
203+
* @return void
204+
*/
205+
public function removeVaasTagsFromFile(int $fileId): void {
206+
$vaasTagIds = $this->getVaasTagIds();
207+
if (empty($vaasTagIds)) {
208+
return;
209+
}
210+
211+
try {
212+
$this->silentTagMapper->unassignTags((string)$fileId, 'files', $vaasTagIds);
213+
$this->logger->debug('Removed stale VaaS tags for missing file ' . $fileId);
214+
} catch (\Throwable $e) {
215+
$this->logger->debug('Failed to remove stale VaaS tags for file ' . $fileId . ': ' . $e->getMessage());
216+
}
217+
}
218+
199219
/**
200220
* @return array
201221
* @throws Exception

‎lib/Service/TagUnscannedService.php‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,17 +8,25 @@
88

99
use OCA\GDataVaas\AppInfo\Application;
1010
use OCP\DB\Exception;
11+
use OCP\Files\NotFoundException;
1112
use OCP\IAppConfig;
1213
use Psr\Log\LoggerInterface;
1314

1415
class TagUnscannedService {
1516
private TagService $tagService;
17+
private FileService $fileService;
1618
private IAppConfig $appConfig;
1719
private LoggerInterface $logger;
1820

19-
public function __construct(LoggerInterface $logger, TagService $tagService, IAppConfig $appConfig) {
21+
public function __construct(
22+
LoggerInterface $logger,
23+
TagService $tagService,
24+
FileService $fileService,
25+
IAppConfig $appConfig,
26+
) {
2027
$this->logger = $logger;
2128
$this->tagService = $tagService;
29+
$this->fileService = $fileService;
2230
$this->appConfig = $appConfig;
2331
}
2432

@@ -61,6 +69,13 @@ public function run(): int {
6169
$fileIds = $this->tagService->getFileIdsWithoutTags($excludedTagIds, 10000);
6270

6371
foreach ($fileIds as $fileId) {
72+
try {
73+
$this->fileService->getNodeFromFileId($fileId);
74+
} catch (NotFoundException) {
75+
$this->logger->debug("Skipping stale file ID $fileId while tagging unscanned files");
76+
continue;
77+
}
78+
6479
if ($this->tagService->hasAnyButUnscannedTag($fileId)) {
6580
continue;
6681
}

0 commit comments

Comments
 (0)