feat(image-cleanup): event-driven cleanup of orphaned ExApp Docker images via HaRP

.github/workflows/tests-deploy.yml

@@ -1237,9 +1237,17 @@ jobs:
           docker inspect nc_app_app-skeleton-python | json_pp > app_harp_bridge_no_tls_container.json
           docker logs nc_app_app-skeleton-python > app_harp_bridge_no_tls_container.log 2>&1
 
-      - name: Unregister Skeleton & Daemon
+      - name: Unregister Skeleton
         run: |
           docker exec nextcloud-docker sudo -u www-data php occ app_api:app:unregister app-skeleton-python
+
+      - name: Test image cleanup
+        env:
+          OCC_PREFIX: docker exec nextcloud-docker sudo -u www-data php occ
+        run: python3 tests/test_image_cleanup.py
+
+      - name: Unregister Daemon
+        run: |
           docker exec nextcloud-docker sudo -u www-data php occ app_api:daemon:unregister harp_proxy
 
       - name: Show all logs

AGENTS.md

@@ -302,8 +302,8 @@ Source: `lib/Command/ExApp/`. `appid` is the ExApp's id.
 | `app_api:app:register <appid> [daemon]` | Install an ExApp. Omit `[daemon]` to use the default. Definition source: App Store (default), or `--info-xml <url-or-abs-path>`, or `--json-info <json>`. `--env NAME=VALUE` (repeatable) sets container env, but only for variables the app **declares** in its manifest (undeclared names are silently ignored); `--mount SRC:DST[:ro\|rw]` (repeatable) adds bind mounts (Docker daemons; recorded but not mounted on K8s). `--wait-finish` blocks until deployed; `--silent`; `--test-deploy-mode` re-registers if already present. See `docs/appapi/exapp-contract.md`. |
 | `app_api:app:enable <appid>` | Enable a registered ExApp. No options. |
 | `app_api:app:disable <appid>` | Disable a registered ExApp. No options. |
-| `app_api:app:update [appid]` | Update one ExApp, or `--all` (with `--showonly` to preview, `--include-disabled` to widen). Reuses the ExApp's stored daemon and deploy options. |
-| `app_api:app:unregister <appid>` | Remove an ExApp. `--rm-data` also deletes its persistent volume (data is **kept** by default). `--force` continues past errors; `--silent`. The Docker image is never removed automatically; prune it manually if disk space matters. |
+| `app_api:app:update [appid]` | Update one ExApp, or `--all` (with `--showonly` to preview, `--include-disabled` to widen). Reuses the ExApp's stored daemon and deploy options. On HaRP Docker daemons the replaced image is cleaned up automatically after the configured grace period; `--purge-old-image-now` deletes it immediately, `--keep-old-image` skips cleanup. |
+| `app_api:app:unregister <appid>` | Remove an ExApp. `--rm-data` also deletes its persistent volume (data is **kept** by default). `--force` continues past errors; `--silent`. On HaRP Docker daemons the ExApp's image is cleaned up automatically after the configured grace period (default 24h, admin settings); `--purge-now` deletes it immediately, `--keep-image` skips cleanup. Docker refuses the delete (409) while any container still uses the image, so shared images are safe. |
 | `app_api:app:list` | List ExApps: `<appid> (<name>): <version> [enabled\|disabled]`. No options. |
 
 Notes:

lib/AppInfo/Application.php

@@ -46,7 +46,13 @@ class Application extends App implements IBootstrap {
 	public const APP_ID = 'app_api';
 	public const TEST_DEPLOY_APPID = 'test-deploy';
 	public const TEST_DEPLOY_INFO_XML = 'https://raw.githubusercontent.com/nextcloud/test-deploy/main/appinfo/info.xml';
-	public const MINIMUM_HARP_VERSION = '0.3.0';
+	public const MINIMUM_HARP_VERSION = '0.4.1'; // image cleanup needs /docker/exapp/image_remove + image_ref in exists
+
+	public const CONF_IMAGE_CLEANUP_ENABLED = 'image_cleanup_enabled';
+	public const CONF_IMAGE_CLEANUP_GRACE_HOURS = 'image_cleanup_grace_hours';
+	public const DEFAULT_IMAGE_CLEANUP_ENABLED = true;
+	public const DEFAULT_IMAGE_CLEANUP_GRACE_HOURS = 24;
+	public const MAX_IMAGE_CLEANUP_GRACE_HOURS = 720; // 30 days
 
 	public function __construct(array $urlParams = []) {
 		parent::__construct(self::APP_ID, $urlParams);

lib/BackgroundJob/OrphanedImageCleanupJob.php

@@ -0,0 +1,145 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\AppAPI\BackgroundJob;
+
+use OCA\AppAPI\DeployActions\DockerActions;
+use OCA\AppAPI\Service\DaemonConfigService;
+use OCA\AppAPI\Service\ExAppImageCleanupService;
+use OCP\AppFramework\Utility\ITimeFactory;
+use OCP\BackgroundJob\QueuedJob;
+use OCP\Util;
+use Psr\Log\LoggerInterface;
+
+/**
+ * Removes a Docker image that was orphaned by an ExApp uninstall or update.
+ *
+ * Scheduled via IJobList::scheduleAfter when an ExApp is removed (or updated to a new image).
+ * The grace period (default 24h) is configured globally; it gives a window for reinstall/rollback
+ * before the image is reclaimed. Docker's own DELETE-with-409 behaviour is the source of truth
+ * for "is the image still in use": if anything (a fresh container, a stopped container, another
+ * ExApp using the same image) still references the ref, Docker refuses and the job logs and exits.
+ *
+ * QueuedJob handles self-removal automatically: the entry is removed from the job queue before
+ * run() is called, so a failure here is not retried by re-queuing. The next orphan event (or a
+ * manual --purge-now from the admin) is the next opportunity.
+ */
+class OrphanedImageCleanupJob extends QueuedJob {
+	public function __construct(
+		ITimeFactory $time,
+		private readonly DaemonConfigService $daemonConfigService,
+		private readonly DockerActions $dockerActions,
+		private readonly ExAppImageCleanupService $imageCleanupService,
+		private readonly LoggerInterface $logger,
+	) {
+		parent::__construct($time);
+	}
+
+	protected function run($argument): void {
+		if (!is_array($argument)) {
+			$this->logger->warning('OrphanedImageCleanupJob received non-array argument; skipping.');
+			return;
+		}
+		$daemonName = (string)($argument['daemon_id'] ?? '');
+		$imageRef = (string)($argument['image_ref'] ?? '');
+		$appid = (string)($argument['appid'] ?? '');
+
+		if ($daemonName === '' || $imageRef === '') {
+			$this->logger->warning('OrphanedImageCleanupJob missing daemon_id or image_ref in argument; skipping.', [
+				'argument' => $argument,
+			]);
+			return;
+		}
+
+		if (!$this->imageCleanupService->isMasterEnabled()) {
+			// The admin disabled cleanup after this job was queued; honor that.
+			$this->logger->info(sprintf(
+				'OrphanedImageCleanupJob: image cleanup is disabled; skipping image "%s" (appid=%s, daemon=%s).',
+				$imageRef,
+				$appid,
+				$daemonName,
+			));
+			return;
+		}
+
+		$daemon = $this->daemonConfigService->getDaemonConfigByName($daemonName);
+		if ($daemon === null) {
+			$this->logger->info(sprintf(
+				'OrphanedImageCleanupJob: daemon "%s" no longer exists; skipping cleanup of image "%s" (appid=%s).',
+				$daemonName,
+				$imageRef,
+				$appid,
+			));
+			return;
+		}
+
+		if ($daemon->getAcceptsDeployId() !== DockerActions::DEPLOY_ID) {
+			// Kubernetes / Manual: image lifecycle isn't AppAPI's responsibility.
+			$this->logger->debug(sprintf(
+				'OrphanedImageCleanupJob: daemon "%s" is not a Docker daemon (deploy_id=%s); skipping image "%s" (appid=%s).',
+				$daemonName,
+				$daemon->getAcceptsDeployId(),
+				$imageRef,
+				$appid,
+			));
+			return;
+		}
+
+		try {
+			$result = $this->dockerActions->removeImage($daemon, $imageRef);
+		} catch (\Throwable $e) {
+			$this->logger->error(sprintf(
+				'OrphanedImageCleanupJob: unexpected exception removing image "%s" on daemon "%s" (appid=%s): %s',
+				$imageRef,
+				$daemonName,
+				$appid,
+				$e->getMessage(),
+			), ['exception' => $e]);
+			return;
+		}
+
+		if ($result['deleted'] === true) {
+			if (($result['reason'] ?? null) === 'not_found') {
+				$this->logger->info(sprintf(
+					'OrphanedImageCleanupJob: image "%s" already gone on daemon "%s" (appid=%s).',
+					$imageRef,
+					$daemonName,
+					$appid,
+				));
+				return;
+			}
+			$this->logger->info(sprintf(
+				'OrphanedImageCleanupJob: removed image "%s" on daemon "%s" (appid=%s, freed=%s).',
+				$imageRef,
+				$daemonName,
+				$appid,
+				Util::humanFileSize((int)($result['bytes_freed'] ?? 0)),
+			));
+			return;
+		}
+
+		$reason = (string)($result['reason'] ?? 'unknown');
+		if ($reason === 'in_use') {
+			$this->logger->info(sprintf(
+				'OrphanedImageCleanupJob: image "%s" still in use on daemon "%s" (appid=%s); leaving it in place.',
+				$imageRef,
+				$daemonName,
+				$appid,
+			));
+			return;
+		}
+		$this->logger->warning(sprintf(
+			'OrphanedImageCleanupJob: failed to remove image "%s" on daemon "%s" (appid=%s, reason=%s).',
+			$imageRef,
+			$daemonName,
+			$appid,
+			$reason,
+		));
+	}
+}

lib/Command/ExApp/Unregister.php

@@ -14,7 +14,9 @@
 use OCA\AppAPI\Service\AppAPIService;
 use OCA\AppAPI\Service\DaemonConfigService;
 use OCA\AppAPI\Service\ExAppDeployOptionsService;
+use OCA\AppAPI\Service\ExAppImageCleanupService;
 use OCA\AppAPI\Service\ExAppService;
+use OCA\AppAPI\Service\ImageCleanupChoice;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
@@ -30,6 +32,7 @@ public function __construct(
 		private readonly KubernetesActions $kubernetesActions,
 		private readonly ExAppService $exAppService,
 		private readonly ExAppDeployOptionsService $exAppDeployOptionsService,
+		private readonly ExAppImageCleanupService $imageCleanupService,
 	) {
 		parent::__construct();
 	}
@@ -52,18 +55,29 @@ protected function configure(): void {
 			'Continue removal even if errors.');
 		$this->addOption('keep-data', null, InputOption::VALUE_NONE, 'Keep ExApp data (volume) [deprecated, data is kept by default].');
 		$this->addOption('rm-data', null, InputOption::VALUE_NONE, 'Remove ExApp data (persistent storage volume).');
+		$this->addOption('purge-now', null, InputOption::VALUE_NONE, 'Delete the ExApp Docker image immediately, without the configured grace period.');
+		$this->addOption('keep-image', null, InputOption::VALUE_NONE, 'Do not schedule any image cleanup; admin will remove the image manually.');
 
 		$this->addUsage('test_app');
 		$this->addUsage('test_app --silent');
 		$this->addUsage('test_app --rm-data');
 		$this->addUsage('test_app --silent --force --rm-data');
+		$this->addUsage('test_app --purge-now');
+		$this->addUsage('test_app --keep-image');
 	}
 
 	protected function execute(InputInterface $input, OutputInterface $output): int {
 		$appId = $input->getArgument('appid');
 		$silent = $input->getOption('silent');
 		$force = $input->getOption('force');
 		$rmData = $input->getOption('rm-data');
+		$purgeNow = (bool)$input->getOption('purge-now');
+		$keepImage = (bool)$input->getOption('keep-image');
+		if ($purgeNow && $keepImage) {
+			$output->writeln('<error>--purge-now and --keep-image are mutually exclusive.</error>');
+			return 1;
+		}
+		$cleanupChoice = ImageCleanupChoice::fromFlags($purgeNow, $keepImage);
 
 		$exApp = $this->exAppService->getExApp($appId);
 		if ($exApp === null) {
@@ -102,6 +116,10 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 			if ($daemonConfig->getAcceptsDeployId() === $this->dockerActions->getAcceptsDeployId()) {
 				$this->dockerActions->initGuzzleClient($daemonConfig);
 
+				// Capture the image ref BEFORE the container is removed; we still need
+				// to know it after the removal so the cleanup job has something to delete.
+				$capturedImageRef = $this->imageCleanupService->captureImageRef($daemonConfig, $appId, $cleanupChoice);
+
 				if (boolval($exApp->getDeployConfig()['harp'] ?? false)) {
 					if ($this->dockerActions->removeExApp($this->dockerActions->buildDockerUrl($daemonConfig), $exApp->getAppid(), removeData: $rmData)) {
 						if (!$silent) {
@@ -111,10 +129,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 						if (!$force) {
 							return 1;
 						}
-					} else {
-						if (!$silent) {
-							$output->writeln(sprintf('ExApp %s successfully removed', $appId));
-						}
+					} elseif (!$silent) {
+						$output->writeln(sprintf('ExApp %s successfully removed', $appId));
 					}
 				} else {
 					$containerName = $this->dockerActions->buildExAppContainerName($appId);
@@ -146,6 +162,15 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 						}
 					}
 				}
+
+				// Act on the captured ref now that the container is gone. If removal
+				// failed above (--force path), Docker's in-use 409 keeps the image safe.
+				$this->imageCleanupService->scheduleCleanup(
+					$capturedImageRef,
+					$exApp,
+					$daemonConfig,
+					$cleanupChoice,
+				);
 			} elseif ($daemonConfig->getAcceptsDeployId() === $this->kubernetesActions->getAcceptsDeployId()) {
 				$this->kubernetesActions->initGuzzleClient($daemonConfig);
 				$harpK8sUrl = $this->kubernetesActions->buildHarpK8sUrl($daemonConfig);

lib/Command/ExApp/Update.php

@@ -17,7 +17,9 @@
 use OCA\AppAPI\Service\AppAPIService;
 use OCA\AppAPI\Service\DaemonConfigService;
 use OCA\AppAPI\Service\ExAppDeployOptionsService;
+use OCA\AppAPI\Service\ExAppImageCleanupService;
 use OCA\AppAPI\Service\ExAppService;
+use OCA\AppAPI\Service\ImageCleanupChoice;
 use Psr\Log\LoggerInterface;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputArgument;
@@ -38,6 +40,7 @@ public function __construct(
 		private readonly ExAppArchiveFetcher $exAppArchiveFetcher,
 		private readonly ExAppFetcher $exAppFetcher,
 		private readonly ExAppDeployOptionsService $exAppDeployOptionsService,
+		private readonly ExAppImageCleanupService $imageCleanupService,
 	) {
 		parent::__construct();
 	}
@@ -56,6 +59,8 @@ protected function configure(): void {
 		$this->addOption('all', null, InputOption::VALUE_NONE, 'Updates all enabled and updatable apps');
 		$this->addOption('showonly', null, InputOption::VALUE_NONE, 'Additional flag for "--all" to only show all updatable apps');
 		$this->addOption('include-disabled', null, InputOption::VALUE_NONE, 'Additional flag for "--all" to also update disabled apps');
+		$this->addOption('purge-old-image-now', null, InputOption::VALUE_NONE, 'Delete the old ExApp Docker image immediately after update, without the configured grace period.');
+		$this->addOption('keep-old-image', null, InputOption::VALUE_NONE, 'Do not schedule any cleanup of the old ExApp image; admin will remove it manually.');
 	}
 
 	protected function execute(InputInterface $input, OutputInterface $output): int {
@@ -66,6 +71,9 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 		} elseif (!empty($appId) && $input->getOption('all')) {
 			$output->writeln('<error>The "--all" flag is mutually exclusive with specifying app</error>');
 			return 1;
+		} elseif ((bool)$input->getOption('purge-old-image-now') && (bool)$input->getOption('keep-old-image')) {
+			$output->writeln('<error>--purge-old-image-now and --keep-old-image are mutually exclusive.</error>');
+			return 1;
 		} elseif ($input->getOption('all')) {
 			$apps = $this->exAppFetcher->get();
 			$appsWithUpdates = array_filter($apps, function (array $app) {
@@ -208,6 +216,14 @@ private function updateExApp(InputInterface $input, OutputInterface $output, str
 		$harpK8sUrl = null;
 		$k8sRoles = [];
 		if ($daemonConfig->getAcceptsDeployId() === $this->dockerActions->getAcceptsDeployId()) {
+			$cleanupChoice = ImageCleanupChoice::fromFlags(
+				(bool)$input->getOption('purge-old-image-now'),
+				(bool)$input->getOption('keep-old-image'),
+			);
+			// Capture the OLD image ref before the deploy replaces the container.
+			// We schedule cleanup for it after the new image is verified running.
+			$oldImageRef = $this->imageCleanupService->captureImageRef($daemonConfig, $appId, $cleanupChoice);
+
 			$deployParams = $this->dockerActions->buildDeployParams($daemonConfig, $appInfo);
 			if (boolval($exApp->getDeployConfig()['harp'] ?? false)) {
 				$deployResult = $this->dockerActions->deployExAppHarp($exApp, $daemonConfig, $deployParams);
@@ -232,6 +248,14 @@ private function updateExApp(InputInterface $input, OutputInterface $output, str
 				return 1;
 			}
 
+			// Deploy + healthcheck both succeeded; safe to schedule cleanup of the old ref.
+			$this->imageCleanupService->scheduleCleanup(
+				$oldImageRef,
+				$exApp,
+				$daemonConfig,
+				$cleanupChoice,
+			);
+
 			$exAppUrl = $this->dockerActions->resolveExAppUrl(
 				$appId,
 				$daemonConfig->getProtocol(),

lib/Controller/ConfigController.php

@@ -32,7 +32,7 @@ public function __construct(
 	/**
 	 * Update the AppAPI admin configuration
 	 *
-	 * @param array<string, string> $values Configuration key-value pairs to store
+	 * @param array<string, bool|int|string> $values Configuration key-value pairs to store
 	 *
 	 * @return DataResponse<Http::STATUS_OK, int, array{}>
 	 *
@@ -41,7 +41,26 @@ public function __construct(
 	#[NoCSRFRequired]
 	public function setAdminConfig(array $values): DataResponse {
 		foreach ($values as $key => $value) {
-			$this->appConfig->setValueString(Application::APP_ID, $key, $value, lazy: true);
+			// The image-cleanup settings are stored typed (bool/int) so the read side can use
+			// getValueBool/getValueInt. Coerce by known key, not by incoming value type: typing
+			// by value would let a single API call flip the stored type of a legacy string
+			// setting (e.g. init_timeout sent as int) and make every later typed read of it
+			// throw AppConfigTypeConflictException.
+			if ($key === Application::CONF_IMAGE_CLEANUP_ENABLED) {
+				$enabled = filter_var($value, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE);
+				if ($enabled === null) {
+					continue; // unrecognized input must not silently flip the switch
+				}
+				$this->appConfig->setValueBool(Application::APP_ID, $key, $enabled, lazy: true);
+			} elseif ($key === Application::CONF_IMAGE_CLEANUP_GRACE_HOURS) {
+				if (!is_numeric($value)) {
+					continue;
+				}
+				$hours = max(0, min(Application::MAX_IMAGE_CLEANUP_GRACE_HOURS, (int)$value));
+				$this->appConfig->setValueInt(Application::APP_ID, $key, $hours, lazy: true);
+			} else {
+				$this->appConfig->setValueString(Application::APP_ID, $key, (string)$value, lazy: true);
+			}
 		}
 		return new DataResponse(1);
 	}