feat(deploy): Revamp deploy flow

.cspell.json

@@ -777,6 +777,9 @@
         "lsyncd",
         "awk",
         "gawk",
+        "picklable",
+        "ndarray",
+        "mult",
         "ruleid"
     ]
 }

dashboard/src/components/group/UpdateReleaseGroupDialog.vue

@@ -536,17 +536,9 @@ export default {
 						throw new DashboardError('Please select an app to proceed');
 					}
 				},
-				onSuccess(candidate) {
-					this.$router.push({
-						name: 'Deploy Candidate',
-						params: {
-							id: candidate,
-							name: this.bench,
-						},
-					});
-					this.restrictMessage = '';
+				onSuccess() {
 					this.show = false;
-					this.$emit('success', candidate);
+					this.$emit('success', null);
 				},
 				onError: this.setErrorMessage.bind(this),
 			};

dashboard/src/objects/group.js

@@ -10,8 +10,8 @@ import PatchAppDialog from '../components/group/PatchAppDialog.vue';
 import { getTeam, switchToTeam } from '../data/team';
 import router from '../router';
 import { confirmDialog, icon, renderDialog } from '../utils/components';
-import { getToastErrorMessage } from '../utils/toast';
 import { date, duration } from '../utils/format';
+import { getToastErrorMessage } from '../utils/toast';
 import { getJobsTab } from './common/jobs';
 import { getPatchesTab } from './common/patches';
 import { tagTab } from './common/tags';
@@ -510,14 +510,17 @@ export default {
 								} else if (group.doc.deploy_information.update_available) {
 									let UpdateReleaseGroupDialog = defineAsyncComponent(
 										() =>
-											import('../components/group/UpdateReleaseGroupDialog.vue'),
+											import(
+												'../components/group/UpdateReleaseGroupDialog.vue'
+											),
 									);
 									renderDialog(
 										h(UpdateReleaseGroupDialog, {
 											bench: group.name,
 											lastDeploy: true,
 											onSuccess(candidate) {
-												group.doc.deploy_information.deploy_in_progress = true;
+												group.doc.deploy_information.has_running_release_pipeline = true;
+												group.doc.deploy_information.update_available = false;
 												if (candidate) {
 													group.doc.deploy_information.last_deploy.name =
 														candidate;
@@ -830,7 +833,9 @@ export default {
 								onClick() {
 									let ConfigEditorDialog = defineAsyncComponent(
 										() =>
-											import('../components/EnvironmentVariableEditorDialog.vue'),
+											import(
+												'../components/EnvironmentVariableEditorDialog.vue'
+											),
 									);
 									renderDialog(
 										h(ConfigEditorDialog, {
@@ -921,7 +926,8 @@ export default {
 								bench: group.name,
 								lastDeploy: group.doc?.deploy_information?.last_deploy,
 								onSuccess(candidate) {
-									group.doc.deploy_information.deploy_in_progress = true;
+									group.doc.deploy_information.has_running_release_pipeline = true;
+									group.doc.deploy_information.update_available = false;
 									if (candidate) {
 										group.doc.deploy_information.last_deploy = {
 											name: candidate,
@@ -932,6 +938,17 @@ export default {
 						);
 					},
 				},
+				{
+					label: 'Validating Deploy',
+					slots: {
+						prefix: () => h(LoadingIndicator, { class: 'w-4 h-4' }),
+					},
+					theme: 'green',
+					condition: () =>
+						!group.doc.deploy_information.deploy_in_progress &&
+						!group.doc.deploy_information.bench_creation_underway &&
+						group.doc.deploy_information.has_running_release_pipeline,
+				},
 				{
 					label: 'Deploy in progress',
 					slots: {

press/api/bench.py

@@ -44,6 +44,7 @@
 	from press.press.doctype.deploy_candidate.deploy_candidate import DeployCandidate
 	from press.press.doctype.deploy_candidate_build.deploy_candidate_build import DeployCandidateBuild
 	from press.press.doctype.marketplace_app.marketplace_app import MarketplaceApp
+	from press.press.doctype.release_pipeline.release_pipeline import ReleasePipeline
 
 
 @frappe.whitelist()
@@ -390,15 +391,18 @@ def dependencies(name: str):
 @frappe.whitelist()
 @protected("Release Group")
 def update_dependencies(name: str, dependencies: str):
-	dependencies = frappe.parse_json(dependencies)
+	dependencies_dict: list[frappe._dict] = frappe.parse_json(dependencies)
 	rg: ReleaseGroup = frappe.get_doc("Release Group", name)
-	if len(rg.dependencies) != len(dependencies):
+
+	if len(rg.dependencies) != len(dependencies_dict):
 		frappe.throw("Need all required dependencies")
-	if diff := set([d["key"] for d in dependencies]) - set(d.dependency for d in rg.dependencies):
+
+	if diff := set([d["key"] for d in dependencies_dict]) - set(d.dependency for d in rg.dependencies):
 		frappe.throw("Invalid dependencies: " + ", ".join(diff))
+
 	for dep, new in zip(
 		sorted(rg.dependencies, key=lambda x: x.dependency),
-		sorted(dependencies, key=lambda x: x["key"]),
+		sorted(dependencies_dict, key=lambda x: x["key"]),
 		strict=False,
 	):
 		if dep.dependency != new["key"]:
@@ -798,15 +802,20 @@ def deploy_and_update(
 	sites: list | None = None,
 	run_will_fail_check: bool = True,
 ):
-	validate_app_hashes(apps)
+	# We check permissions early on and don't change permissions in the middle of the Workflow
+	current_team = get_current_team()
+	rg_team = frappe.db.get_value("Release Group", name, "team")
 
-	# Returns name of the Deploy Candidate that is running the build
-	return get_bench_update(
-		name,
-		apps,
-		sites,
-		False,
-	).deploy(run_will_fail_check)
+	if rg_team != current_team:
+		frappe.throw("Bench can only be deployed by the bench owner", exc=frappe.PermissionError)
+
+	release_pipeline: ReleasePipeline = frappe.get_doc(
+		{"doctype": "Release Pipeline", "release_group": name, "team": current_team}
+	)
+	release_pipeline.insert()
+	release_pipeline.create_release.run_as_workflow(
+		apps=apps, sites=sites, run_will_fail_check=run_will_fail_check
+	)
 
 
 @frappe.whitelist()
@@ -936,9 +945,9 @@ def validate_branch(name: str, app: str, branch: str):
 def get_branches_for_marketplace_app(app: str, marketplace_app: str, app_source: AppSource) -> list[dict]:
 	"""Return list of branches allowed for this `marketplace` app"""
 	branch_set = set()
-	marketplace_app: MarketplaceApp = frappe.get_doc("Marketplace App", marketplace_app)
+	marketplace_app_doc: MarketplaceApp = frappe.get_doc("Marketplace App", marketplace_app)
 
-	for marketplace_app_source in marketplace_app.sources:
+	for marketplace_app_source in marketplace_app_doc.sources:
 		app_source = frappe.get_doc("App Source", marketplace_app_source.source)
 		branch_set.add(app_source.branch)
 
@@ -1146,10 +1155,10 @@ def show_app_versions(name: str, dc_name: str) -> list[dict[str, Any]]:
 		{
 			"name": app.app,
 			"hash": app.hash[:7],
-			"branch": sources.get(app.source).get("branch"),
-			"repository": sources.get(app.source).get("repository"),
-			"repository_owner": sources.get(app.source).get("repository_owner"),
-			"repository_url": sources.get(app.source).get("repository_url"),
+			"branch": sources.get(app.source, {}).get("branch"),
+			"repository": sources.get(app.source, {}).get("repository"),
+			"repository_owner": sources.get(app.source, {}).get("repository_owner"),
+			"repository_url": sources.get(app.source, {}).get("repository_url"),
 		}
 		for app in deploy_candidate.apps
 		if app

press/api/tests/test_bench.py

@@ -147,31 +147,6 @@ def test_deploy_and_update_fn_creates_bench_update(self):
 		self.assertEqual(dc_count_after, dc_count_before + 1)
 		self.assertEqual(bu_count_after, bu_count_before + 1)
 
-	@patch(
-		"press.press.doctype.deploy_candidate.deploy_candidate.frappe.enqueue_doc",
-		new=foreground_enqueue_doc,
-	)
-	@patch("press.press.doctype.deploy_candidate.deploy_candidate.frappe.db.commit", new=Mock())
-	def test_deploy_and_update_fn_fails_without_release_argument(self):
-		group = new(
-			{
-				"title": "Test Bench",
-				"apps": [{"name": self.app.name, "source": self.app_source.name}],
-				"version": self.version,
-				"cluster": "Default",
-				"saas_app": None,
-				"server": None,
-			}
-		)
-
-		self.assertRaises(
-			frappe.exceptions.ValidationError,
-			deploy_and_update,
-			group,
-			[{"app": self.app.name}],
-			[],
-		)
-
 	@patch("press.press.doctype.deploy_candidate.deploy_candidate.frappe.db.commit", new=Mock())
 	def test_deploy_fn_fails_without_apps(self):
 		frappe.set_user(self.team.user)

press/exceptions.py

@@ -103,3 +103,7 @@ class ArchiveBenchError(ValidationError):
 
 class MonitorServerDown(ValidationError):
 	pass
+
+
+class ReleasePipelineFailure(Exception):
+	pass

press/hooks.py

@@ -353,6 +353,7 @@
 		],
 		"* * * * *": [
 			"press.press.doctype.virtual_disk_snapshot.virtual_disk_snapshot.sync_physical_backup_snapshots",
+			"press.workflow_engine.doctype.press_workflow_task.press_workflow_task.retry_tasks",
 			"press.press.doctype.deploy_candidate_build.deploy_candidate_build.run_scheduled_builds",
 			"press.press.doctype.agent_request_failure.agent_request_failure.remove_old_failures",
 			"press.saas.doctype.site_access_token.site_access_token.cleanup_expired_access_tokens",
@@ -365,6 +366,7 @@
 			"press.press.doctype.site.saas_pool.create",
 			"press.press.doctype.virtual_disk_snapshot.virtual_disk_snapshot.sync_rolling_snapshots",
 			"press.press.doctype.database_server.database_server.auto_purge_binlogs_by_size_limit",
+			"press.workflow_engine.doctype.press_workflow.press_workflow.retry_workflows",
 		],
 		"*/30 * * * *": [
 			"press.press.doctype.site_update.scheduled_auto_updates.trigger",

press/modules.txt

@@ -4,4 +4,5 @@ Marketplace
 SaaS
 Partner
 Infrastructure
-Incident Management
+Incident Management
+Workflow Engine

press/press/doctype/bench/bench.py

@@ -1317,27 +1317,23 @@ def cancel_and_retry_bench_job_if_required(job: AgentJob) -> bool:
 	if not has_retryable_error:
 		return False
 
-	job.cancel_job()
-
-	frappe.db.set_value("Agent Job", job.name, "status", "Failure")
-	frappe.db.set_value("Bench", job.bench, "status", "Broken")
-
-	# Trigger immediate archival of bench to allow retry
 	bench: Bench = frappe.get_doc("Bench", job.bench)
-	bench.archive(retry_new_bench=True)
-	return True
-
-
-def retry_new_bench_job_if_possible(bench: Bench):
-	"""Check if there are retries left, if yes then trigger a new bench job immediately."""
 	retry_count = frappe.db.count(
 		"Bench", {"build": bench.build, "server": bench.server, "group": bench.group}
 	)
 
 	if retry_count >= 3:
-		return
+		# We can't retry anymore so accept the fate and proceed with archival with job processing
+		return False
 
-	bench.retry_bench()
+	job.cancel_job()
+
+	frappe.db.set_value("Agent Job", job.name, "status", "Failure")
+	frappe.db.set_value("Bench", job.bench, "status", "Broken")
+
+	bench = bench.reload()
+	bench.archive(retry_new_bench=True)
+	return True
 
 
 def process_new_bench_job_update(job: AgentJob):  # noqa: C901
@@ -1445,7 +1441,7 @@ def process_archive_bench_job_update(job: AgentJob):
 	retry_new_bench = request_data.get("retry_new_bench", False)
 
 	if updated_status == "Archived" and retry_new_bench:
-		retry_new_bench_job_if_possible(bench)
+		bench.retry_bench()  # We know now for sure that the bench can be retired
 
 
 def process_add_ssh_user_job_update(job):

press/press/doctype/bench_update/bench_update.py

@@ -84,10 +84,19 @@ def validate_inplace_update(self):
 				frappe.ValidationError,
 			)
 
-	def deploy(self, run_will_fail_check=False) -> str:
+	def deploy(
+		self,
+		run_will_fail_check=False,
+		validate_pre_candidate_checks: bool = True,
+		create_build: bool = True,
+	) -> str:
+		"""Creates and returns candidate name or build name depending on the point of invocation."""
 		rg: ReleaseGroup = frappe.get_doc("Release Group", self.group)
-		candidate = rg.create_deploy_candidate(self.apps, run_will_fail_check)
-		deploy = candidate.schedule_build_and_deploy()
+		candidate = rg.create_deploy_candidate(
+			apps_to_update=self.apps,
+			run_will_fail_check=run_will_fail_check,
+			validate_pre_candidate_checks=validate_pre_candidate_checks,
+		)
 
 		self.candidate = candidate.name
 		self.save()
@@ -97,6 +106,12 @@ def deploy(self, run_will_fail_check=False) -> str:
 				f"Invalid name found for deploy candidate '{candidate.name}' of type {type(candidate.name)}"
 			)
 
+		if not create_build:
+			# In case we are not scheduling build from here (eg. new build flow) return candidate name here
+			return candidate.name
+
+		deploy = candidate.schedule_build_and_deploy()
+
 		return deploy["name"]
 
 	def update_inplace(self) -> str:
@@ -164,15 +179,17 @@ def get_bench_update(
 	apps: list,
 	sites: list | None = None,
 	is_inplace_update: bool = False,
+	ignore_permissions_check: bool = False,
 ) -> BenchUpdate:
 	if sites is None:
 		sites = []
 
-	current_team = get_current_team()
-	rg_team = frappe.db.get_value("Release Group", name, "team")
+	if not ignore_permissions_check:
+		current_team = get_current_team()
+		rg_team = frappe.db.get_value("Release Group", name, "team")
 
-	if rg_team != current_team:
-		frappe.throw("Bench can only be deployed by the bench owner", exc=frappe.PermissionError)
+		if rg_team != current_team:
+			frappe.throw("Bench can only be deployed by the bench owner", exc=frappe.PermissionError)
 
 	bench_update: "BenchUpdate" = frappe.get_doc(
 		{
@@ -192,4 +209,5 @@ def get_bench_update(
 			"is_inplace_update": is_inplace_update,
 		}
 	).insert(ignore_permissions=True)
+
 	return bench_update

press/press/doctype/deploy_candidate_build/deploy_candidate_build.py

@@ -1167,6 +1167,7 @@ def pre_build(self, **kwargs):
 			queue=queue,
 			timeout=2400,
 			enqueue_after_commit=True,
+			job_id=f"deploy_candidate_build:{self.name}",
 		)
 
 		frappe.set_user(user)