feat(concourse): switch remaining Pulumi pipelines to preview-gated topology

pyproject.toml

@@ -47,7 +47,7 @@ dependencies = [
   "pulumi-rootly",
   "pulumi-terraform>=6.0.1",
   "pulumi-command>=1.0,<2",
-  "ol-concourse>=0.16,<0.17",
+  "ol-concourse>=0.18,<0.19",
   "pulumiverse-grafana>=2.0.0,<3",
   "pulumiverse-sentry==0.0.9",
   "pyjwt[crypto]>=2.9,<3",

src/ol_concourse/pipelines/infrastructure/aws/pipeline.py

@@ -32,6 +32,8 @@
     simple_pulumi_chain = pulumi_jobs_chain(
         simple_pulumi_code,
         refresh_stack=True,
+        topology="preview-gated",
+        auto_deploy_stages=["CI"],
         project_name=_SIMPLE_SERVICE_PROJECT_NAMES[service],
         stack_names=["CI", "QA", "Production"],
         project_source_path=PULUMI_CODE_PATH.joinpath(f"infrastructure/aws/{service}/"),
@@ -73,6 +75,7 @@
     oneoff_pulumi_chain = pulumi_jobs_chain(
         oneoff_pulumi_code,
         refresh_stack=True,
+        topology="preview-gated",
         project_name=f"ol-infrastructure-aws-{service}",
         stack_names=["default"],
         project_source_path=PULUMI_CODE_PATH.joinpath(f"infrastructure/aws/{service}/"),

src/ol_concourse/pipelines/infrastructure/concourse/pipeline.py

@@ -69,6 +69,8 @@
 concourse_pulumi_fragment = pulumi_jobs_chain(
     concourse_pulumi_code,
     refresh_stack=True,
+    topology="preview-gated",
+    auto_deploy_stages=["CI"],
     stack_names=["CI", "QA", "Production"],
     project_name="ol-infrastructure-concourse-application",
     project_source_path=PULUMI_CODE_PATH.joinpath("applications/concourse/"),

src/ol_concourse/pipelines/infrastructure/consul/pipeline.py

@@ -72,6 +72,8 @@
     consul_pulumi_infrastructure_fragment = pulumi_jobs_chain(
         consul_pulumi_infrastructure_code,
         refresh_stack=True,
+        topology="preview-gated",
+        auto_deploy_stages=["CI"],
         project_name="ol-infrastructure-consul",
         stack_names=[f"{network}.{stage}" for stage in stages],
         project_source_path=PULUMI_CODE_PATH.joinpath("infrastructure/consul/"),
@@ -91,6 +93,8 @@
     consul_pulumi_substructure_fragment = pulumi_jobs_chain(
         consul_pulumi_substructure_code,
         refresh_stack=True,
+        topology="preview-gated",
+        auto_deploy_stages=["CI"],
         project_name="ol-substructure-consul",
         stack_names=[f"{network}.{stage}" for stage in stages],
         project_source_path=PULUMI_CODE_PATH.joinpath("substructure/consul/"),

src/ol_concourse/pipelines/infrastructure/dagster/pipeline.py

@@ -242,6 +242,7 @@ def build_dagster_docker_pipeline() -> Pipeline:
     pulumi_fragment = pulumi_jobs_chain(
         pulumi_code,
         refresh_stack=True,
+        topology="preview-gated",
         stack_names=["QA", "Production"],
         project_name="ol-infrastructure-dagster-server",
         project_source_path=PULUMI_CODE_PATH.joinpath("applications/dagster/"),

src/ol_concourse/pipelines/infrastructure/jupyterhub/pipeline.py

@@ -24,6 +24,8 @@
 jupyter_pulumi_fragment = pulumi_jobs_chain(
     jupyterhub_pulumi_code,
     refresh_stack=True,
+    topology="preview-gated",
+    auto_deploy_stages=["CI"],
     stack_names=["CI", "QA", "Production"],
     project_name="ol-infrastructure-jupyterhub",
     project_source_path=PULUMI_CODE_PATH.joinpath("applications/jupyterhub/"),

src/ol_concourse/pipelines/infrastructure/kubewatch/pipeline.py

@@ -21,6 +21,14 @@
 from ol_concourse.pipelines.secrets_map import project_secrets_paths
 from ol_concourse.pipelines.versions_map import project_version_paths
 
+# Shared between the two pipeline builders below so the deploy-job lookup in
+# build_kubewatch_pipeline() derives its expected name from the same values
+# passed to pulumi_jobs_chain(), rather than duplicating them as a literal.
+_WEBHOOK_HANDLER_PROJECT_NAME = "ol-infrastructure-kubewatch-webhook-handler"
+_WEBHOOK_HANDLER_STACK_NAMES = [
+    f"applications.{env}" for env in ("CI", "QA", "Production")
+]
+
 
 def build_kubewatch_webhook_handler_pipeline() -> PipelineFragment:
     """Build pipeline for kubewatch webhook handler.
@@ -77,9 +85,11 @@ def build_kubewatch_webhook_handler_pipeline() -> PipelineFragment:
 
     webhook_handler_fragment = pulumi_jobs_chain(
         refresh_stack=True,
+        topology="preview-gated",
+        auto_deploy_stages=["CI"],
         pulumi_code=webhook_handler_pulumi_code,
-        stack_names=[f"applications.{env}" for env in ("CI", "QA", "Production")],
-        project_name="ol-infrastructure-kubewatch-webhook-handler",
+        stack_names=_WEBHOOK_HANDLER_STACK_NAMES,
+        project_name=_WEBHOOK_HANDLER_PROJECT_NAME,
         project_source_path=PULUMI_CODE_PATH.joinpath(
             "applications/kubewatch_webhook_handler/"
         ),
@@ -121,10 +131,27 @@ def build_kubewatch_pipeline() -> PipelineFragment:
     webhook_handler_fragment = build_kubewatch_webhook_handler_pipeline()
     webhook_handler_code = webhook_handler_fragment.resources[-1]
 
-    # Create dependencies for each kubewatch job to wait for webhook handler
+    # Create dependencies for each kubewatch job to wait for webhook handler.
+    # preview-gated expands each gated stage into a preview/deploy pair, so
+    # webhook_handler_fragment.jobs is no longer one job per environment --
+    # look up each stage's *deploy* job by name (derived the same way
+    # pulumi_jobs_chain names it: deploy-{project_name}-{slug}, slug being
+    # the stack name lowercased with dots replaced by dashes) rather than by
+    # position.
+    webhook_handler_jobs_by_name = {
+        str(job.name): job for job in webhook_handler_fragment.jobs
+    }
     custom_dependencies = {}
-    for idx, _env in enumerate(("CI", "QA", "Production")):
-        webhook_handler_job = webhook_handler_fragment.jobs[idx]
+    for idx, stack_name in enumerate(_WEBHOOK_HANDLER_STACK_NAMES):
+        slug = stack_name.lower().replace(".", "-")
+        deploy_job_name = f"deploy-{_WEBHOOK_HANDLER_PROJECT_NAME}-{slug}"
+        webhook_handler_job = webhook_handler_jobs_by_name.get(deploy_job_name)
+        if webhook_handler_job is None:
+            msg = (
+                f"Expected job {deploy_job_name!r} in the webhook handler "
+                f"fragment; got {sorted(webhook_handler_jobs_by_name)}"
+            )
+            raise ValueError(msg)
         custom_dependencies[idx] = [
             GetStep(
                 get=webhook_handler_code.name,
@@ -135,6 +162,8 @@ def build_kubewatch_pipeline() -> PipelineFragment:
 
     kubewatch_fragment = pulumi_jobs_chain(
         refresh_stack=True,
+        topology="preview-gated",
+        auto_deploy_stages=["CI"],
         pulumi_code=kubewatch_pulumi_code,
         stack_names=[f"applications.{env}" for env in ("CI", "QA", "Production")],
         project_name="ol-infrastructure-kubewatch",

src/ol_concourse/pipelines/infrastructure/ocw_site/pipeline.py

@@ -25,6 +25,7 @@
 
 pulumi_jobs = pulumi_jobs_chain(
     refresh_stack=True,
+    topology="preview-gated",
     pulumi_code=ocw_site_pulumi_code,
     stack_names=[
         "QA",

src/ol_concourse/pipelines/infrastructure/omnigraph/pipeline.py

@@ -135,6 +135,8 @@ def build_omnigraph_pipeline() -> PipelineFragment:
     # a promoted image actually changes each stage's Deployment pod spec.
     deploy_fragment = pulumi_jobs_chain(
         refresh_stack=True,
+        topology="preview-gated",
+        auto_deploy_stages=["CI"],
         pulumi_code=pulumi_code,
         stack_names=list(ENVIRONMENTS),
         project_name="ol-application-omnigraph",

src/ol_concourse/pipelines/infrastructure/simple_pulumi/pipeline.py

@@ -321,6 +321,8 @@ def _build_docker_image_job(
         app_name="airbyte",
         pulumi_project_path="applications/airbyte/",
         pulumi_project_name="ol-application-airbyte",
+        topology="preview-gated",
+        auto_deploy_stages=["CI"],
     ),
     "celery-monitoring": SimplePulumiParams(
         app_name="celery-monitoring",
@@ -331,29 +333,39 @@ def _build_docker_image_job(
             image_tag="0.7.5",  # Must match LEEK_VERSION in bridge.lib.versions
             ecr_region=ECR_REGION,
         ),
+        topology="preview-gated",
+        auto_deploy_stages=["CI"],
     ),
     "clickhouse": SimplePulumiParams(
         app_name="clickhouse",
         pulumi_project_path="applications/clickhouse/",
         pulumi_project_name="ol-application-clickhouse",
         stages=["CI", "QA", "Production"],
+        topology="preview-gated",
+        auto_deploy_stages=["CI"],
     ),
     "data_warehouse": SimplePulumiParams(
         app_name="data_warehouse",
         pulumi_project_path="infrastructure/aws/data_warehouse/",
         pulumi_project_name="ol-infrastructure-data-warehouse",
         stages=["CI", "QA", "Production"],
+        topology="preview-gated",
+        auto_deploy_stages=["CI"],
     ),
     "digital-credentials": SimplePulumiParams(
         app_name="digital-credentials",
         pulumi_project_path="applications/digital_credentials/",
         pulumi_project_name="ol-application-digital-credentials",
+        topology="preview-gated",
+        auto_deploy_stages=["CI"],
     ),
     "fastly-redirector": SimplePulumiParams(
         app_name="fastly-redirector",
         pulumi_project_path="applications/fastly_redirector/",
         pulumi_project_name="ol-application-fastly-redirector",
         refresh_stack=False,
+        topology="preview-gated",
+        auto_deploy_stages=["CI"],
     ),
     "gcp": SimplePulumiParams(
         app_name="gcp",
@@ -386,46 +398,59 @@ def _build_docker_image_job(
         app_name="grafana-alerting",
         pulumi_project_path="infrastructure/grafana_alerting/",
         pulumi_project_name="ol-infrastructure-grafana-alerting",
+        topology="preview-gated",
+        auto_deploy_stages=["CI"],
     ),
     "jupyterhub-data": SimplePulumiParams(
         app_name="jupyterhub-data",
         pulumi_project_path="applications/jupyterhub_data/",
         pulumi_project_name="ol-application-jupyterhub-data",
+        topology="preview-gated",
+        auto_deploy_stages=["CI"],
     ),
     "marimo-data": SimplePulumiParams(
         app_name="marimo-data",
         pulumi_project_path="applications/marimo_data/",
         pulumi_project_name="ol-application-marimo-data",
+        topology="preview-gated",
+        auto_deploy_stages=["CI"],
     ),
     "mongodb-atlas": SimplePulumiParams(
         app_name="mongodb-atlas",
         pulumi_project_path="infrastructure/mongodb_atlas/",
         pulumi_project_name="ol-infrastructure-mongodb-atlas",
         deployment_groups=["mitx", "mitx-staging", "mitxonline", "xpro"],
+        topology="preview-gated",
+        auto_deploy_stages=["CI"],
     ),
     "ocw-site": SimplePulumiParams(
         app_name="ocw-site",
         pulumi_project_path="applications/ocw_site/",
         pulumi_project_name="ol-application-ocw-site",
         stages=["QA", "Production"],
         refresh_stack=False,
+        topology="preview-gated",
     ),
     "open-discussions": SimplePulumiParams(
         app_name="open-discussions",
         pulumi_project_path="applications/open_discussions/",
         pulumi_project_name="ol-application-open-discussions",
         stages=["QA", "Production"],
+        topology="preview-gated",
     ),
     "open-metadata": SimplePulumiParams(
         app_name="open-metadata",
         pulumi_project_path="applications/open_metadata/",
         pulumi_project_name="ol-application-open-metadata",
+        topology="preview-gated",
+        auto_deploy_stages=["CI"],
     ),
     "open-metadata-substructure": SimplePulumiParams(
         app_name="open-metadata-substructure",
         pulumi_project_path="substructure/open_metadata/",
         pulumi_project_name="ol-substructure-open-metadata",
         stages=["QA", "Production"],
+        topology="preview-gated",
     ),
     "opensearch": SimplePulumiParams(
         app_name="opensearch",
@@ -442,18 +467,24 @@ def _build_docker_image_job(
             "open_metadata",
             "xpro",
         ],
+        topology="preview-gated",
+        auto_deploy_stages=["CI"],
     ),
     "opik": SimplePulumiParams(
         app_name="opik",
         pulumi_project_path="applications/opik/",
         pulumi_project_name="ol-application-opik",
         stages=["CI", "QA", "Production"],
+        topology="preview-gated",
+        auto_deploy_stages=["CI"],
     ),
     "qdrant-cloud": SimplePulumiParams(
         app_name="qdrant-cloud",
         pulumi_project_path="infrastructure/qdrant_cloud/",
         pulumi_project_name="ol-infrastructure-qdrant-cloud",
         stack_prefix="mitlearn",
+        topology="preview-gated",
+        auto_deploy_stages=["CI"],
     ),
     "rootly": SimplePulumiParams(
         app_name="rootly",
@@ -484,6 +515,8 @@ def _build_docker_image_job(
             "lakehouse",
         ],
         stages=["CI", "QA", "Production"],
+        topology="preview-gated",
+        auto_deploy_stages=["CI"],
     ),
     # The substructure stack makes direct TCP connections to the StarRocks NLB, which
     # is internal to the data VPC.  Each ops VPC is only peered with its same-env data
@@ -523,26 +556,36 @@ def _build_docker_image_job(
         app_name="tika",
         pulumi_project_path="applications/tika/",
         pulumi_project_name="ol-application-tika",
+        topology="preview-gated",
+        auto_deploy_stages=["CI"],
     ),
     "toolhive-apps": SimplePulumiParams(
         app_name="toolhive-apps",
         pulumi_project_path="applications/toolhive_apps/",
         pulumi_project_name="ol-application-toolhive-apps",
+        topology="preview-gated",
+        auto_deploy_stages=["CI"],
     ),
     "toolhive-data": SimplePulumiParams(
         app_name="toolhive-data",
         pulumi_project_path="applications/toolhive_data/",
         pulumi_project_name="ol-application-toolhive-data",
+        topology="preview-gated",
+        auto_deploy_stages=["CI"],
     ),
     "toolhive-operator": SimplePulumiParams(
         app_name="toolhive-operator",
         pulumi_project_path="applications/toolhive_operator/",
         pulumi_project_name="ol-application-toolhive-operator",
+        topology="preview-gated",
+        auto_deploy_stages=["CI"],
     ),
     "toolhive-swe": SimplePulumiParams(
         app_name="toolhive-swe",
         pulumi_project_path="applications/toolhive_swe/",
         pulumi_project_name="ol-application-toolhive-swe",
+        topology="preview-gated",
+        auto_deploy_stages=["CI"],
     ),
     # NOTE: omnigraph and witan are deliberately NOT simple_pulumi (deploy-only)
     # apps. Each builds a container image from a foreign repo (agent-kit) and
@@ -553,6 +596,8 @@ def _build_docker_image_job(
         pulumi_project_path="infrastructure/vector_log_proxy/",
         pulumi_project_name="ol-infrastructure-vector-log-proxy",
         stack_prefix="operations",
+        topology="preview-gated",
+        auto_deploy_stages=["CI"],
     ),
     "aws-ecr": SimplePulumiParams(
         app_name="aws-ecr",
@@ -570,16 +615,21 @@ def _build_docker_image_job(
         # barely changed in a year never needed one. The default CI stage
         # was failing every run with StackNotFoundError.
         stages=["QA", "Production"],
+        topology="preview-gated",
     ),
     "b2b-partners-storage": SimplePulumiParams(
         app_name="b2b-partners-storage",
         pulumi_project_path="applications/b2b_partners_storage/",
         pulumi_project_name="ol-application-b2b-partners-storage",
+        topology="preview-gated",
+        auto_deploy_stages=["CI"],
     ),
     "mailgun": SimplePulumiParams(
         app_name="mailgun",
         pulumi_project_path="applications/mailgun/",
         pulumi_project_name="ol-application-mailgun",
+        topology="preview-gated",
+        auto_deploy_stages=["CI"],
     ),
     "monitoring": SimplePulumiParams(
         app_name="monitoring",

src/ol_concourse/pipelines/infrastructure/superset/pipeline.py

@@ -130,6 +130,8 @@ def build_superset_docker_pipeline() -> Pipeline:
     pulumi_fragment = pulumi_jobs_chain(
         pulumi_code_repo,
         refresh_stack=True,
+        topology="preview-gated",
+        auto_deploy_stages=["CI"],
         stack_names=["CI", "QA", "Production"],
         project_name="ol-infrastructure-superset-server",
         project_source_path=PULUMI_CODE_PATH.joinpath("applications/superset/"),

src/ol_concourse/pipelines/infrastructure/vault/pipeline.py

@@ -88,6 +88,8 @@
 vault_pulumi_fragment = pulumi_jobs_chain(
     vault_pulumi_code,
     refresh_stack=True,
+    topology="preview-gated",
+    auto_deploy_stages=["CI"],
     project_name="ol-infrastructure-vault-server",
     stack_names=[f"operations.{stage}" for stage in ("CI", "QA", "Production")],
     project_source_path=PULUMI_CODE_PATH.joinpath("infrastructure/vault/"),
@@ -107,6 +109,8 @@
         pulumi_jobs_chain(
             vault_pulumi_substructure_code,
             refresh_stack=True,
+            topology="preview-gated",
+            auto_deploy_stages=["CI"],
             project_name=f"ol-substructure-vault-{substructure}",
             stack_names=[f"operations.{stage}" for stage in ("CI", "QA", "Production")],
             project_source_path=PULUMI_CODE_PATH.joinpath(