fix(webapp): extend mint-kind flip grace to global flips

d-cs · d-cs · commit 32e46e935069 · 2026-07-10T15:53:02.000+01:00
Global runOpsMintKind flips now stamp the previous kind and flip time
like per-org flips, so a global flip is a deterministic cutover and
cannot route two concurrent triggers that share an idempotency key to
different databases. The resolver reads the grace stamp from the same
source as the kind.
diff --git a/apps/webapp/app/routes/admin.api.v1.feature-flags.ts b/apps/webapp/app/routes/admin.api.v1.feature-flags.ts
@@ -1,9 +1,11 @@
 import type { ActionFunctionArgs } from "@remix-run/server-runtime";
 import { json } from "@remix-run/server-runtime";
 import { prisma } from "~/db.server";
+import { env } from "~/env.server";
 import { requireAdminApiRequest } from "~/services/personalAccessToken.server";
 import { makeSetMultipleFlags } from "~/v3/featureFlags.server";
-import { validatePartialFeatureFlags } from "~/v3/featureFlags";
+import { FEATURE_FLAG, type FeatureFlagCatalog, validatePartialFeatureFlags } from "~/v3/featureFlags";
+import { stampMintKindFlip } from "~/v3/runOpsMigration/mintFlipGrace";
 
 export async function action({ request }: ActionFunctionArgs) {
   await requireAdminApiRequest(request);
@@ -24,9 +26,50 @@ export async function action({ request }: ActionFunctionArgs) {
       );
     }
 
-    const featureFlags = validationResult.data;
+    // Derived grace-stamp fields are computed server-side; never trust them from the body.
+    const {
+      runOpsMintKindPrev: _ignoredPrev,
+      runOpsMintKindFlippedAt: _ignoredFlippedAt,
+      ...requestedFlags
+    } = validationResult.data;
+
+    let flagsToWrite: Partial<FeatureFlagCatalog> = requestedFlags;
+
+    if (requestedFlags.runOpsMintKind !== undefined) {
+      // Read the current GLOBAL mint flags so the stamp is computed against the authoritative
+      // stored state, mirroring the per-org route. stampMintKindFlip writes prev/flippedAt only
+      // on a genuine global flip, and carries an in-flight stamp forward on a same-target save.
+      const existingRows = await prisma.featureFlag.findMany({
+        where: {
+          key: {
+            in: [
+              FEATURE_FLAG.runOpsMintKind,
+              FEATURE_FLAG.runOpsMintKindPrev,
+              FEATURE_FLAG.runOpsMintKindFlippedAt,
+            ],
+          },
+        },
+        select: { key: true, value: true },
+      });
+      const existingGlobal: Record<string, unknown> = {};
+      for (const row of existingRows) {
+        existingGlobal[row.key] = row.value;
+      }
+
+      // Anchor the cutover to the control-plane DB clock, not this process's wall clock.
+      const [{ now: controlPlaneNow }] =
+        await prisma.$queryRaw<{ now: Date }[]>`SELECT now() AS now`;
+
+      flagsToWrite = stampMintKindFlip(
+        existingGlobal,
+        { ...requestedFlags },
+        controlPlaneNow.getTime(),
+        env.RUN_OPS_MINT_FLIP_GRACE_MS
+      ) as Partial<FeatureFlagCatalog>;
+    }
+
     const setMultipleFlags = makeSetMultipleFlags(prisma);
-    const updatedFlags = await setMultipleFlags(featureFlags);
+    const updatedFlags = await setMultipleFlags(flagsToWrite);
 
     return json({
       success: true,
diff --git a/apps/webapp/app/v3/runOpsMigration/mintFlipGrace.test.ts b/apps/webapp/app/v3/runOpsMigration/mintFlipGrace.test.ts
@@ -1,5 +1,10 @@
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
-import { effectiveMintKind, stampMintKindFlip, type MintFlagResolution } from "./mintFlipGrace";
+import {
+  effectiveMintKind,
+  resolveMintFlag,
+  stampMintKindFlip,
+  type MintFlagResolution,
+} from "./mintFlipGrace";
 
 // GRACE-LINGER: during [flippedAt, flippedAt + GRACE) every process — stale or fresh —
 // must resolve to the SAME (old) kind; at/after the cutover every process resolves to
@@ -157,3 +162,67 @@ describe("stampMintKindFlip", () => {
     ).toBe("runOpsId");
   });
 });
+
+// SOURCE-CONSISTENCY: the kind and its grace stamp must come from the SAME source. A per-org
+// runOpsMintKind override wins both the kind and the stamp; with no per-org override, BOTH the
+// kind and the stamp come from the global FeatureFlag rows. Never mix (e.g. a per-org kind with
+// the global stamp), which would date a grace window against the wrong flip.
+describe("resolveMintFlag", () => {
+  it("a per-org override wins the kind AND owns the stamp, ignoring the global stamp entirely", () => {
+    const perOrg = {
+      runOpsMintKind: "runOpsId",
+      runOpsMintKindPrev: "cuid",
+      runOpsMintKindFlippedAt: new Date(T).toISOString(),
+    };
+    const global = {
+      runOpsMintKind: "cuid",
+      runOpsMintKindPrev: "runOpsId",
+      runOpsMintKindFlippedAt: new Date(T + 500_000).toISOString(),
+    };
+    expect(resolveMintFlag(perOrg, global)).toEqual({
+      kind: "runOpsId",
+      prev: "cuid",
+      flippedAtMs: T,
+    });
+  });
+
+  it("with NO per-org override, the kind AND the stamp come from the global rows (global flip is graced)", () => {
+    const global = {
+      runOpsMintKind: "runOpsId",
+      runOpsMintKindPrev: "cuid",
+      runOpsMintKindFlippedAt: new Date(T).toISOString(),
+    };
+    const resolution = resolveMintFlag({}, global);
+    expect(resolution).toEqual({ kind: "runOpsId", prev: "cuid", flippedAtMs: T });
+    // Mid-grace: a global flip resolves to the OLD kind for the whole window.
+    expect(effectiveMintKind(resolution, T + GRACE_MS - 1, GRACE_MS)).toBe("cuid");
+    expect(effectiveMintKind(resolution, T + GRACE_MS, GRACE_MS)).toBe("runOpsId");
+  });
+
+  it("a per-org override with NO per-org stamp does NOT borrow the global stamp (kind stays ungraced)", () => {
+    const perOrg = { runOpsMintKind: "runOpsId" };
+    const global = {
+      runOpsMintKind: "cuid",
+      runOpsMintKindPrev: "cuid",
+      runOpsMintKindFlippedAt: new Date(T).toISOString(),
+    };
+    expect(resolveMintFlag(perOrg, global)).toEqual({
+      kind: "runOpsId",
+      prev: undefined,
+      flippedAtMs: undefined,
+    });
+  });
+
+  it("defaults to cuid with no stamp when neither source has a kind", () => {
+    expect(resolveMintFlag({}, {})).toEqual({
+      kind: "cuid",
+      prev: undefined,
+      flippedAtMs: undefined,
+    });
+    expect(resolveMintFlag(null, null)).toEqual({
+      kind: "cuid",
+      prev: undefined,
+      flippedAtMs: undefined,
+    });
+  });
+});
diff --git a/apps/webapp/app/v3/runOpsMigration/mintFlipGrace.ts b/apps/webapp/app/v3/runOpsMigration/mintFlipGrace.ts
@@ -30,19 +30,40 @@ function readMintKind(flags: Record<string, unknown>, key: string): RunIdMintKin
   return value === "cuid" || value === "runOpsId" ? value : undefined;
 }
 
-function resolveEffectiveFromFlags(
-  flags: Record<string, unknown> | null | undefined,
-  nowMs: number,
-  graceMs: number
-): RunIdMintKind {
+// Reads the { kind, prev, flippedAtMs } trio out of one flag record — either an org's
+// featureFlags override blob or the global FeatureFlag rows projected into a record. Pure.
+export function readMintResolution(
+  flags: Record<string, unknown> | null | undefined
+): MintFlagResolution {
   const source = flags ?? {};
   const kind = readMintKind(source, "runOpsMintKind") ?? DEFAULT_MINT_KIND;
   const prev = readMintKind(source, "runOpsMintKindPrev");
   const flippedAtRaw = source.runOpsMintKindFlippedAt;
   const parsed = typeof flippedAtRaw === "string" ? Date.parse(flippedAtRaw) : NaN;
   const flippedAtMs = Number.isNaN(parsed) ? undefined : parsed;
+  return { kind, prev, flippedAtMs };
+}
+
+// SOURCE-CONSISTENT resolution: a per-org runOpsMintKind override wins the kind AND owns the
+// grace stamp; with no per-org override, the kind AND the stamp both come from the global rows.
+// The stamp is never read from a different source than the kind, which would date a grace
+// window against the wrong flip.
+export function resolveMintFlag(
+  perOrgOverrides: Record<string, unknown> | null | undefined,
+  globalFlags: Record<string, unknown> | null | undefined
+): MintFlagResolution {
+  if (readMintKind(perOrgOverrides ?? {}, "runOpsMintKind") !== undefined) {
+    return readMintResolution(perOrgOverrides);
+  }
+  return readMintResolution(globalFlags);
+}
 
-  return effectiveMintKind({ kind, prev, flippedAtMs }, nowMs, graceMs);
+function resolveEffectiveFromFlags(
+  flags: Record<string, unknown> | null | undefined,
+  nowMs: number,
+  graceMs: number
+): RunIdMintKind {
+  return effectiveMintKind(readMintResolution(flags), nowMs, graceMs);
 }
 
 // Stamps a grace window only when the outgoing TARGET kind differs from the stored one (a
diff --git a/apps/webapp/app/v3/runOpsMigration/runOpsMintKind.server.ts b/apps/webapp/app/v3/runOpsMigration/runOpsMintKind.server.ts
@@ -3,10 +3,9 @@ import { env } from "~/env.server";
 import { logger } from "~/services/logger.server";
 import { BoundedTtlCache } from "~/services/realtime/boundedTtlCache";
 import { singleton } from "~/utils/singleton";
-import { FEATURE_FLAG, FeatureFlagCatalog } from "~/v3/featureFlags";
-import { makeFlag } from "~/v3/featureFlags.server";
+import { FEATURE_FLAG } from "~/v3/featureFlags";
 import { DEFAULT_CP_CACHE_TTL_MS } from "./controlPlaneCache.server";
-import { effectiveMintKind, type MintFlagResolution } from "./mintFlipGrace";
+import { effectiveMintKind, resolveMintFlag, type MintFlagResolution } from "./mintFlipGrace";
 import { isSplitEnabled } from "./splitMode.server";
 
 export type RunIdMintKind = "cuid" | "runOpsId";
@@ -36,7 +35,6 @@ export async function computeRunIdMintKind(
 }
 
 // ENV-BOUND wrapper — the only place env/$replica/isSplitEnabled are read.
-const flagFn = singleton("runOpsMintFlag", () => makeFlag($replica));
 const mintCache = singleton(
   "runOpsMintCache",
   () =>
@@ -64,14 +62,6 @@ if (env.RUN_OPS_MINT_FLIP_GRACE_MS <= env.RUN_OPS_MINT_FLAG_CACHE_TTL_MS + contr
   );
 }
 
-function readValidatedFlag<T extends "runOpsMintKindPrev" | "runOpsMintKindFlippedAt">(
-  overrides: Record<string, unknown>,
-  key: T
-): string | undefined {
-  const parsed = FeatureFlagCatalog[FEATURE_FLAG[key]].safeParse(overrides[FEATURE_FLAG[key]]);
-  return parsed.success ? parsed.data : undefined;
-}
-
 export async function resolveRunIdMintKind(environment: {
   organizationId: string;
   id: string;
@@ -106,22 +96,30 @@ export async function resolveRunIdMintKind(environment: {
 
       const overridesRecord = (overrides as Record<string, unknown>) ?? {};
 
-      const kind = await flagFn({
-        key: FEATURE_FLAG.runOpsMintKind,
-        defaultValue: "cuid",
-        overrides: overridesRecord,
+      // One global read over the three mint-flag keys (kind + grace stamp), folded into the
+      // single cache-miss round-trip. This replaces the former single-key flag read, so a
+      // GLOBAL flip is now grace-stamped WITHOUT adding any new per-mint/per-resolve query.
+      // (The cache-hit branch above never touches the DB.)
+      const globalRows = await $replica.featureFlag.findMany({
+        where: {
+          key: {
+            in: [
+              FEATURE_FLAG.runOpsMintKind,
+              FEATURE_FLAG.runOpsMintKindPrev,
+              FEATURE_FLAG.runOpsMintKindFlippedAt,
+            ],
+          },
+        },
+        select: { key: true, value: true },
       });
+      const globalFlags: Record<string, unknown> = {};
+      for (const row of globalRows) {
+        globalFlags[row.key] = row.value;
+      }
 
-      // Read the grace-linger stamp DIRECTLY from the already-in-memory org overrides — no
-      // extra DB read, and no forcing these optional fields through flagFn's defaultValue path.
-      const prev = readValidatedFlag(overridesRecord, "runOpsMintKindPrev") as
-        | RunIdMintKind
-        | undefined;
-      const flippedAtRaw = readValidatedFlag(overridesRecord, "runOpsMintKindFlippedAt");
-      const parsedFlippedAt = flippedAtRaw !== undefined ? Date.parse(flippedAtRaw) : NaN;
-      const flippedAtMs = Number.isNaN(parsedFlippedAt) ? undefined : parsedFlippedAt;
-
-      const resolution: MintFlagResolution = { kind, prev, flippedAtMs };
+      // Source-consistent: a per-org override wins the kind AND its stamp; otherwise the
+      // global row wins the kind AND its stamp.
+      const resolution: MintFlagResolution = resolveMintFlag(overridesRecord, globalFlags);
       mintCache.set(orgId, resolution);
       return effectiveMintKind(resolution, Date.now(), env.RUN_OPS_MINT_FLIP_GRACE_MS);
     },
