feat(xdbg): V3→V4 migration latency monitor (#3338)

## Summary
- Adds a **V3→V4 migration latency test** to xdbg (`test
migration-latency` subcommand)
- Integrates the test into the **client monitor entrypoint.sh loop**
(runs every 5 minutes alongside identity/group/message tests)
- Fixes a **pre-existing bug** where all PushGateway metric pushes
silently failed (fire-and-forget `tokio::spawn` was cancelled when
short-lived xdbg subprocesses exited)

## What it does
1. Creates a V3 identity and group on production
2. Sends a tagged message via V3
3. Polls the V4 testnet replication node until the message appears (or
times out)
4. Records `xdbg_migration_latency_seconds` histogram,
`xdbg_migration_success_total` / `xdbg_migration_failure_total` counters
5. Pushes all metrics to PushGateway (awaited, not fire-and-forget)

## Key design decisions
- Migration test always uses V3 **production** and V4 **testnet**
(`grpc.testnet.xmtp.network`), regardless of `WORKSPACE` env — this is
the only migration path that exists
- `push_metrics()` and `record_phase_metric()` changed from sync/spawn
to **async with `.await`** — this fixes metrics for ALL existing tests
(identity, group, message), not just migration

## Files changed
- `src/app/test.rs` — new `migration_latency_test()` + `.await` on
existing push calls
- `src/metrics.rs` — dedicated migration histogram/counters + async push
fix
- `src/args.rs` — `MigrationLatency` variant + CLI args
- `docker/entrypoint.sh` — migration test integration in loop
- `src/app/generate/{identity,groups,messages}.rs` — `.await` on push
calls (async fix)

## Verified in production
Deployed to testnet-dev ECS, two full loop iterations confirmed:
- Migration latency: **2.2s – 3.9s**, 100% success rate
- All metrics flowing to PushGateway (zero push errors)
- Identity, group, message tests all passing

Author: api-Hypernova

apps/xmtp_debug/docker/entrypoint.sh

@@ -4,7 +4,9 @@
 # monitoring loop — the daemon recovers on the next iteration.
 set -uo pipefail
 
-: "${XDBG_LOOP_PAUSE:=300}" # default interval between loop iterations
+: "${XDBG_LOOP_PAUSE:=300}"    # default interval between loop iterations
+: "${XDBG_V4_NODE_URL:=}"      # V4/D14N node URL for migration latency test
+: "${XDBG_MIGRATION_TIMEOUT:=120}" # timeout for migration latency polling
 
 function log {
     echo "[$(date '+%F %T')] $*"
@@ -20,6 +22,13 @@ case "${WORKSPACE}" in
 esac
 log "WORKSPACE='${WORKSPACE:-<unset>}' -> backend='${BACKEND}'"
 
+# Migration test: always writes to V3 production and reads from V4 testnet,
+# because the only migrator path is V3 production → V4 testnet.
+# XDBG_V4_NODE_URL can be overridden via env, but defaults to the testnet
+# D14N replication node regardless of WORKSPACE.
+: "${XDBG_V4_NODE_URL:=https://grpc.testnet.xmtp.network:443}"
+log "V4 node URL (migration): ${XDBG_V4_NODE_URL}"
+
 while true; do
   log "Reset environment.."
   XDBG_LOOP_PAUSE=0 xdbg -d -b "${BACKEND}" --perf --clear \
@@ -46,6 +55,17 @@ while true; do
       || log "WARNING: message step $x failed"
     log "Running health checks..."
     bash "$(dirname "$0")/web-healthcheck.sh" || log "WARNING: health check failed"
+
+    # Migration latency test: always V3 production → V4 testnet (the only migration path).
+    # Omits -d and --perf (D14N mode) since this writes to V3.
+    # Uses -b production regardless of WORKSPACE.
+    log "Migration latency test..."
+    XDBG_LOOP_PAUSE=0 xdbg -b production test migration-latency \
+      --v4-node-url "${XDBG_V4_NODE_URL}" \
+      --migration-timeout "${XDBG_MIGRATION_TIMEOUT}" \
+      --iterations 1 \
+      || log "WARNING: migration latency test $x failed"
+
     log "Sleeping ${XDBG_LOOP_PAUSE} seconds..."
     sleep "${XDBG_LOOP_PAUSE}"
   done

apps/xmtp_debug/src/app/generate/groups.rs

@@ -86,7 +86,8 @@ impl GenerateGroups {
                             create_secs,
                             "group_create",
                             "xdbg_debug",
-                        );
+                        )
+                        .await;
 
                         if let Some(ref invitee) = first_invitee {
                             check_member_visibility(&group_id, invitee, &network_clone).await;
@@ -99,7 +100,8 @@ impl GenerateGroups {
                             total_secs,
                             "group_total",
                             "xdbg_debug",
-                        );
+                        )
+                        .await;
 
                         bar_pointer.inc(1);
 
@@ -207,7 +209,7 @@ async fn create_group_on_network(
         member_ids.len() as f64,
         &[("phase", "add_members")],
     );
-    push_metrics("xdbg_debug");
+    push_metrics("xdbg_debug").await;
 
     let group_id = group.group_id.clone();
     drop(client_guard);
@@ -261,5 +263,5 @@ async fn check_member_visibility(
         visibility_secs,
         &[("phase", "member_visibility"), ("success", vis_ok)],
     );
-    push_metrics("xdbg_debug");
+    push_metrics("xdbg_debug").await;
 }

apps/xmtp_debug/src/app/generate/identity.rs

@@ -86,7 +86,8 @@ impl GenerateIdentity {
                             init_secs,
                             "client_init",
                             "xdbg_debug",
-                        );
+                        )
+                        .await;
 
                         bar_pointer
                             .set_message(format!("generated client {}", c.identity().inbox_id()));
@@ -197,7 +198,8 @@ impl GenerateIdentity {
                             register_secs,
                             "register",
                             "xdbg_debug",
-                        );
+                        )
+                        .await;
 
                         Ok(identity)
                     }
@@ -309,7 +311,7 @@ async fn poll_association_readiness(network: &args::BackendOpts, inbox_id_hex: &
         1.0,
         &[("phase", "assoc_ready"), ("success", assoc_ok)],
     );
-    push_metrics("xdbg_debug");
+    push_metrics("xdbg_debug").await;
 
     Ok(())
 }
@@ -334,7 +336,8 @@ async fn measure_sync_and_lookup(
         sync_secs,
         "identity_read_sync",
         "xdbg_debug",
-    );
+    )
+    .await;
 
     // -- identity lookup latency --
     let t_lookup = Instant::now();
@@ -349,7 +352,8 @@ async fn measure_sync_and_lookup(
         lookup_secs,
         "identity_read",
         "xdbg_debug",
-    );
+    )
+    .await;
 
     Ok(())
 }
@@ -375,7 +379,8 @@ async fn verify_identities_readable(
             verify_secs,
             "verify_identity_read",
             "xdbg_debug",
-        );
+        )
+        .await;
     }
     Ok(())
 }

apps/xmtp_debug/src/app/generate/messages.rs

@@ -321,7 +321,7 @@ impl GenerateMessages {
             let send_latency = start.elapsed();
             let send_secs = send_latency.as_secs_f64();
 
-            record_phase_metric("send_message", send_secs, "send_message", "xdbg_debug");
+            record_phase_metric("send_message", send_secs, "send_message", "xdbg_debug").await;
 
             Ok(send_latency)
         } else {