Add meeting session timing tracking and signaling

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - Added PPS metrics on the Encoded Transform when used.
+- Added meeting session lifecycle timing tracking and signaling via `MeetingSessionTimingManager`.
 
 ### Removed
 

demos/browser/package.json

@@ -7,7 +7,7 @@
     "build:fast": "vite build --",
     "build": "npm run deps && npm install && tsc --noEmit && npm run build:fast",
     "start:fast": "node dev-server.js",
-    "start:watch": "WATCH_SDK=true node dev-server.js",
+    "start:sdk-autorefresh": "SDK_AUTOREFRESH=true node dev-server.js",
     "start": "npm run deps && npm install && npm run start:fast"
   },
   "devDependencies": {

demos/browser/vite.config.ts

@@ -3,13 +3,12 @@
 
 import { defineConfig, type Plugin } from 'vite';
 import { resolve } from 'path';
-import { rmSync } from 'fs';
 import { watch } from 'chokidar';
 import { viteSingleFile } from 'vite-plugin-singlefile';
 import ejsSvgPlugin from './plugins/vite-plugin-ejs-svg';
 
 const app = process.env.npm_config_app || process.env.APP || 'meetingV2';
-const watchSdk = process.env.WATCH_SDK === 'true';
+const watchSdk = process.env.SDK_AUTOREFRESH === 'true';
 
 /**
  * This is exactly what we document in the CSP guide.
@@ -89,12 +88,11 @@ function fullReloadPlugin(): Plugin {
         sdkWatcher.on('change', () => {
           if (debounce) clearTimeout(debounce);
           debounce = setTimeout(() => {
-            console.log('[full-reload] SDK build changed, clearing cache and restarting...');
-            try { rmSync(resolve(__dirname, 'node_modules/.vite'), { recursive: true, force: true }); } catch {}
-            server.restart();
+            console.log('[full-reload] SDK build changed, reloading browser...');
+            server.ws.send({ type: 'full-reload' });
           }, 1000);
         });
-        console.log('[full-reload] Watching SDK build output for changes.');
+        console.log('[full-reload] Watching SDK build output for changes (SDK_AUTOREFRESH).');
       }
 
       server.watcher.on('change', () => {
@@ -118,6 +116,7 @@ export default defineConfig({
   },
   optimizeDeps: {
     include: ['amazon-chime-sdk-js'],
+    force: true,
   },
   define: {
     global: 'globalThis',

package.json

@@ -134,5 +134,8 @@
     "singleQuote": true,
     "trailingComma": "es5",
     "printWidth": 100
+  },
+  "overrides": {
+    "fast-xml-parser": "5.3.6"
   }
 }

protocol/SignalingProtocol.proto

@@ -27,6 +27,7 @@ message SdkSignalFrame {
         PRIMARY_MEETING_JOIN_ACK = 26;
         PRIMARY_MEETING_LEAVE = 27;
         NOTIFICATION = 34;
+        MEETING_SESSION_TIMING = 43;
     }
     required uint64 timestamp_ms = 1;
     required Type type = 2;
@@ -52,6 +53,7 @@ message SdkSignalFrame {
     optional SdkPrimaryMeetingJoinAckFrame primary_meeting_join_ack = 27;
     optional SdkPrimaryMeetingLeaveFrame primary_meeting_leave = 28;
     optional SdkNotificationFrame notification = 35;
+    optional SdkMeetingSessionTimingFrame meeting_session_timing = 44;
 }
 
 message SdkErrorFrame {
@@ -283,6 +285,8 @@ message SdkMetric {
         VIDEO_DISCARDED_PPS = 47;
         VIDEO_PLIS_SENT = 48;
         VIDEO_RECEIVED_JITTER_MS = 49;
+        VIDEO_LOCAL_RENDER_FPS = 52;
+        VIDEO_REMOTE_RENDER_FPS = 56;
         VIDEO_INPUT_HEIGHT = 60;
         VIDEO_ENCODE_HEIGHT = 64;
         VIDEO_SENT_QP_SUM = 66;
@@ -503,3 +507,60 @@ enum SdkVideoCodecCapability {
     VP9_PROFILE_0 = 8;
     AV1_MAIN_PROFILE = 11;
 };
+
+message SdkMeetingSessionTimingFrame {
+    repeated SdkMeetingSessionSignalingTiming signaling = 1;
+    repeated SdkMeetingSessionRemoteAudioTiming remote_audio = 2;
+    repeated SdkMeetingSessionLocalAudioTiming local_audio = 3;
+    repeated SdkMeetingSessionLocalVideoTiming local_video = 4;
+    repeated SdkMeetingSessionRemoteVideoTiming remote_videos = 5;
+}
+
+message SdkMeetingSessionSignalingTiming {
+    optional int64 start_ms = 1;
+    optional int64 join_sent_ms = 2;
+    optional int64 join_ack_received_ms = 3;
+    optional int64 transport_connected_ms = 4;
+    optional int64 create_offer_ms = 5;
+    optional int64 set_local_description_ms = 6;
+    optional int64 set_remote_description_ms = 7;
+    optional int64 ice_gathering_start_ms = 8;
+    optional int64 ice_gathering_complete_ms = 9;
+    optional int64 ice_connected_ms = 10;
+    optional int64 subscribe_sent_ms = 11;
+    optional int64 subscribe_ack_ms = 12;
+    optional bool timed_out = 13;
+}
+
+message SdkMeetingSessionRemoteAudioTiming {
+    optional int64 added_ms = 1;
+    optional int64 first_packet_received_ms = 2;
+    optional int64 first_frame_rendered_ms = 3;
+    optional bool timed_out = 4;
+    optional bool removed = 5;
+}
+
+message SdkMeetingSessionLocalAudioTiming {
+    optional int64 added_ms = 1;
+    optional int64 first_frame_captured_ms = 2;
+    optional int64 first_packet_sent_ms = 3;
+    optional bool timed_out = 4;
+    optional bool removed = 5;
+}
+
+message SdkMeetingSessionLocalVideoTiming {
+    optional int64 added_ms = 1;
+    optional int64 first_frame_captured_ms = 2;
+    optional int64 first_frame_sent_ms = 3;
+    optional bool timed_out = 4;
+    optional bool removed = 5;
+}
+
+message SdkMeetingSessionRemoteVideoTiming {
+    optional uint32 group_id = 1;
+    optional int64 added_ms = 2;
+    optional int64 first_packet_received_ms = 3;
+    optional int64 first_frame_rendered_ms = 4;
+    optional bool timed_out = 5;
+    optional bool removed = 6;
+}

script/audit-deps

@@ -6,7 +6,7 @@ end
 
 # Just so the devs can see.
 puts 'Auditing development dependencies. You should address any findings.'
-system('npm audit --omit=prod')
+system('npm audit --include=dev')
 
 puts '---'
 

script/barrelize.js

@@ -120,6 +120,16 @@ walk('src')
     importStrings.push(importLine);
     exportStrings.push(exportLine);
 
+    if (typeToImport === 'MeetingSessionTiming') {
+      importStrings.push(`import { MeetingSessionSignalingTiming, MeetingSessionRemoteAudioTiming, MeetingSessionLocalAudioTiming, MeetingSessionLocalVideoTiming, MeetingSessionRemoteVideoTiming, MeetingSessionTimingObserver } from '${pathToImport}/MeetingSessionTiming';`);
+      exportStrings.push('MeetingSessionSignalingTiming');
+      exportStrings.push('MeetingSessionRemoteAudioTiming');
+      exportStrings.push('MeetingSessionLocalAudioTiming');
+      exportStrings.push('MeetingSessionLocalVideoTiming');
+      exportStrings.push('MeetingSessionRemoteVideoTiming');
+      exportStrings.push('MeetingSessionTimingObserver');
+    }
+
     // Because these two types are very intertwined.
     if (typeToImport === 'VideoPreferences') {
       importStrings.push(`import { MutableVideoPreferences } from '${pathToImport}/VideoPreferences';`);

script/generate-media-transform-worker-code.js

@@ -20,10 +20,13 @@ const workerTsconfig = 'tsconfig.mediatransformworker.json';
 const workerTsconfigContent = `{
   "extends": "./tsconfig.base.json",
   "compilerOptions": {
-    "module": "ES2015",
+    "module": "es2015",
     "moduleResolution": "node",
-    "outDir": "../build",
-    "rootDir": "../src"
+    "outDir": "../build/mediatransformworker",
+    "rootDir": "../src",
+    "tsBuildInfoFile": "./tsconfig.mediatransformworker.tsbuildinfo",
+    "incremental": true,
+    "composite": false
   },
   "include": [
     "../src/encodedtransformworker/**/*.ts"
@@ -55,7 +58,7 @@ fs.unlinkSync(`${configDir}/${workerTsconfig}`);
 
 // Read all transpiled worker files and bundle them inline
 // Order matters: dependencies must come before classes that use them
-const buildDir = './build/encodedtransformworker';
+const buildDir = './build/mediatransformworker/encodedtransformworker';
 const workerFiles = [
   'EncodedTransform.js',
   'RedundantAudioEncodedTransform.js',

src/audiovideocontroller/AudioVideoController.ts

@@ -5,6 +5,7 @@ import ActiveSpeakerDetector from '../activespeakerdetector/ActiveSpeakerDetecto
 import AudioMixController from '../audiomixcontroller/AudioMixController';
 import AudioVideoControllerFacade from '../audiovideocontroller/AudioVideoControllerFacade';
 import AudioVideoObserver from '../audiovideoobserver/AudioVideoObserver';
+import { EncodedTransformMediaMetricsObserver } from '../encodedtransformmanager/MediaMetricsEncodedTransformManager';
 import EventController from '../eventcontroller/EventController';
 import Logger from '../logger/Logger';
 import MediaStreamBroker from '../mediastreambroker/MediaStreamBroker';
@@ -16,7 +17,8 @@ import VideoTileController from '../videotilecontroller/VideoTileController';
 /**
  * [[AudioVideoController]] manages the signaling and peer connections.
  */
-export default interface AudioVideoController extends AudioVideoControllerFacade {
+export default interface AudioVideoController
+  extends AudioVideoControllerFacade, EncodedTransformMediaMetricsObserver {
   /**
    * Iterates through each observer, so that their notification functions may
    * be called.

src/audiovideocontroller/AudioVideoControllerState.ts

@@ -13,6 +13,7 @@ import MediaStreamBroker from '../mediastreambroker/MediaStreamBroker';
 import MeetingSessionConfiguration from '../meetingsession/MeetingSessionConfiguration';
 import MeetingSessionTURNCredentials from '../meetingsession/MeetingSessionTURNCredentials';
 import MeetingSessionVideoAvailability from '../meetingsession/MeetingSessionVideoAvailability';
+import MeetingSessionTimingManager from '../meetingsessiontiming/MeetingSessionTimingManager';
 import RealtimeController from '../realtimecontroller/RealtimeController';
 import ReconnectController from '../reconnectcontroller/ReconnectController';
 import RemovableObserver from '../removableobserver/RemovableObserver';
@@ -74,6 +75,8 @@ export default class AudioVideoControllerState {
 
   encodedTransformWorkerManager: EncodedTransformWorkerManager | null = null;
 
+  meetingSessionTimingManager: MeetingSessionTimingManager | null = null;
+
   indexFrame: SdkIndexFrame | null = null;
 
   iceCandidates: RTCIceCandidate[] = [];