WIP: feat(recorder): unify studio recorder on preview events

apps/chrome-extension/src/extension/recorder/generators/yamlGenerator.ts

@@ -3,16 +3,35 @@ import type {
   StreamingCodeGenerationOptions,
 } from '@midscene/core';
 import {
-  generateYamlTest as generateYamlTestCore,
-  generateYamlTestStream as generateYamlTestStreamCore,
+  generateRecorderYamlTest,
+  generateRecorderYamlTestStream,
 } from '@midscene/core/ai-model';
 import type { ChromeRecordedEvent } from '@midscene/recorder';
 import type { IModelConfig } from '@midscene/shared/env';
+import type { MidsceneRecorderTarget } from '@midscene/shared/recorder';
 import { recordLogger } from '../logger';
 import { extractNavigationAndViewportInfo } from './playwrightGenerator';
 import { handleTestGenerationError } from './shared/testGenerationUtils';
 import type { YamlGenerationOptions } from './shared/types';
 
+function createWebRecorderTarget(
+  events: ChromeRecordedEvent[],
+): MidsceneRecorderTarget {
+  const navigationInfo = extractNavigationAndViewportInfo(events);
+  const url = navigationInfo.urls[0] || '';
+  const viewport = navigationInfo.initialViewport;
+  return {
+    platformId: 'web',
+    deviceId: url || undefined,
+    label: url || 'Web',
+    values: {
+      url,
+      ...(viewport?.width ? { viewportWidth: viewport.width } : {}),
+      ...(viewport?.height ? { viewportHeight: viewport.height } : {}),
+    },
+  };
+}
+
 /**
  * Generates YAML test configuration from recorded events using AI
  * Uses the core package implementation
@@ -23,22 +42,16 @@ export const generateYamlTest = async (
   modelConfig: IModelConfig,
 ): Promise<string> => {
   try {
-    // Extract navigation and viewport information
-    const navigationInfo = extractNavigationAndViewportInfo(events);
-
     recordLogger.info('Starting AI-powered YAML test generation', {
       eventsCount: events.length,
     });
 
-    // Merge navigation and viewport info into options
-    const enhancedOptions = {
-      ...options,
-      navigationInfo,
-    };
-
-    const yamlContent = await generateYamlTestCore(
-      events,
-      enhancedOptions,
+    const yamlContent = await generateRecorderYamlTest(
+      {
+        ...options,
+        target: createWebRecorderTarget(events),
+        events,
+      },
       modelConfig,
     );
 
@@ -106,25 +119,20 @@ export const generateYamlTestStream = async (
   modelConfig: IModelConfig,
 ): Promise<StreamingAIResponse> => {
   try {
-    // Extract navigation and viewport information
-    const navigationInfo = extractNavigationAndViewportInfo(events);
-
     recordLogger.info(
       'Starting AI-powered YAML test generation with streaming',
       {
         eventsCount: events.length,
       },
     );
 
-    // Merge navigation and viewport info into options
-    const enhancedOptions = {
-      ...options,
-      navigationInfo,
-    };
-
-    const result = await generateYamlTestStreamCore(
-      events,
-      enhancedOptions,
+    const result = await generateRecorderYamlTestStream(
+      {
+        ...options,
+        target: createWebRecorderTarget(events),
+        events,
+      },
+      options,
       modelConfig,
     );
 

apps/chrome-extension/tests/recorder-yaml-generator.test.ts

@@ -0,0 +1,98 @@
+import {
+  generateRecorderYamlTest,
+  generateRecorderYamlTestStream,
+} from '@midscene/core/ai-model';
+import type { IModelConfig } from '@midscene/shared/env';
+import { describe, expect, it, vi } from 'vitest';
+import {
+  generateYamlTest,
+  generateYamlTestStream,
+} from '../src/extension/recorder/generators/yamlGenerator';
+
+vi.mock('@midscene/core/ai-model', () => ({
+  generateRecorderYamlTest: vi.fn(
+    async () => 'web:\n  url: "https://example.com"\n',
+  ),
+  generateRecorderYamlTestStream: vi.fn(async () => ({
+    content: 'web:\n  url: "https://example.com"\n',
+    usage: undefined,
+    isStreamed: true,
+  })),
+}));
+
+vi.mock('../src/extension/recorder/logger', () => ({
+  recordLogger: {
+    error: vi.fn(),
+    info: vi.fn(),
+    warn: vi.fn(),
+    success: vi.fn(),
+    debug: vi.fn(),
+  },
+}));
+
+const mockedModelConfig = {
+  modelName: 'mock',
+  modelDescription: 'mock',
+  intent: 'default',
+  slot: 'default',
+} as const satisfies IModelConfig;
+
+describe('chrome extension recorder YAML generator', () => {
+  it('passes a web recorder target to core YAML generation', async () => {
+    await generateYamlTest(
+      [
+        {
+          type: 'navigation',
+          url: 'https://example.com',
+          title: 'Example',
+          pageInfo: { width: 1280, height: 720 },
+          timestamp: 1,
+          hashId: 'nav-1',
+        },
+      ],
+      { testName: 'extension recording' },
+      mockedModelConfig,
+    );
+
+    expect(generateRecorderYamlTest).toHaveBeenCalledWith(
+      expect.objectContaining({
+        target: {
+          platformId: 'web',
+          deviceId: 'https://example.com',
+          label: 'https://example.com',
+          values: {
+            url: 'https://example.com',
+            viewportWidth: 1280,
+            viewportHeight: 720,
+          },
+        },
+      }),
+      mockedModelConfig,
+    );
+  });
+
+  it('passes streaming options through to core YAML generation', async () => {
+    const onChunk = vi.fn();
+    await generateYamlTestStream(
+      [
+        {
+          type: 'click',
+          elementDescription: 'Login button',
+          pageInfo: { width: 1280, height: 720 },
+          timestamp: 2,
+          hashId: 'click-1',
+        },
+      ],
+      { stream: true, onChunk },
+      mockedModelConfig,
+    );
+
+    expect(generateRecorderYamlTestStream).toHaveBeenCalledWith(
+      expect.objectContaining({
+        target: expect.objectContaining({ platformId: 'web' }),
+      }),
+      expect.objectContaining({ stream: true, onChunk }),
+      mockedModelConfig,
+    );
+  });
+});

apps/report/src/components/detail-side/index.tsx

@@ -169,6 +169,14 @@ const renderMetaContent = (
 const extractTaskImages = (
   param: any,
 ): Array<{ name: string; url: string }> | undefined => {
+  // For aiAct planning tasks
+  if (
+    param?.userInstruction?.images &&
+    Array.isArray(param.userInstruction.images)
+  ) {
+    return param.userInstruction.images;
+  }
+
   // For locate params (Planning and Action Space tasks)
   if (param?.prompt?.images && Array.isArray(param.prompt.images)) {
     return param.prompt.images;
@@ -479,11 +487,10 @@ const DetailSide = (): JSX.Element => {
     const memoriesStatus = (planningTask.param as any)?.memoriesStatus;
 
     if (planningTask.param?.userInstruction) {
-      // Ensure userInstruction is a string
       const instructionContent =
         typeof planningTask.param.userInstruction === 'string'
           ? planningTask.param.userInstruction
-          : JSON.stringify(planningTask.param.userInstruction);
+          : planningTask.param.userInstruction.prompt;
 
       taskInput = MetaKV({
         data: [

apps/studio/package.json

@@ -5,8 +5,8 @@
   "type": "module",
   "scripts": {
     "build": "rsbuild build && node scripts/sync-static-assets.mjs",
-    "dev": "pnpm run dev:deps && concurrently -k -n deps,build,app \"nx run-many --target=dev --projects=@midscene/visualizer,@midscene/playground-app,@midscene/android-playground,@midscene/playground,@midscene/web --parallel --exclude-task-dependencies\" \"rsbuild dev\" \"node scripts/wait-for-electron-build.mjs && node scripts/launch-electron-dev.mjs\"",
-    "dev:deps": "nx run @midscene/visualizer:build --outputStyle=stream && nx run-many --target=build --projects=@midscene/playground-app,@midscene/android-playground,@midscene/playground,@midscene/web --parallel --outputStyle=stream",
+    "dev": "pnpm run dev:deps && concurrently -k -n deps,build,app \"nx run-many --target=dev --projects=@midscene/visualizer,@midscene/playground-app,@midscene/android-playground,@midscene/computer,@midscene/computer-playground,@midscene/playground,@midscene/web,@midscene/recorder --parallel --exclude-task-dependencies\" \"rsbuild dev\" \"node scripts/wait-for-electron-build.mjs && node scripts/launch-electron-dev.mjs\"",
+    "dev:deps": "nx run @midscene/visualizer:build --outputStyle=stream && nx run-many --target=build --projects=@midscene/playground-app,@midscene/android-playground,@midscene/computer,@midscene/computer-playground,@midscene/playground,@midscene/web,@midscene/recorder --parallel --outputStyle=stream",
     "package:release": "node scripts/package-electron.mjs",
     "preview": "rsbuild preview --environment renderer",
     "start": "node scripts/sync-static-assets.mjs && node scripts/launch-electron-prod.mjs",
@@ -25,11 +25,13 @@
     "@midscene/ios": "workspace:*",
     "@midscene/playground": "workspace:*",
     "@midscene/playground-app": "workspace:*",
+    "@midscene/recorder": "workspace:*",
     "@midscene/shared": "workspace:*",
     "@midscene/visualizer": "workspace:*",
     "@midscene/web": "workspace:*",
     "antd": "^5.21.6",
     "electron-updater": "6.8.3",
+    "jszip": "3.10.1",
     "puppeteer": "24.6.0",
     "react": "18.3.1",
     "react-dom": "18.3.1"

apps/studio/rsbuild.config.ts

@@ -16,6 +16,8 @@ import {
 // `file://` HTML. A relative prefix works in both places; an absolute
 // `/static/...` prefix leaves packaged/build smoke runs with a blank renderer.
 const rendererAssetPrefix = './';
+const studioRecorderEntryEnabled =
+  process.env.VITE_STUDIO_RECORDER_ENABLED !== 'false';
 
 export default defineConfig({
   tools: {
@@ -79,6 +81,10 @@ export default defineConfig({
         __dirname,
         '../../packages/visualizer/src/index.tsx',
       ),
+      '@midscene/visualizer/history-selector$': path.join(
+        __dirname,
+        '../../packages/visualizer/src/component/history-selector/index.tsx',
+      ),
       '@midscene/web/static$': path.join(
         __dirname,
         '../../packages/web-integration/src/static/index.ts',
@@ -102,6 +108,9 @@ export default defineConfig({
         },
         define: {
           __APP_VERSION__: JSON.stringify(appVersion),
+          __STUDIO_RECORDER_ENTRY_ENABLED__: JSON.stringify(
+            studioRecorderEntryEnabled,
+          ),
         },
       },
       output: {