Skip to content

Commit 08dfe6d

Browse files
committed
Merge branch 'main' into refactor/prompt-view-abstractions
2 parents 129d17d + 5df2516 commit 08dfe6d

36 files changed

Lines changed: 1110 additions & 3 deletions

.devcontainer/Dockerfile

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
# .NET 9 base. The .NET 10 SDK (required by global.json) and Docker (for the e2e
2+
# VHS recording) are layered in as devcontainer features — see devcontainer.json.
3+
# No bookworm-based .NET 10 image is published, so we add the SDK rather than
4+
# switch the distro.
15
FROM mcr.microsoft.com/devcontainers/dotnet:1-9.0-bookworm
26

37
RUN rm -f /etc/apt/sources.list.d/yarn.list \

.devcontainer/devcontainer-lock.json

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,16 @@
55
"resolved": "ghcr.io/devcontainers/features/common-utils@sha256:c42fdefe6d737a3a6f61cc52b23c7c9a565d08cc4d9c303669a7cf2ee5fd81fc",
66
"integrity": "sha256:c42fdefe6d737a3a6f61cc52b23c7c9a565d08cc4d9c303669a7cf2ee5fd81fc"
77
},
8+
"ghcr.io/devcontainers/features/docker-in-docker:2": {
9+
"version": "2.17.0",
10+
"resolved": "ghcr.io/devcontainers/features/docker-in-docker@sha256:25b9f05705ffba7dbe503230ac76081419306f8c8bc88e0ce78c4ecd99a0c78c",
11+
"integrity": "sha256:25b9f05705ffba7dbe503230ac76081419306f8c8bc88e0ce78c4ecd99a0c78c"
12+
},
13+
"ghcr.io/devcontainers/features/dotnet:2": {
14+
"version": "2.5.0",
15+
"resolved": "ghcr.io/devcontainers/features/dotnet@sha256:0fc16547ed4db6d7ff2a9f5981d2b93eb314e568affb9958029ad794f1f9a093",
16+
"integrity": "sha256:0fc16547ed4db6d7ff2a9f5981d2b93eb314e568affb9958029ad794f1f9a093"
17+
},
818
"ghcr.io/devcontainers/features/github-cli:1": {
919
"version": "1.1.0",
1020
"resolved": "ghcr.io/devcontainers/features/github-cli@sha256:d22f50b70ed75339b4eed1ba9ecde3a1791f90e88d37936517e3bace0bbad671",

.devcontainer/devcontainer.json

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,18 @@
1111
"ghcr.io/devcontainers/features/common-utils:2": {
1212
"installZsh": true,
1313
"configureZshAsDefaultShell": true
14-
}
14+
},
15+
// The base image (dotnet:1-9.0-bookworm) ships only .NET 9, but global.json
16+
// pins SDK 10.0.100 and test/e2e/run.sh publishes -f net10.0. A bookworm-based
17+
// .NET 10 image isn't published, so layer the SDK in here instead of changing
18+
// the distro. .NET 9 stays available from the base for the net9.0 test projects.
19+
"ghcr.io/devcontainers/features/dotnet:2": {
20+
"version": "10.0"
21+
},
22+
// Docker engine inside the devcontainer so the e2e VHS recording
23+
// (test/e2e/run.sh) can run the pinned VHS image locally, byte-for-byte
24+
// identical to CI. Requires a host that permits privileged containers.
25+
"ghcr.io/devcontainers/features/docker-in-docker:2": {}
1526
},
1627
"customizations": {
1728
"vscode": {
@@ -28,6 +39,6 @@
2839
}
2940
}
3041
},
31-
"postCreateCommand": "dotnet --info && node --version && npm --version",
42+
"postCreateCommand": "dotnet --info && node --version && npm --version && docker --version",
3243
"remoteUser": "vscode"
3344
}

.github/workflows/e2e-demo.yml

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
name: e2e-demo
2+
3+
on:
4+
push:
5+
paths:
6+
- 'src/Skillz/**'
7+
- 'test/e2e/**'
8+
- 'test/fixtures/sample-skills/**'
9+
- 'Directory.Build.props'
10+
- 'Directory.Packages.props'
11+
- 'global.json'
12+
- '.github/workflows/e2e-demo.yml'
13+
pull_request:
14+
paths:
15+
- 'src/Skillz/**'
16+
- 'test/e2e/**'
17+
- 'test/fixtures/sample-skills/**'
18+
- 'Directory.Build.props'
19+
- 'Directory.Packages.props'
20+
- 'global.json'
21+
- '.github/workflows/e2e-demo.yml'
22+
workflow_dispatch:
23+
24+
permissions:
25+
contents: read
26+
pull-requests: write
27+
28+
concurrency:
29+
group: e2e-demo-${{ github.event.pull_request.number || github.ref }}
30+
cancel-in-progress: true
31+
32+
jobs:
33+
record-and-verify:
34+
runs-on: ubuntu-latest
35+
steps:
36+
- uses: actions/checkout@v4
37+
38+
- uses: actions/setup-dotnet@v4
39+
with:
40+
global-json-file: global.json
41+
42+
- name: Record + verify all flows
43+
id: record
44+
continue-on-error: true
45+
run: ./test/e2e/run.sh
46+
47+
- name: Upload snapshot artifacts
48+
if: always()
49+
uses: actions/upload-artifact@v4
50+
with:
51+
name: skillz-e2e-snapshots
52+
path: |
53+
test/e2e/out/*.gif
54+
test/e2e/out/report/**
55+
if-no-files-found: warn
56+
57+
- name: Comment changed snapshots on PR
58+
if: always() && github.event_name == 'pull_request'
59+
uses: actions/github-script@v7
60+
env:
61+
RUN_ID: ${{ github.run_id }}
62+
with:
63+
script: |
64+
const { owner, repo } = context.repo;
65+
const pr = context.payload.pull_request;
66+
const prNum = pr.number;
67+
const baseRef = pr.base.ref;
68+
const MARKER = '<!-- skillz-e2e-snapshots -->';
69+
const bust = `?run=${process.env.RUN_ID}`;
70+
71+
const rawBase = `https://raw.githubusercontent.com/${pr.base.repo.full_name}/${pr.base.sha}`;
72+
const headFull = pr.head.repo && pr.head.repo.full_name;
73+
const rawHead = headFull
74+
? `https://raw.githubusercontent.com/${headFull}/${pr.head.sha}`
75+
: null;
76+
77+
const files = await github.paginate(github.rest.pulls.listFiles,
78+
{ owner, repo, pull_number: prNum, per_page: 100 });
79+
const re = /^test\/e2e\/([^/]+)-flow\.gif$/;
80+
const changed = [];
81+
for (const f of files) {
82+
const m = f.filename.match(re);
83+
if (m) changed.push(
84+
{ flow: m[1], status: f.status, path: f.filename, prev: f.previous_filename });
85+
}
86+
changed.sort((a, b) => a.flow.localeCompare(b.flow));
87+
88+
async function upsertComment(body) {
89+
const { data: comments } = await github.rest.issues.listComments(
90+
{ owner, repo, issue_number: prNum, per_page: 100 });
91+
const existing = comments.find(c => c.body && c.body.includes(MARKER));
92+
if (existing) {
93+
await github.rest.issues.updateComment(
94+
{ owner, repo, comment_id: existing.id, body });
95+
} else {
96+
await github.rest.issues.createComment(
97+
{ owner, repo, issue_number: prNum, body });
98+
}
99+
}
100+
101+
if (changed.length === 0) {
102+
try {
103+
await upsertComment(
104+
`${MARKER}\n✅ **E2E snapshots** — no flow recordings changed vs \`${baseRef}\`.`);
105+
} catch (e) {
106+
core.warning(`comment failed (expected on forks): ${e}`);
107+
}
108+
return;
109+
}
110+
111+
const img = url => `<img src="${url}${bust}" width="480" alt="recording">`;
112+
const ICON = { added: '🆕', removed: '🗑️' };
113+
114+
const blocks = [];
115+
for (const { flow, status, path, prev } of changed) {
116+
const before = `${rawBase}/${prev || path}`;
117+
const after = rawHead ? `${rawHead}/${path}` : null;
118+
let inner;
119+
if (status === 'added') {
120+
inner = after ? `\n\n${img(after)}\n` : '\n\n_after unavailable_\n';
121+
} else if (status === 'removed') {
122+
inner = `\n\n${img(before)}\n`;
123+
} else {
124+
const afterCell = after ? img(after) : '<em>unavailable</em>';
125+
inner =
126+
`\n\n<table><tr><th>Before — <code>${baseRef}</code></th><th>After — this PR</th></tr>`
127+
+ `<tr><td>${img(before)}</td><td>${afterCell}</td></tr></table>\n`;
128+
}
129+
blocks.push(
130+
`<details><summary>${ICON[status] || '🔁'} ${flow} — ${status}</summary>${inner}</details>`);
131+
}
132+
133+
const body = [
134+
MARKER,
135+
'### 🎬 E2E snapshots changed',
136+
'',
137+
`These flow recordings differ from \`${baseRef}\` (added, removed, or modified). `
138+
+ 'Expand each for the before/after.',
139+
'',
140+
...blocks,
141+
'',
142+
`<sub>Run ${process.env.RUN_ID} · full set in the <b>skillz-e2e-snapshots</b> artifact.</sub>`,
143+
].join('\n');
144+
145+
try { await upsertComment(body); }
146+
catch (e) { core.warning(`comment failed (expected on forks): ${e}`); }
147+
148+
- name: Fail if any flow drifted from its golden
149+
if: always() && steps.record.outcome == 'failure'
150+
run: |
151+
echo "::error::E2E recordings drifted from their committed golden — review the skillz-e2e-snapshots artifact, then run ./test/e2e/run.sh --update <flow> if intended." >&2
152+
exit 1

.github/workflows/release.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ jobs:
157157
158158
nuget:
159159
name: 📦 Pack NuGet Tool
160-
needs: version
160+
needs: [version, build, release]
161161
runs-on: ubuntu-latest
162162
steps:
163163
- name: 📦 Checkout

test/e2e/.gitignore

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# Published binary (74 MB, regenerated by run.sh)
2+
bin/
3+
# Raw VHS recordings + extracted frame + screenshots (regenerated by run.sh)
4+
out/

test/e2e/README.md

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
# End-to-end terminal recordings (VHS)
2+
3+
This directory drives the **real `skillz` binary through its TUI** — one
4+
[VHS](https://github.com/charmbracelet/vhs) tape per command flow — and produces
5+
two artifacts per flow from a single recording:
6+
7+
| Artifact | Purpose |
8+
| --- | --- |
9+
| `<flow>-flow.gif` | Animated demo for PRs / README |
10+
| `<flow>-flow.golden.txt` | Final-frame text snapshot, diffed in CI as an integration test |
11+
12+
![skillz add — interactive flow](add-flow.gif)
13+
14+
The same recording is both the demo *and* the assertion: one tape, one run.
15+
16+
## Why a recording (we already have snapshot tests)
17+
18+
The in-process tests (`Skillz.Tests`) drive the prompts through Spectre's
19+
`TestConsole` — fast and deterministic, but they never exercise the *compiled*
20+
binary: `Program.cs`, DI wiring, `System.CommandLine` parsing, or real terminal
21+
rendering. This tier does, end to end, through a real PTY. It is the slow,
22+
high-confidence layer — a handful of representative flows, not exhaustive.
23+
24+
## The flows
25+
26+
| Flow | Drives | Validates |
27+
| --- | --- | --- |
28+
| `add` | interactive: skill multi-select → searchable agent picker → scope → method → confirm | the full interactive install, **symlink** path |
29+
| `copy` | `add --agent claude-code --copy --skill … -y` | non-interactive install, **copy** path + `Copied:` summary |
30+
| `global` | `add --agent claude-code --global --skill … -y` | **global scope** (`$HOME`-rooted paths) |
31+
| `init` | `init my-skill` | skill scaffolding + next-steps output |
32+
| `list` | `list` (after a hidden install) | the installed-skills table |
33+
| `remove` | interactive: multi-select → y/n confirm → summary | **interactive removal** + lock/symlink cleanup |
34+
| `update` | interactive scope picker → Both | the update check (hermetic: local skills are never network-checked) |
35+
| `error` | `add ./missing` | the **failure UX**: error message + non-zero exit |
36+
37+
Each flow's tape is the source of truth; keep `MARKERS`/`ALL_FLOWS` in
38+
[`run.sh`](run.sh) in sync with the tape set.
39+
40+
## How it works
41+
42+
```
43+
<flow>-flow.tape ──▶ VHS container (ttyd + ffmpeg) ──▶ <flow>-flow.gif
44+
└──▶ <flow>-flow.txt ──▶ extract-frame.sh ──▶ diff vs golden
45+
```
46+
47+
1. **`<flow>-flow.tape`** is a VHS script. Interactive flows gate key transitions
48+
on `Wait+Screen /.../` sentinels, so the recording syncs on **state** rather
49+
than wall-clock timing (short `Sleep`s are still used to let the UI settle).
50+
A hidden setup block puts the published binary on `PATH`, works in a throwaway
51+
`/tmp/work`, and (for `list`/`remove`/`update`) pre-installs fixtures so the
52+
demo has real state.
53+
2. **`run.sh`** publishes a self-contained `linux-x64` binary once, then for each
54+
flow mounts the repo read-only into the pinned VHS container and records.
55+
3. **`extract-frame.sh`** reduces VHS's multi-frame `.txt` capture to the final
56+
completed frame, keyed on a per-flow marker (e.g. `Done!`, `Successfully removed`).
57+
4. Each frame is diffed against `<flow>-flow.golden.txt`.
58+
59+
## Running it
60+
61+
```bash
62+
./test/e2e/run.sh # record + verify EVERY flow (what CI does)
63+
./test/e2e/run.sh remove update # record + verify only the named flows
64+
./test/e2e/run.sh --update # accept new output: refresh all goldens + GIFs
65+
./test/e2e/run.sh --update init # refresh a single flow
66+
REBUILD=1 ./test/e2e/run.sh # force re-publish of the binary first
67+
```
68+
69+
`run.sh` exits non-zero if any flow's frame differs from its golden (**FAIL**) or
70+
has no golden yet (**NEW**), and collects the changed/new GIFs, frames, and diffs
71+
under `out/report/` for CI.
72+
73+
Requirements: `docker` + the .NET SDK. Nothing else — `ttyd`, `ffmpeg`, and the
74+
fonts are baked into the pinned container.
75+
76+
**In this repo's devcontainer**, both are provided as features (Docker via
77+
`docker-in-docker`, .NET 10 via the `dotnet` feature) — run **Dev Containers:
78+
Rebuild Container** once, then `./test/e2e/run.sh` works as above. Docker-in-Docker
79+
needs a host that permits privileged containers.
80+
81+
## What makes it deterministic
82+
83+
Two independent runs produce a **byte-identical** final frame. The levers:
84+
85+
- **Assert on text, never on the GIF.** GIF bytes go through ffmpeg/gifski and are
86+
not stable across versions/platforms. The character grid (`.txt`) is.
87+
- **Final frame only.** Intermediate frames vary with timing; the end state does not.
88+
- **Pinned VHS image** (by digest) — a new VHS release can't silently reflow output.
89+
- **Fixed geometry / theme / `CursorBlink false`** in every tape.
90+
- **Hermetic inputs**: a local fixture (no network), a fixed `/tmp/work` cwd, and a
91+
pinned `$HOME` where it appears in output, so every path is constant. A clean
92+
container has no agent env (`AI_AGENT`, `CLAUDECODE`, …) and no agent config, so
93+
skillz renders the real interactive prompts with stable defaults.
94+
- **One skill where order matters.** The install report lists skills in discovery
95+
order (filesystem-dependent), so the non-interactive `copy`/`global`/`list` flows
96+
pin a single skill with `--skill`. `remove` lists skills sorted, and `update`
97+
never enumerates local skills, so those use all three fixtures.
98+
99+
If skillz legitimately changes its output (e.g. a new universal agent), the diff
100+
fails — that's the test working. Re-run with `--update <flow>` and commit the new
101+
golden + GIF.
102+
103+
## CI: artifacts + PR comment
104+
105+
The [`e2e-demo`](../../.github/workflows/e2e-demo.yml) workflow records and verifies
106+
every flow on pushes and PRs that touch the CLI, the tapes, or the fixtures, and:
107+
108+
- **Uploads** all GIFs plus `out/report/**` (changed frames, diffs, per-flow status)
109+
as the `skillz-e2e-snapshots` build artifact.
110+
- On a **PR**, posts a single collapsed comment — one expandable `<details>` per
111+
flow that **changed** (🔴) or is **new** (🆕) — with the recording inline. The
112+
GIFs are hosted on an `e2e-snapshots` side branch so GitHub renders them in the
113+
comment. When everything matches, the comment resets to a ✅ line.
114+
- **Fails the job** (after uploading + commenting) if any flow changed or is new.
115+
116+
> Inline hosting needs write access, so it is skipped for **fork** PRs (the GIFs
117+
> are still in the artifact). Same-repo PRs get the inline previews.
118+
119+
## Files
120+
121+
| File | |
122+
| --- | --- |
123+
| `<flow>-flow.tape` | The VHS script for a flow (source of truth) |
124+
| `<flow>-flow.golden.txt` | Committed final-frame snapshot |
125+
| `<flow>-flow.gif` | Committed demo (regenerate with `run.sh --update <flow>`) |
126+
| `run.sh` | Publish → record each flow → extract → verify/update + build report |
127+
| `extract-frame.sh` | VHS `.txt` → final frame (per-flow marker) |
128+
| `bin/`, `out/` | gitignored: published binary, raw recordings, report |
129+
130+
Fixture skills live in [`../fixtures/sample-skills`](../fixtures/sample-skills).

test/e2e/add-flow.gif

434 KB
Loading

test/e2e/add-flow.golden.txt

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
$ skillz add ./sample-skills
2+
3+
/$$$$$$ /$$ /$$ /$$ /$$
4+
/$$__ $$| $$ |__/| $$| $$
5+
| $$ \__/| $$ /$$ /$$| $$| $$ /$$$$$$$$
6+
| $$$$$$ | $$ /$$/| $$| $$| $$|____ /$$/
7+
\____ $$| $$$$$$/ | $$| $$| $$ /$$$$/
8+
/$$ \ $$| $$_ $$ | $$| $$| $$ /$$__/
9+
| $$$$$$/| $$ \ $$| $$| $$| $$ /$$$$$$$$
10+
\______/ |__/ \__/|__/|__/|__/|________/
11+
Source: /tmp/work/sample-skills
12+
Found 3 skill(s)
13+
Which agents do you want to install to? Selected: Claude Code (claude-code)
14+
Install 2 skill(s) [commit-helper, pr-describer] to 16 agent(s) [amp, antigravity, claude-code,
15+
cline, codex, cursor, deepagents, dexto, firebender, gemini-cli, github-copilot, kimi-cli,
16+
opencode, replit, universal, warp] [y/n] (y): y
17+
18+
┌─Installation Summary────────────────────────────────────────────────────────────────────────┐
19+
│ Canonical: /tmp/work/.agents/skills │
20+
│ Universal: Amp, Antigravity, Cline, Codex, Cursor, Deep Agents, Dexto, Firebender, Gemini │
21+
│ CLI, GitHub Copilot, Kimi Code CLI, OpenCode, Replit, Universal, Warp │
22+
│ Symlinked: Claude Code │
23+
└─────────────────────────────────────────────────────────────────────────────────────────────┘
24+
┌─Installed 2 skill(s)────────────────────────────────────────────────────────────────────────┐
25+
│ ✓ commit-helper │
26+
│ → /tmp/work/.agents/skills/commit-helper │
27+
│ ✓ pr-describer │
28+
│ → /tmp/work/.agents/skills/pr-describer │
29+
└─────────────────────────────────────────────────────────────────────────────────────────────┘
30+
31+
Done! Review skills before use; they run with full agent permissions.
32+
$

0 commit comments

Comments
 (0)