Skip to content

Commit a1a5d87

Browse files
JesperDramschclaude
JesperDramsch
and
claude
authored
ci: add canonical source update workflow (#205)
Add webhook-triggered workflow for aggregating conference data from canonical sources (PSF calendar ICS and python-organizers CSV). - Accepts repository_dispatch webhooks with source type parameter - Runs existing merge scripts (pixi run merge) - Commits to auto/conference-updates branch (same as changedetect workflow) - Creates/updates rolling PR using same pattern as existing workflow - Supports manual trigger with dry-run option for testing This complements the changedetect workflow by handling deterministic imports from canonical data sources without Claude API calls. Co-authored-by: Claude <noreply@anthropic.com>
1 parent d3437a7 commit a1a5d87

File tree

1 file changed

+306
-0
lines changed

1 file changed

+306
-0
lines changed

‎.github/workflows/canonical-source-update.yml‎

Lines changed: 306 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,306 @@
1+
name: Canonical Source Update
2+
3+
on:
4+
# Triggered by IFTTT/changedetection webhooks when canonical sources update
5+
repository_dispatch:
6+
types: [canonical-source-update]
7+
8+
# Manual trigger for testing
9+
workflow_dispatch:
10+
inputs:
11+
source:
12+
description: 'Source to update from'
13+
required: true
14+
type: choice
15+
options:
16+
- all
17+
- ics
18+
- csv
19+
default: all
20+
dry_run:
21+
description: 'Dry run (no commit/PR)'
22+
required: false
23+
type: boolean
24+
default: false
25+
26+
# Prevent concurrent runs to avoid merge conflicts on the accumulator branch
27+
concurrency:
28+
group: canonical-source-update
29+
cancel-in-progress: false
30+
31+
permissions:
32+
contents: write
33+
pull-requests: write
34+
35+
env:
36+
UPDATE_BRANCH: auto/conference-updates
37+
38+
jobs:
39+
update-from-canonical:
40+
runs-on: ubuntu-latest
41+
timeout-minutes: 15
42+
43+
steps:
44+
- name: Checkout repository
45+
uses: actions/checkout@v6
46+
with:
47+
fetch-depth: 0
48+
token: ${{ github.token }}
49+
persist-credentials: true
50+
51+
- name: Configure git identity
52+
run: |
53+
git config user.name "github-actions[bot]"
54+
git config user.email "github-actions[bot]@users.noreply.github.com"
55+
56+
- name: Parse trigger source
57+
id: source
58+
env:
59+
EVENT_NAME: ${{ github.event_name }}
60+
PAYLOAD_SOURCE: ${{ github.event.client_payload.source }}
61+
INPUT_SOURCE: ${{ inputs.source }}
62+
run: |
63+
if [ "$EVENT_NAME" = "repository_dispatch" ]; then
64+
# Webhook trigger - get source from payload
65+
SOURCE="${PAYLOAD_SOURCE:-all}"
66+
echo "Triggered by webhook for source: $SOURCE"
67+
else
68+
# Manual trigger
69+
SOURCE="${INPUT_SOURCE:-all}"
70+
echo "Manual trigger for source: $SOURCE"
71+
fi
72+
73+
# Validate source
74+
case "$SOURCE" in
75+
all|ics|csv)
76+
echo "source=$SOURCE" >> $GITHUB_OUTPUT
77+
;;
78+
*)
79+
echo "::warning title=Unknown Source::Unknown source '$SOURCE', defaulting to 'all'"
80+
echo "source=all" >> $GITHUB_OUTPUT
81+
;;
82+
esac
83+
84+
echo "## Trigger Info" >> $GITHUB_STEP_SUMMARY
85+
echo "- **Event:** $EVENT_NAME" >> $GITHUB_STEP_SUMMARY
86+
echo "- **Source:** $SOURCE" >> $GITHUB_STEP_SUMMARY
87+
88+
- name: Setup accumulator branch
89+
run: |
90+
git remote set-url origin "https://x-access-token:${{ github.token }}@github.com/${{ github.repository }}.git"
91+
92+
if git ls-remote --heads origin $UPDATE_BRANCH | grep -q $UPDATE_BRANCH; then
93+
echo "::notice title=Branch::Checking out existing accumulator branch"
94+
git fetch origin $UPDATE_BRANCH
95+
git checkout $UPDATE_BRANCH
96+
97+
# Sync with main - try rebase first, then merge
98+
if ! git rebase origin/main; then
99+
echo "::warning title=Rebase Failed::Attempting merge instead"
100+
git rebase --abort 2>/dev/null || true
101+
if ! git merge origin/main --no-edit; then
102+
git merge --abort 2>/dev/null || true
103+
echo "::error title=Branch Sync Failed::Could not rebase or merge with main"
104+
exit 1
105+
fi
106+
fi
107+
else
108+
echo "::notice title=Branch::Creating new accumulator branch"
109+
git checkout -b $UPDATE_BRANCH
110+
fi
111+
112+
- name: Snapshot data files
113+
run: |
114+
cp _data/conferences.yml /tmp/conferences_before.yml
115+
cp _data/archive.yml /tmp/archive_before.yml 2>/dev/null || true
116+
117+
- name: Setup Pixi
118+
uses: prefix-dev/setup-pixi@v0.9.3
119+
120+
- name: Run merge scripts
121+
id: merge
122+
env:
123+
SOURCE: ${{ steps.source.outputs.source }}
124+
run: |
125+
echo "::group::Running merge for source: $SOURCE"
126+
127+
case "$SOURCE" in
128+
ics)
129+
echo "Importing from Python official calendar (ICS)..."
130+
pixi run python ./utils/import_python_official.py
131+
pixi run sort
132+
;;
133+
csv)
134+
echo "Importing from Python organizers (CSV)..."
135+
pixi run python ./utils/import_python_organizers.py
136+
pixi run sort
137+
;;
138+
all)
139+
echo "Running full merge pipeline..."
140+
pixi run merge
141+
;;
142+
esac
143+
144+
echo "::endgroup::"
145+
146+
# Capture which source was processed for commit message
147+
echo "source_label=$SOURCE" >> $GITHUB_OUTPUT
148+
149+
- name: Check for changes
150+
id: check_changes
151+
run: |
152+
# Check if any data files changed
153+
CHANGED=false
154+
155+
if ! diff -q _data/conferences.yml /tmp/conferences_before.yml > /dev/null 2>&1; then
156+
echo "conferences.yml has changes"
157+
CHANGED=true
158+
fi
159+
160+
if [ -f /tmp/archive_before.yml ] && ! diff -q _data/archive.yml /tmp/archive_before.yml > /dev/null 2>&1; then
161+
echo "archive.yml has changes"
162+
CHANGED=true
163+
fi
164+
165+
echo "changed=$CHANGED" >> $GITHUB_OUTPUT
166+
167+
if [ "$CHANGED" = "true" ]; then
168+
echo "::notice title=Changes Detected::Data files were updated"
169+
170+
# Show summary of changes
171+
echo "## Changes" >> $GITHUB_STEP_SUMMARY
172+
echo "\`\`\`diff" >> $GITHUB_STEP_SUMMARY
173+
diff -u /tmp/conferences_before.yml _data/conferences.yml | head -50 >> $GITHUB_STEP_SUMMARY || true
174+
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
175+
else
176+
echo "::notice title=No Changes::Data files are up to date"
177+
echo "## No Changes" >> $GITHUB_STEP_SUMMARY
178+
echo "Canonical sources did not have any new data." >> $GITHUB_STEP_SUMMARY
179+
fi
180+
181+
- name: Commit and push
182+
if: steps.check_changes.outputs.changed == 'true' && inputs.dry_run != true
183+
id: commit
184+
env:
185+
SOURCE: ${{ steps.source.outputs.source }}
186+
run: |
187+
git remote set-url origin "https://x-access-token:${{ github.token }}@github.com/${{ github.repository }}.git"
188+
189+
# Stage data files
190+
git add _data/conferences.yml _data/archive.yml _data/legacy.yml 2>/dev/null || git add _data/conferences.yml
191+
192+
# Create descriptive commit message based on source
193+
case "$SOURCE" in
194+
ics)
195+
COMMIT_MSG="conf: data merge from PSF calendar"
196+
;;
197+
csv)
198+
COMMIT_MSG="conf: data merge from python-organizers"
199+
;;
200+
all)
201+
COMMIT_MSG="conf: data merge"
202+
;;
203+
esac
204+
205+
git commit -m "$COMMIT_MSG" -m "Source: canonical-$SOURCE" -m "Triggered: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
206+
git push origin $UPDATE_BRANCH --force-with-lease
207+
208+
echo "commit_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
209+
echo "commit_msg=$COMMIT_MSG" >> $GITHUB_OUTPUT
210+
211+
- name: Create or update PR
212+
if: steps.check_changes.outputs.changed == 'true' && inputs.dry_run != true
213+
uses: actions/github-script@v8
214+
env:
215+
UPDATE_BRANCH: ${{ env.UPDATE_BRANCH }}
216+
SOURCE: ${{ steps.source.outputs.source }}
217+
COMMIT_SHA: ${{ steps.commit.outputs.commit_sha }}
218+
COMMIT_MSG: ${{ steps.commit.outputs.commit_msg }}
219+
with:
220+
script: |
221+
const branch = process.env.UPDATE_BRANCH;
222+
const source = process.env.SOURCE;
223+
const sha = process.env.COMMIT_SHA.substring(0, 7);
224+
const date = new Date().toISOString().split('T')[0];
225+
226+
// Source labels for the "Conference" column (matches existing workflow format)
227+
// Using 📊 emoji to distinguish canonical source updates from website scrapes
228+
const sourceLabels = {
229+
'ics': '📊 PSF Calendar',
230+
'csv': '📊 python-organizers',
231+
'all': '📊 Canonical sources'
232+
};
233+
const sourceLabel = sourceLabels[source] || `📊 ${source}`;
234+
235+
// Check for existing PR from the update branch
236+
const { data: prs } = await github.rest.pulls.list({
237+
owner: context.repo.owner,
238+
repo: context.repo.repo,
239+
head: `${context.repo.owner}:${branch}`,
240+
state: 'open'
241+
});
242+
243+
// Use same table format as check-conference-update.yml (Conference | Commit | Date)
244+
const entry = `| ${sourceLabel} | [${sha}](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/commit/${process.env.COMMIT_SHA}) | ${date} |`;
245+
246+
if (prs.length > 0) {
247+
// Update existing PR - append to table
248+
const pr = prs[0];
249+
let body = pr.body || '';
250+
251+
// Add entry before the END_UPDATES marker
252+
body = body.replace(/(<!-- END_UPDATES -->)/, `${entry}\n$1`);
253+
254+
await github.rest.pulls.update({
255+
owner: context.repo.owner,
256+
repo: context.repo.repo,
257+
pull_number: pr.number,
258+
body
259+
});
260+
261+
console.log(`Updated PR #${pr.number}`);
262+
core.notice(`Updated PR #${pr.number} with canonical source merge`);
263+
} else {
264+
// Create new PR - use EXACT same format as check-conference-update.yml
265+
const { data: newPr } = await github.rest.pulls.create({
266+
owner: context.repo.owner,
267+
repo: context.repo.repo,
268+
title: '🐍 Conference updates',
269+
head: branch,
270+
base: 'main',
271+
body: `## 🐍 Automated Conference Updates\n\n| Conference | Commit | Date |\n|------------|--------|------|\n${entry}\n<!-- END_UPDATES -->`
272+
});
273+
274+
// Add labels
275+
await github.rest.issues.addLabels({
276+
owner: context.repo.owner,
277+
repo: context.repo.repo,
278+
issue_number: newPr.number,
279+
labels: ['automated', 'conference-update']
280+
});
281+
282+
console.log(`Created PR #${newPr.number}`);
283+
core.notice(`Created PR #${newPr.number} for canonical source merge`);
284+
}
285+
286+
- name: Summary
287+
if: always()
288+
env:
289+
SOURCE: ${{ steps.source.outputs.source }}
290+
CHANGED: ${{ steps.check_changes.outputs.changed }}
291+
DRY_RUN: ${{ inputs.dry_run }}
292+
run: |
293+
echo "" >> $GITHUB_STEP_SUMMARY
294+
echo "## Result" >> $GITHUB_STEP_SUMMARY
295+
296+
if [ "$CHANGED" = "true" ]; then
297+
if [ "$DRY_RUN" = "true" ]; then
298+
echo "- Status: Changes detected (dry run - not committed)" >> $GITHUB_STEP_SUMMARY
299+
else
300+
echo "- Status: ✅ Changes committed and PR updated" >> $GITHUB_STEP_SUMMARY
301+
fi
302+
else
303+
echo "- Status: ✓ No new data from canonical sources" >> $GITHUB_STEP_SUMMARY
304+
fi
305+
306+
echo "- Source: $SOURCE" >> $GITHUB_STEP_SUMMARY

0 commit comments

Comments
 (0)