Render and Deploy Website and Update Freeze Cache

Render and Deploy Website and Update Freeze Cache #299

Workflow file for this run

.github/workflows/quarto-publish-update-cache.yml at 0cc33b7

	# ╔══════════════════════════════════════════════════════════════════════╗
	# ║ Nightly / Manual Full Render and Freeze Cache Update ║
	# ║ ║
	# ║ Purpose: Ensure the freeze cache is comprehensive and up-to-date. ║
	# ║ - Restores existing freeze cache (no longer deletes it) ║
	# ║ - Pre-renders ALL non-frozen QMD files (populates workspace ║
	# ║ _freeze BEFORE babelquarto runs, so the saved cache is full) ║
	# ║ - Renders the full site via babelquarto (pandoc-only, fast) ║
	# ║ - Adds Julia support ║
	# ║ - Saves updated freeze cache (even on failure) ║
	# ║ - Optional: force full rebuild via manual dispatch ║
	# ╚══════════════════════════════════════════════════════════════════════╝

	on:
	workflow_dispatch:
	inputs:
	force_full_rebuild:
	description: 'Force full rebuild (delete freeze cache and re-execute all files)'
	type: boolean
	default: false
	schedule:
	# Run every day at 11 PM UTC
	- cron: '0 23 * * *'

	name: Render and Deploy Website and Update Freeze Cache

	env:
	RUST_BACKTRACE: 1

	jobs:
	build-deploy:
	runs-on: ubuntu-latest
	timeout-minutes: 240
	# Use a dedicated concurrency group so nightly builds do NOT share the
	# same group as the main "Github Page" push workflow. The main workflow
	# has cancel-in-progress: true; if they shared a group, every push to
	# main would cancel any in-progress nightly build before it could save
	# the freeze cache.
	concurrency:
	group: quarto-build-nightly-cache-update
	env:
	GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
	TMP_DIR: /mnt/TMP
	permissions:
	contents: write
	pages: write
	id-token: write
	steps:
	- name: Checkout repo
	uses: actions/checkout@v4
	with:
	fetch-depth: 0

	# ── Restore freeze cache ─────────────────────────────────
	# Restore the latest available freeze cache. The primary key uses the
	# same prefix as the main workflow ("freeze-<OS>-") so both workflows
	# share the same cache pool.
	- name: Restore freeze cache
	id: freeze-restore
	if: inputs.force_full_rebuild != true
	uses: actions/cache/restore@v4
	with:
	path: _freeze
	key: freeze-${{ runner.os }}-nightly-${{ github.run_id }}-${{ github.run_attempt }}
	restore-keys: \|
	freeze-${{ runner.os }}-nightly-
	freeze-${{ runner.os }}-

	- name: Clear freeze for forced full rebuild
	if: inputs.force_full_rebuild == true
	run: \|
	rm -rf _freeze
	echo "🗑️ Freeze cache cleared – full rebuild will execute all files"

	- name: Report freeze cache status
	run: \|
	if [ -d "_freeze" ]; then
	FREEZE_COUNT=$(find _freeze -name "*.json" -type f 2>/dev/null \| wc -l)
	echo "✅ Freeze cache available: $FREEZE_COUNT frozen output(s)"
	else
	echo "⚠️ No freeze cache – full execution required"
	fi

	# ── Quarto Setup ────────────────────────────────────────
	- name: Set up Quarto
	uses: quarto-dev/quarto-actions/setup@v2
	env:
	GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	with:
	tinytex: true

	# ── R Setup ──────────────────────────────────────────────
	- uses: r-lib/actions/setup-r@v2
	with:
	use-public-rspm: true

	- name: Cache R packages
	uses: actions/cache@v4
	with:
	path: \|
	~/.local/share/renv
	~/.cache/R/renv
	/usr/local/lib/R/site-library
	~/R
	key: r-packages-${{ runner.os }}-${{ hashFiles('DESCRIPTION') }}
	restore-keys: \|
	r-packages-${{ runner.os }}-

	- uses: r-lib/actions/setup-r-dependencies@v2
	env:
	PKG_SYSREQS: "true"

	# ── Python Setup ─────────────────────────────────────────
	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: '3.11'
	cache: 'pip'

	- name: Install Python dependencies
	run: \|
	python -m pip install --upgrade pip
	if [ -f requirements.txt ]; then
	pip install -r requirements.txt
	fi

	# ── Julia Setup ──────────────────────────────────────────
	- name: Set up Julia
	uses: julia-actions/setup-julia@v2
	with:
	version: '1'

	- name: Cache Julia packages
	uses: actions/cache@v4
	with:
	path: ~/.julia
	key: julia-packages-${{ runner.os }}-${{ hashFiles('Julia/Project.toml') }}
	restore-keys: \|
	julia-packages-${{ runner.os }}-

	- name: Install Julia packages
	run: \|
	# Find the QuartoNotebookRunner version required by this Quarto installation.
	# Quarto bundles its own julia-engine Project.toml that pins an exact QNR version.
	JULIA_ENGINE_PROJ=$(find /opt/quarto -name "Project.toml" -path "/julia-engine/" 2>/dev/null \| head -1)
	QNR_VERSION=""
	if [ -n "$JULIA_ENGINE_PROJ" ]; then
	QNR_VERSION=$(grep -oP 'QuartoNotebookRunner = "=([^"]+)"' "$JULIA_ENGINE_PROJ" \
	\| grep -oP '[0-9]+\.[0-9]+\.[0-9]+' \|\| true)
	echo "Quarto requires QuartoNotebookRunner=${QNR_VERSION}"
	fi

	# Create a dedicated QNR environment. Setting QUARTO_JULIA_PROJECT to this
	# path tells Quarto to skip ensureQuartoNotebookRunnerEnvironment (which calls
	# Pkg.update() and needs network at render time) and simply verify QNR loads.
	QNR_ENV="${HOME}/.julia/environments/quarto-julia-env"
	mkdir -p "${QNR_ENV}"

	if [ -n "$QNR_VERSION" ]; then
	julia --project="${QNR_ENV}" -e "
	using Pkg
	Pkg.add(name=\"QuartoNotebookRunner\", version=\"${QNR_VERSION}\")
	Pkg.instantiate()
	using QuartoNotebookRunner
	println(\"QuartoNotebookRunner \$(pkgversion(QuartoNotebookRunner)) ready\")
	"
	else
	julia --project="${QNR_ENV}" -e '
	using Pkg
	Pkg.add("QuartoNotebookRunner")
	Pkg.instantiate()
	using QuartoNotebookRunner
	println("QuartoNotebookRunner $(pkgversion(QuartoNotebookRunner)) ready")
	'
	fi

	# Export so the render step can pick it up.
	echo "QUARTO_JULIA_PROJECT=${QNR_ENV}" >> "$GITHUB_ENV"

	# Pre-instantiate the tutorial project environment (Julia/Project.toml).
	# QNR workers use JULIA_PROJECT=@. and cd into the notebook directory,
	# so they automatically pick up Julia/Project.toml for CairoMakie etc.
	julia --project=Julia -e '
	using Pkg
	Pkg.resolve()
	Pkg.instantiate()
	using CairoMakie, DataFrames, Statistics
	println("Tutorial packages ready")
	'

	# ── Dependency Scanning ──────────────────────────────────
	- name: Scan and update dependencies
	run: \|
	Rscript update-dependencies.R
	python manage-dependencies.py --update

	- name: Commit updated dependency files
	run: \|
	git config --local user.name "$GITHUB_ACTOR"
	git config --local user.email "$GITHUB_ACTOR@users.noreply.github.com"
	git add DESCRIPTION requirements.txt Julia/Project.toml
	git commit -m 'chore: update dependencies' \|\| echo "No changes to commit"
	git push origin \|\| echo "No changes to push"

	# ── Render ───────────────────────────────────────────────
	- name: Create temporary directory
	run: \|
	sudo mkdir -p ${{ env.TMP_DIR }}
	sudo chown -R $USER:$USER ${{ env.TMP_DIR }}

	- name: Check disk usage before rendering
	run: \|
	echo "=== Initial Disk Usage ==="
	df -h

	# Pre-render Julia notebooks to avoid OOM during the R-heavy main render
	- name: Pre-render Julia notebooks
	env:
	QUARTO_JULIA_PROJECT: ${{ env.QUARTO_JULIA_PROJECT }}
	TMPDIR: ${{ env.TMP_DIR }}
	TMP: ${{ env.TMP_DIR }}
	TEMP: ${{ env.TMP_DIR }}
	run: \|
	echo "Pre-rendering Julia tutorials..."
	failed=0
	for qmd in Julia/*.qmd; do
	echo " → $qmd"
	if quarto render "$qmd" 2>&1; then
	echo " ✓ OK: $qmd"
	else
	echo " ✗ Failed: $qmd"
	failed=1
	fi
	done
	julia_log="${HOME}/.cache/quarto/julia/julia_server_log.txt"
	if [ -f "$julia_log" ]; then
	echo "=== QNR server log ==="
	tail -50 "$julia_log"
	echo "======================"
	fi
	[ $failed -eq 0 ] \|\| exit 1

	# ── Pre-render ALL non-frozen R QMD files ────────────────
	# ROOT CAUSE FIX: babelquarto::render_website() renders in a temp dir,
	# creates _freeze there, copies only _site back, then deletes the temp
	# dir. The workspace _freeze is NEVER updated by babelquarto, so the
	# "Save freeze cache" step below would save a nearly-empty directory –
	# making the cache useless for the next run.
	#
	# FIX: pre-render every non-frozen .qmd here so the WORKSPACE _freeze
	# is fully populated BEFORE babelquarto runs. babelquarto copies the
	# workspace _freeze into its temp dir and skips code execution (pandoc
	# only), cutting babelquarto from ~90 min down to ~20 min.
	# The saved cache will now contain all frozen outputs (~500+ JSON files).
	#
	# Timing:
	# Cold cache (first run): pre-render ~60-80 min + babelquarto ~20 min
	# Warm cache (subsequent): pre-render 0 files + babelquarto ~20 min
	- name: Pre-render all non-frozen QMD files
	env:
	TMPDIR: ${{ env.TMP_DIR }}
	TMP: ${{ env.TMP_DIR }}
	TEMP: ${{ env.TMP_DIR }}
	QUARTO_JULIA_PROJECT: ${{ env.QUARTO_JULIA_PROJECT }}
	run: \|
	echo "Pre-rendering non-frozen QMD files to populate workspace _freeze..."
	RENDERED=0
	SKIPPED=0
	FAILED=0

	while IFS= read -r -d '' qmd; do
	rel="${qmd#./}"

	# Julia files are handled by the dedicated Julia pre-render step above
	[[ "$rel" == Julia/* ]] && { SKIPPED=$((SKIPPED+1)); continue; }

	# Skip if a _freeze entry already exists for this file
	freeze_dir="_freeze/${rel%.qmd}"
	if [ -d "$freeze_dir" ]; then
	SKIPPED=$((SKIPPED+1))
	continue
	fi

	echo " → $rel"
	if quarto render "$qmd" 2>&1; then
	RENDERED=$((RENDERED+1))
	else
	echo " ⚠️ Warning: failed to render $rel (non-fatal)"
	FAILED=$((FAILED+1))
	fi
	done < <(find . -name "*.qmd" \
	-not -path "./_freeze/*" \
	-not -path "./_site/*" \
	-not -path "./.quarto/*" \
	-print0 \| sort -z)

	echo ""
	echo "📊 Pre-render summary: $RENDERED rendered, $SKIPPED skipped (frozen/Julia), $FAILED failed"
	[ $FAILED -gt 0 ] && echo "::warning::$FAILED QMD file(s) failed to pre-render"
	TOTAL_FROZEN=$(find _freeze -name "*.json" -type f 2>/dev/null \| wc -l)
	echo "❄️ Freeze cache now has $TOTAL_FROZEN frozen output(s) (JSON entries)"

	# ── Generate skill ZIP ───────────────────────────────────
	# Must run BEFORE babelquarto so the zip exists at project root when
	# babelquarto copies the project to its temp dir for rendering.
	# Quarto then copies it to _site/ via the resources entry in _quarto.yml.
	- name: Generate unified skill ZIP package
	run: \|
	SKILL_TMP=$(mktemp -d)
	cp SKILL.md "${SKILL_TMP}/SKILL.md"
	cp files/gallery_data.csv "${SKILL_TMP}/gallery_data.csv"
	cp files/gallery_data_zh.csv "${SKILL_TMP}/gallery_data_zh.csv"
	cp files/bizard-skill-readme.md "${SKILL_TMP}/README.md"
	(cd "${SKILL_TMP}" && zip -r - SKILL.md gallery_data.csv gallery_data_zh.csv README.md) > bizard-skill.zip
	rm -rf "${SKILL_TMP}"
	echo "✅ Created bizard-skill.zip at project root ($(du -h bizard-skill.zip \| cut -f1))"

	- name: Render website with Babelquarto
	env:
	TMPDIR: ${{ env.TMP_DIR }}
	TMP: ${{ env.TMP_DIR }}
	TEMP: ${{ env.TMP_DIR }}
	R_COMPILE_PKGS: 0
	QUARTO_JULIA_PROJECT: ${{ env.QUARTO_JULIA_PROJECT }}
	run: Rscript -e 'babelquarto::render_website()'

	# ── Fix zh page figure files ──────────────────────────────
	# See quarto-publish.yml for full explanation. Short version: the
	# frozen chunk HTML in *.zh.qmd freeze entries references
	# <name>.zh_files/figure-html/ but babelquarto never places the actual
	# PNG files there. Copy the identical English figures over.
	- name: Fix zh page figure files
	run: \|
	echo "Copying figure files into zh output directories..."
	COPIED=0
	while IFS= read -r -d '' zh_html; do
	zh_dir=$(dirname "$zh_html")
	name=$(basename "$zh_html" .html)

	rel="${zh_dir#_site/zh}"
	rel="${rel#/}"

	if [ -z "$rel" ]; then
	eng_files="_site/${name}_files"
	else
	eng_files="_site/${rel}/${name}_files"
	fi

	zh_files="${zh_dir}/${name}.zh_files"

	if [ -d "$eng_files" ] && [ ! -d "$zh_files" ]; then
	cp -r "$eng_files" "$zh_files"
	echo " ✓ Copied: $eng_files → $zh_files"
	COPIED=$((COPIED+1))
	fi
	done < <(find _site/zh -name "*.html" -print0)
	echo "✅ Done: copied figure files for ${COPIED} zh pages"

	- name: Check disk usage after rendering
	run: \|
	echo "=== Final Disk Usage ==="
	df -h

	# ── Save freeze cache (ALWAYS – even on failure) ─────────
	# Use the same "freeze-<OS>-" prefix as the main workflow so that the
	# main workflow's restore-keys can find this nightly-built cache.
	# The weekly date in the key limits cache growth to ~7 entries (GitHub
	# evicts cache entries unused for more than 7 days). The run_attempt
	# suffix makes each attempt within a week a distinct immutable entry.
	- name: Get weekly date for cache key
	id: date
	run: echo "week=$(date -u +'%Y-W%V')" >> "$GITHUB_OUTPUT"

	- name: Save freeze cache
	if: always()
	uses: actions/cache/save@v4
	with:
	path: _freeze
	key: freeze-${{ runner.os }}-nightly-${{ steps.date.outputs.week }}-${{ github.run_attempt }}

	# Also save under the shared prefix so both the main workflow and the
	# next nightly run can restore from it via the "freeze-<OS>-" prefix.
	- name: Save freeze cache (shared key)
	if: always()
	uses: actions/cache/save@v4
	with:
	path: _freeze
	key: freeze-${{ runner.os }}-nightly-${{ github.run_id }}

	# ── Deploy ───────────────────────────────────────────────
	- name: Upload Quarto artifacts
	uses: actions/upload-pages-artifact@v3
	with:
	path: "_site"

	- name: Deploy to GitHub Pages
	uses: quarto-dev/quarto-actions/publish@v2
	with:
	target: gh-pages
	render: false
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

	- name: Cleanup temporary directory
	if: always()
	run: \|
	TMP_PATH="${{ env.TMP_DIR }}"
	if [ -n "$TMP_PATH" ] && [[ "$TMP_PATH" =~ ^/mnt/TMP ]] && [ -d "$TMP_PATH" ]; then
	sudo rm -rf "$TMP_PATH"
	echo "Removed $TMP_PATH directory"
	else
	echo "Skipping cleanup - validation failed (path: $TMP_PATH)"
	fi

Navigation Menu

Uh oh!

Render and Deploy Website and Update Freeze Cache #299

Workflow file

Render and Deploy Website and Update Freeze Cache #299

Uh oh!

Workflow file for this run