Train via Kaggle and publish versioned model to Hugging Face #25
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Train via Kaggle and publish versioned model to Hugging Face | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| version: | |
| description: "Version number (e.g. 1, 2, 3)" | |
| required: true | |
| default: "1" | |
| namespace: | |
| description: "Hugging Face namespace (user or org)" | |
| required: true | |
| default: "HyperlinksSpace" | |
| max_train_samples: | |
| description: "Training sample count" | |
| required: true | |
| default: "3000" | |
| max_eval_samples: | |
| description: "Evaluation sample count" | |
| required: true | |
| default: "600" | |
| epochs: | |
| description: "Number of epochs" | |
| required: true | |
| default: "2" | |
| batch_size: | |
| description: "Batch size" | |
| required: true | |
| default: "16" | |
| learning_rate: | |
| description: "Learning rate" | |
| required: true | |
| default: "1e-4" | |
| jobs: | |
| train-kaggle-publish-hf: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Install dependencies | |
| run: pip install --upgrade kaggle huggingface_hub | |
| - name: Configure Kaggle API credentials | |
| env: | |
| KAGGLE_USERNAME: ${{ secrets.KAGGLE_USERNAME }} | |
| KAGGLE_KEY: ${{ secrets.KAGGLE_KEY }} | |
| run: | | |
| python - <<'PY' | |
| import os | |
| username = os.getenv("KAGGLE_USERNAME", "").strip() | |
| key = os.getenv("KAGGLE_KEY", "").strip() | |
| if not username or not key: | |
| raise SystemExit("Missing KAGGLE_USERNAME or KAGGLE_KEY GitHub secrets.") | |
| with open(os.environ["GITHUB_ENV"], "a", encoding="utf-8") as fh: | |
| fh.write(f"KAGGLE_USERNAME_CLEAN={username}\n") | |
| fh.write(f"KAGGLE_KEY_CLEAN={key}\n") | |
| print("Kaggle credentials present.") | |
| PY | |
| - name: Resolve authenticated Kaggle owner | |
| env: | |
| KAGGLE_USERNAME: ${{ env.KAGGLE_USERNAME_CLEAN }} | |
| KAGGLE_KEY: ${{ env.KAGGLE_KEY_CLEAN }} | |
| run: | | |
| python - <<'PY' | |
| import os | |
| import subprocess | |
| from kaggle.api.kaggle_api_extended import KaggleApi | |
| expected_owner = os.getenv("KAGGLE_USERNAME", "").strip() | |
| key = os.getenv("KAGGLE_KEY", "").strip() | |
| if not key: | |
| raise SystemExit("Missing KAGGLE_KEY secret.") | |
| if not expected_owner: | |
| raise SystemExit("Missing KAGGLE_USERNAME secret.") | |
| # Force an authenticated API client using env vars; then read the resolved account. | |
| api = KaggleApi() | |
| api.authenticate() | |
| owner = str(api.config_values.get("username", "")).strip() | |
| if not owner: | |
| raise SystemExit("Kaggle API authentication succeeded but no username was resolved.") | |
| if owner.lower() != expected_owner.lower(): | |
| raise SystemExit( | |
| f"KAGGLE_USERNAME secret ({expected_owner}) does not match API key owner ({owner}). " | |
| "Update secrets so both belong to the same Kaggle account." | |
| ) | |
| # Verify credentials with a real authenticated CLI request before push. | |
| try: | |
| subprocess.check_output( | |
| ["kaggle", "kernels", "list", "--mine", "--page-size", "1"], | |
| text=True, | |
| stderr=subprocess.STDOUT, | |
| ) | |
| except subprocess.CalledProcessError as exc: | |
| raise SystemExit( | |
| "Kaggle CLI auth preflight failed. Regenerate KAGGLE_KEY for this account and update " | |
| "KAGGLE_USERNAME/KAGGLE_KEY secrets.\n" | |
| f"CLI output:\n{exc.output}" | |
| ) from exc | |
| print(f"Authenticated Kaggle owner: {owner}") | |
| with open("kaggle-owner.txt", "w", encoding="utf-8") as f: | |
| f.write(owner + "\n") | |
| PY | |
| owner="$(tr -d '\r\n' < kaggle-owner.txt)" | |
| echo "KAGGLE_OWNER=$owner" >> "$GITHUB_ENV" | |
| - name: Build and push Kaggle training kernel | |
| env: | |
| INPUT_VERSION: ${{ github.event.inputs.version }} | |
| KAGGLE_OWNER: ${{ env.KAGGLE_OWNER }} | |
| KAGGLE_USERNAME: ${{ env.KAGGLE_OWNER }} | |
| KAGGLE_KEY: ${{ env.KAGGLE_KEY_CLEAN }} | |
| INPUT_MAX_TRAIN_SAMPLES: ${{ github.event.inputs.max_train_samples }} | |
| INPUT_MAX_EVAL_SAMPLES: ${{ github.event.inputs.max_eval_samples }} | |
| INPUT_EPOCHS: ${{ github.event.inputs.epochs }} | |
| INPUT_BATCH_SIZE: ${{ github.event.inputs.batch_size }} | |
| INPUT_LEARNING_RATE: ${{ github.event.inputs.learning_rate }} | |
| GITHUB_REPOSITORY: ${{ github.repository }} | |
| GITHUB_SHA: ${{ github.sha }} | |
| GITHUB_RUN_ID: ${{ github.run_id }} | |
| GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} | |
| run: | | |
| python - <<'PY' | |
| import json | |
| import os | |
| import time | |
| from pathlib import Path | |
| version = os.environ["INPUT_VERSION"].strip() | |
| if not version.isdigit() or int(version) < 1: | |
| raise SystemExit("version must be a positive integer.") | |
| owner = os.environ["KAGGLE_OWNER"].strip() | |
| if not owner: | |
| raise SystemExit("Missing resolved KAGGLE_OWNER.") | |
| repo = os.environ["GITHUB_REPOSITORY"].strip() | |
| sha = os.environ["GITHUB_SHA"].strip() | |
| max_train_samples = os.environ["INPUT_MAX_TRAIN_SAMPLES"].strip() | |
| max_eval_samples = os.environ["INPUT_MAX_EVAL_SAMPLES"].strip() | |
| epochs = os.environ["INPUT_EPOCHS"].strip() | |
| batch_size = os.environ["INPUT_BATCH_SIZE"].strip() | |
| learning_rate = os.environ["INPUT_LEARNING_RATE"].strip() | |
| run_id = os.environ["GITHUB_RUN_ID"].strip() | |
| run_attempt = os.environ["GITHUB_RUN_ATTEMPT"].strip() | |
| nonce = str(int(time.time())) | |
| # Keep slug short and place uniqueness early to avoid platform slug truncation collisions. | |
| run_tail = run_id[-6:] if len(run_id) > 6 else run_id | |
| slug = f"tm-v{version}-{nonce}-{sha[:6]}-{run_tail}a{run_attempt}" | |
| workspace = Path(".kaggle_kernel") | |
| workspace.mkdir(parents=True, exist_ok=True) | |
| out_dir = f"/kaggle/working/TinyModel{version}" | |
| notebook = { | |
| "cells": [ | |
| { | |
| "cell_type": "code", | |
| "execution_count": None, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "!pip -q install -U huggingface_hub transformers datasets tokenizers\n", | |
| f"!git clone https://github.com/{repo}.git /kaggle/working/TinyModel\n", | |
| "!cd /kaggle/working/TinyModel && git checkout " + sha + "\n", | |
| f"!python /kaggle/working/TinyModel/scripts/train_tinymodel1_agnews.py --output-dir {out_dir} --max-train-samples {max_train_samples} --max-eval-samples {max_eval_samples} --epochs {epochs} --batch-size {batch_size} --learning-rate {learning_rate}\n", | |
| f"!ls -la {out_dir}\n", | |
| ], | |
| } | |
| ], | |
| "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}}, | |
| "nbformat": 4, | |
| "nbformat_minor": 5, | |
| } | |
| (workspace / "notebook.ipynb").write_text(json.dumps(notebook), encoding="utf-8") | |
| kernel_meta = { | |
| "id": f"{owner}/{slug}", | |
| "title": slug, | |
| "code_file": "notebook.ipynb", | |
| "language": "python", | |
| "kernel_type": "notebook", | |
| "is_private": "true", | |
| "enable_gpu": "true", | |
| "enable_internet": "true", | |
| "dataset_sources": [], | |
| "competition_sources": [], | |
| "kernel_sources": [], | |
| } | |
| (workspace / "kernel-metadata.json").write_text(json.dumps(kernel_meta, indent=2), encoding="utf-8") | |
| print(f"Created kernel {owner}/{slug}") | |
| (workspace / "kernel-slug.txt").write_text(slug + "\n", encoding="utf-8") | |
| PY | |
| slug="$(tr -d '\r\n' < .kaggle_kernel/kernel-slug.txt)" | |
| echo "KAGGLE_KERNEL_SLUG=$slug" >> "$GITHUB_ENV" | |
| kaggle kernels push -p ".kaggle_kernel" | |
| - name: Wait for Kaggle kernel completion | |
| env: | |
| KAGGLE_OWNER: ${{ env.KAGGLE_OWNER }} | |
| KAGGLE_USERNAME: ${{ env.KAGGLE_OWNER }} | |
| KAGGLE_KEY: ${{ env.KAGGLE_KEY_CLEAN }} | |
| INPUT_VERSION: ${{ github.event.inputs.version }} | |
| run: | | |
| python - <<'PY' | |
| import json | |
| import os | |
| import re | |
| import shutil | |
| import subprocess | |
| import time | |
| owner = os.environ["KAGGLE_OWNER"].strip() | |
| slug = os.environ["KAGGLE_KERNEL_SLUG"].strip() | |
| version = os.environ["INPUT_VERSION"].strip() | |
| ref = f"{owner}/{slug}" | |
| print(f"Waiting for kernel: {ref}") | |
| timeout_sec = 3 * 60 * 60 | |
| start = time.time() | |
| terminal_ok = {"complete", "completed", "success", "succeeded"} | |
| terminal_fail = {"error", "failed", "cancelled", "canceled"} | |
| while True: | |
| try: | |
| # Output-first check: if Kaggle already finalized artifacts, proceed immediately. | |
| probe_dir = ".kaggle_probe" | |
| if os.path.isdir(probe_dir): | |
| shutil.rmtree(probe_dir, ignore_errors=True) | |
| os.makedirs(probe_dir, exist_ok=True) | |
| try: | |
| probe_out = subprocess.check_output( | |
| ["kaggle", "kernels", "output", ref, "-p", probe_dir], | |
| text=True, | |
| stderr=subprocess.STDOUT, | |
| ) | |
| has_artifacts = os.path.isdir(os.path.join(probe_dir, f"TinyModel{version}")) | |
| print(f"output_probe_has_artifacts={has_artifacts}") | |
| print(f"output_probe_raw_repr={probe_out.strip()!r}") | |
| if has_artifacts: | |
| print("Kaggle outputs are available; proceeding without additional status polling.") | |
| break | |
| except subprocess.CalledProcessError as probe_exc: | |
| print("output probe not ready; falling back to status polling...") | |
| print(probe_exc.output.strip()) | |
| out = subprocess.check_output( | |
| ["kaggle", "kernels", "status", ref], | |
| text=True, | |
| stderr=subprocess.STDOUT, | |
| ) | |
| raw = out.strip() | |
| print(f"status_cli_raw_repr={raw!r}") | |
| value = "" | |
| if raw.startswith("{"): | |
| status = json.loads(raw) | |
| value = str(status.get("status", "")).lower() | |
| else: | |
| for line in raw.splitlines(): | |
| line_l = line.strip().lower() | |
| if line_l.startswith("status:"): | |
| value = line_l.split(":", 1)[1].strip() | |
| break | |
| if not value: | |
| print("status output not yet parseable; retrying...") | |
| print(raw) | |
| if time.time() - start > timeout_sec: | |
| raise SystemExit("Timed out waiting for Kaggle kernel completion.") | |
| time.sleep(30) | |
| continue | |
| normalized = re.sub(r"[^a-z]", "", value.lower()) | |
| elapsed = int(time.time() - start) | |
| print(f"status_raw={value} status_normalized={normalized} elapsed_s={elapsed}") | |
| if normalized in terminal_ok: | |
| break | |
| if normalized in terminal_fail: | |
| raise SystemExit(f"Kaggle kernel failed with status={value}") | |
| except subprocess.CalledProcessError as exc: | |
| # Newly pushed kernels can return transient lookup/auth errors for a short time. | |
| print("status lookup failed; retrying...") | |
| print(exc.output.strip()) | |
| if time.time() - start > timeout_sec: | |
| raise SystemExit("Timed out waiting for Kaggle kernel completion.") | |
| time.sleep(30) | |
| PY | |
| - name: Download Kaggle output artifacts | |
| env: | |
| KAGGLE_OWNER: ${{ env.KAGGLE_OWNER }} | |
| KAGGLE_USERNAME: ${{ env.KAGGLE_OWNER }} | |
| KAGGLE_KEY: ${{ env.KAGGLE_KEY_CLEAN }} | |
| INPUT_VERSION: ${{ github.event.inputs.version }} | |
| run: | | |
| mkdir -p ".kaggle_output" | |
| kaggle kernels output "${KAGGLE_OWNER}/${KAGGLE_KERNEL_SLUG}" -p ".kaggle_output" | |
| test -d ".kaggle_output/TinyModel${{ github.event.inputs.version }}" | |
| - name: Publish TinyModel{version} to Hugging Face | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| run: | | |
| python scripts/publish_hf_artifact.py \ | |
| --namespace "${{ github.event.inputs.namespace }}" \ | |
| --name "TinyModel${{ github.event.inputs.version }}" \ | |
| --repo-type model \ | |
| --source-dir ".kaggle_output/TinyModel${{ github.event.inputs.version }}" |