Extract data #1844

	name: Extract data

	on:
	push:
	branches:
	- main
	schedule:
	- cron: "0 1 * * *"

	jobs:
	build:
	runs-on: ubuntu-latest
	steps:
	- name: checkout repo content
	uses: actions/checkout@v4

	- name: install git lfs
	run: \|
	curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh \| sudo bash
	sudo apt-get install git-lfs
	git lfs install

	- name: Install uv
	uses: astral-sh/setup-uv@v4

	- name: setup python
	run: uv python install 3.10

	- name: install python packages
	run: uv sync

	- name: track large files
	run: git lfs track "database/pipeline.pkl"

	- name: execute py script # run run.py to get the latest data
	env:
	TWITTER_TOKEN: ${{ secrets.TWITTER_TOKEN }}
	HACKERNEWS_PASSWORD: ${{ secrets.HACKERNEWS_PASSWORD }}
	HACKERNEWS_USERNAME: ${{ secrets.HACKERNEWS_USERNAME }}
	ZOTERO_API_KEY: ${{ secrets.ZOTERO_API_KEY }}
	ZOTERO_LIBRARY_ID: ${{ secrets.ZOTERO_LIBRARY_ID }}
	HUGGINGFACE_TOKEN: ${{ secrets.HUGGINGFACE_TOKEN }}
	run: uv run python run.py

	- name: commit files
	run: \|
	git config --local user.email "[email protected]"
	git config --local user.name "GitHub Action"
	git add .gitattributes
	git add -A
	git commit -m "update data" -a

	- name: push changes
	uses: ad-m/[email protected]
	with:
	github_token: ${{ secrets.GITHUB_TOKEN }}
	branch: main

Provide feedback