diff --git a/.github/workflows/deploy-eng.yml b/.github/workflows/deploy-eng.yml index 3c45fdde1..b5857a2c3 100644 --- a/.github/workflows/deploy-eng.yml +++ b/.github/workflows/deploy-eng.yml @@ -2,6 +2,12 @@ name: Eng - Build and Deploy on: workflow_dispatch: + inputs: + bust_cache: + description: 'Force fresh API fetch (bypass disk cache)' + required: false + type: boolean + default: false schedule: # Here are the times for the cron: # @@ -28,6 +34,7 @@ jobs: env: BUILD_LANG: ${{ matrix.languages }} + BUST_CACHE: ${{ inputs.bust_cache }} ALGOLIA_API_KEY: ${{ secrets.ALGOLIA_API_KEY }} ALGOLIA_APP_ID: ${{ secrets.ALGOLIA_APP_ID }} @@ -71,8 +78,16 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile + - name: Restore Turbo cache + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: node_modules/.cache/turbo + key: turbo-${{ matrix.languages }}-${{ github.sha }} + restore-keys: | + turbo-${{ matrix.languages }}- + - name: Build site - run: pnpm run build + run: pnpm turbo run build env: NODE_ENV: production diff --git a/.github/workflows/deploy-i18n.yml b/.github/workflows/deploy-i18n.yml index e69fc5736..fd7f635e1 100644 --- a/.github/workflows/deploy-i18n.yml +++ b/.github/workflows/deploy-i18n.yml @@ -2,6 +2,12 @@ name: i18n - Build and Deploy on: workflow_dispatch: + inputs: + bust_cache: + description: 'Force fresh API fetch (bypass disk cache)' + required: false + type: boolean + default: false schedule: # Here are the times for the cron: # @@ -29,6 +35,7 @@ jobs: env: BUILD_LANG: ${{ matrix.languages }} + BUST_CACHE: ${{ inputs.bust_cache }} ALGOLIA_API_KEY: ${{ secrets.ALGOLIA_API_KEY }} ALGOLIA_APP_ID: ${{ secrets.ALGOLIA_APP_ID }} @@ -88,8 +95,16 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile + - name: Restore Turbo cache + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: node_modules/.cache/turbo + key: turbo-${{ matrix.languages }}-${{ github.sha }} + restore-keys: | + turbo-${{ matrix.languages }}- + - name: Build site - run: pnpm run build + run: pnpm turbo run build env: NODE_ENV: production diff --git a/.gitignore b/.gitignore index dadb350bc..3a4727418 100644 --- a/.gitignore +++ b/.gitignore @@ -83,3 +83,6 @@ cypress/screenshots config/i18n/locales/**/trending.json docker/languages/ + +# Turborepo +.turbo diff --git a/package.json b/package.json index 5b3a00b2c..bdba47fcd 100644 --- a/package.json +++ b/package.json @@ -25,11 +25,12 @@ "cypress:run:espanol": "pnpm run cypress -- run --spec 'cypress/e2e/espanol/**/*'", "cypress:watch": "pnpm run cypress -- open", "dev": "pnpm run develop", + "fetch:data": "node scripts/fetch-data.js", "predevelop": "pnpm run clean && node ./tools/download-trending.js", "develop": "cross-env ELEVENTY_ENV=dev NODE_OPTIONS=--max-old-space-size=8192 eleventy --serve", "predevelop:ci": "pnpm run clean && node ./tools/download-trending.js", "develop:ci": "cross-env ELEVENTY_ENV=ci NODE_OPTIONS=--max-old-space-size=8192 eleventy --serve", - "prebuild": "pnpm run clean && node ./tools/download-trending.js", + "prebuild": "pnpm run clean", "build": "cross-env ELEVENTY_ENV=prod NODE_OPTIONS=--max-old-space-size=8192 eleventy", "build:ci": "cross-env ELEVENTY_ENV=ci eleventy", "postbuild": "node ./config/i18n/generate-serve-config.js", @@ -109,6 +110,7 @@ "probe-image-size": "7.2.3", "shx": "0.4.0", "terser": "5.44.1", + "turbo": "^2.8.14", "typescript": "5.9.3" }, "pnpm": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5aab91510..098db5444 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -130,6 +130,9 @@ importers: terser: specifier: 5.44.1 version: 5.44.1 + turbo: + specifier: ^2.8.14 + version: 2.8.14 typescript: specifier: 5.9.3 version: 5.9.3 @@ -1049,6 +1052,7 @@ packages: engines: { node: '>= 10' } cpu: [arm64] os: [linux] + libc: [glibc] '@napi-rs/nice-linux-arm64-musl@1.1.1': resolution: @@ -1058,6 +1062,7 @@ packages: engines: { node: '>= 10' } cpu: [arm64] os: [linux] + libc: [musl] '@napi-rs/nice-linux-ppc64-gnu@1.1.1': resolution: @@ -1067,6 +1072,7 @@ packages: engines: { node: '>= 10' } cpu: [ppc64] os: [linux] + libc: [glibc] '@napi-rs/nice-linux-riscv64-gnu@1.1.1': resolution: @@ -1076,6 +1082,7 @@ packages: engines: { node: '>= 10' } cpu: [riscv64] os: [linux] + libc: [glibc] '@napi-rs/nice-linux-s390x-gnu@1.1.1': resolution: @@ -1085,6 +1092,7 @@ packages: engines: { node: '>= 10' } cpu: [s390x] os: [linux] + libc: [glibc] '@napi-rs/nice-linux-x64-gnu@1.1.1': resolution: @@ -1094,6 +1102,7 @@ packages: engines: { node: '>= 10' } cpu: [x64] os: [linux] + libc: [glibc] '@napi-rs/nice-linux-x64-musl@1.1.1': resolution: @@ -1103,6 +1112,7 @@ packages: engines: { node: '>= 10' } cpu: [x64] os: [linux] + libc: [musl] '@napi-rs/nice-openharmony-arm64@1.1.1': resolution: @@ -1441,6 +1451,7 @@ packages: } cpu: [arm64] os: [linux] + libc: [glibc] '@unrs/resolver-binding-linux-arm64-musl@1.11.1': resolution: @@ -1449,6 +1460,7 @@ packages: } cpu: [arm64] os: [linux] + libc: [musl] '@unrs/resolver-binding-linux-ppc64-gnu@1.11.1': resolution: @@ -1457,6 +1469,7 @@ packages: } cpu: [ppc64] os: [linux] + libc: [glibc] '@unrs/resolver-binding-linux-riscv64-gnu@1.11.1': resolution: @@ -1465,6 +1478,7 @@ packages: } cpu: [riscv64] os: [linux] + libc: [glibc] '@unrs/resolver-binding-linux-riscv64-musl@1.11.1': resolution: @@ -1473,6 +1487,7 @@ packages: } cpu: [riscv64] os: [linux] + libc: [musl] '@unrs/resolver-binding-linux-s390x-gnu@1.11.1': resolution: @@ -1481,6 +1496,7 @@ packages: } cpu: [s390x] os: [linux] + libc: [glibc] '@unrs/resolver-binding-linux-x64-gnu@1.11.1': resolution: @@ -1489,6 +1505,7 @@ packages: } cpu: [x64] os: [linux] + libc: [glibc] '@unrs/resolver-binding-linux-x64-musl@1.11.1': resolution: @@ -1497,6 +1514,7 @@ packages: } cpu: [x64] os: [linux] + libc: [musl] '@unrs/resolver-binding-wasm32-wasi@1.11.1': resolution: @@ -5798,6 +5816,61 @@ packages: integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w== } + turbo-darwin-64@2.8.14: + resolution: + { + integrity: sha512-9sFi7n2lLfEsGWi5OEoA/eTtQU2BPKtzSYKqufMtDeRmqMT9vKjbv9gJCRkllSVE9BOXA0qXC3diyX8V8rKIKw== + } + cpu: [x64] + os: [darwin] + + turbo-darwin-arm64@2.8.14: + resolution: + { + integrity: sha512-aS4yJuy6A1PCLws+PJpZP0qCURG8Y5iVx13z/WAbKyeDTY6W6PiGgcEllSaeLGxyn++382ztN/EZH85n2zZ6VQ== + } + cpu: [arm64] + os: [darwin] + + turbo-linux-64@2.8.14: + resolution: + { + integrity: sha512-XC6wPUDJkakjhNLaS0NrHDMiujRVjH+naEAwvKLArgqRaFkNxjmyNDRM4eu3soMMFmjym6NTxYaF74rvET+Orw== + } + cpu: [x64] + os: [linux] + + turbo-linux-arm64@2.8.14: + resolution: + { + integrity: sha512-ChfE7isyVNjZrVSPDwcfqcHLG/FuIBbOFxnt1FM8vSuBGzHAs8AlTdwFNIxlEMJfZ8Ad9mdMxdmsCUPIWiQ6cg== + } + cpu: [arm64] + os: [linux] + + turbo-windows-64@2.8.14: + resolution: + { + integrity: sha512-FTbIeQL1ycLFW2t9uQNMy+bRSzi3Xhwun/e7ZhFBdM+U0VZxxrtfYEBM9CHOejlfqomk6Jh7aRz0sJoqYn39Hg== + } + cpu: [x64] + os: [win32] + + turbo-windows-arm64@2.8.14: + resolution: + { + integrity: sha512-KgZX12cTyhY030qS7ieT8zRkhZZE2VWJasDFVUSVVn17nR7IShpv68/7j5UqJNeRLIGF1XPK0phsP5V5yw3how== + } + cpu: [arm64] + os: [win32] + + turbo@2.8.14: + resolution: + { + integrity: sha512-UCTxeMNYT1cKaHiIFdLCQ7ulI+jw5i5uOnJOrRXsgUD7G3+OjlUjwVd7JfeVt2McWSVGjYA3EVW/v1FSsJ5DtA== + } + hasBin: true + tweetnacl@0.14.5: resolution: { @@ -9832,6 +9905,33 @@ snapshots: dependencies: safe-buffer: 5.2.1 + turbo-darwin-64@2.8.14: + optional: true + + turbo-darwin-arm64@2.8.14: + optional: true + + turbo-linux-64@2.8.14: + optional: true + + turbo-linux-arm64@2.8.14: + optional: true + + turbo-windows-64@2.8.14: + optional: true + + turbo-windows-arm64@2.8.14: + optional: true + + turbo@2.8.14: + optionalDependencies: + turbo-darwin-64: 2.8.14 + turbo-darwin-arm64: 2.8.14 + turbo-linux-64: 2.8.14 + turbo-linux-arm64: 2.8.14 + turbo-windows-64: 2.8.14 + turbo-windows-arm64: 2.8.14 + tweetnacl@0.14.5: {} type-check@0.4.0: diff --git a/scripts/fetch-data.js b/scripts/fetch-data.js new file mode 100644 index 000000000..458d98242 --- /dev/null +++ b/scripts/fetch-data.js @@ -0,0 +1,28 @@ +import { fetchFromHashnode } from '../utils/hashnode/fetch-from-hashnode.js'; +import { fetchFromGhost } from '../utils/ghost/fetch-from-ghost.js'; + +try { + console.log('Downloading trending data...'); + await import('../tools/download-trending.js'); + console.log('Trending data downloaded.'); + + console.log('Fetching data from Hashnode and Ghost...'); + + const hashnodePosts = await fetchFromHashnode('posts'); + const hashnodePages = await fetchFromHashnode('pages'); + const ghostPosts = await fetchFromGhost('posts'); + const ghostPages = await fetchFromGhost('pages'); + + console.log( + [ + 'Fetch summary:', + ` Hashnode posts: ${hashnodePosts.length}`, + ` Hashnode pages: ${hashnodePages.length}`, + ` Ghost posts: ${ghostPosts.length}`, + ` Ghost pages: ${ghostPages.length}` + ].join('\n') + ); +} catch (err) { + console.error('fetch-data failed:', err); + process.exit(1); +} diff --git a/turbo.json b/turbo.json new file mode 100644 index 000000000..88b8a904e --- /dev/null +++ b/turbo.json @@ -0,0 +1,48 @@ +{ + "$schema": "https://turbo.build/schema.json", + "globalDependencies": ["pnpm-lock.yaml"], + "tasks": { + "fetch:data": { + "cache": false + }, + "build": { + "dependsOn": ["fetch:data"], + "inputs": [ + "src/**", + "utils/**", + "config/**", + ".eleventy.js", + "package.json", + ".cache/**" + ], + "outputs": ["dist/**", "docker/languages/**"], + "env": [ + "ELEVENTY_ENV", + "BUILD_LANG", + "LOCALE_FOR_UI", + "LOCALE_FOR_GHOST", + "SITE_DOMAIN", + "POSTS_PER_PAGE", + "ADS_ENABLED", + "DO_NOT_FETCH_FROM_GHOST", + "HASHNODE_API_URL" + ] + }, + "lint:code": { + "outputs": [] + }, + "lint:pretty": { + "outputs": [] + }, + "lint:i18n-schema": { + "outputs": [] + }, + "test": { + "outputs": [], + "env": ["ELEVENTY_ENV"] + }, + "type-check": { + "outputs": [] + } + } +} diff --git a/utils/disk-cache.js b/utils/disk-cache.js new file mode 100644 index 000000000..184b44cd5 --- /dev/null +++ b/utils/disk-cache.js @@ -0,0 +1,45 @@ +import fs from 'fs'; +import path from 'path'; + +export function readCache(filePath, ttlMs) { + if (process.env.BUST_CACHE === 'true') { + console.log(`[cache] bust: ${filePath}`); + return null; + } + + let raw; + try { + raw = fs.readFileSync(filePath, 'utf-8'); + } catch { + console.log(`[cache] miss: ${filePath}`); + return null; + } + + let parsed; + try { + parsed = JSON.parse(raw); + } catch { + console.log(`[cache] corrupt: ${filePath}`); + return null; + } + + if (typeof parsed.cachedAt !== 'number') { + console.log(`[cache] corrupt (missing cachedAt): ${filePath}`); + return null; + } + + const age = Date.now() - parsed.cachedAt; + if (age > ttlMs) { + console.log(`[cache] miss (expired): ${filePath}`); + return null; + } + + console.log(`[cache] hit: ${filePath}`); + return parsed.data; +} + +export function writeCache(filePath, data) { + const dir = path.dirname(filePath); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(filePath, JSON.stringify({ cachedAt: Date.now(), data })); +} diff --git a/utils/disk-cache.test.js b/utils/disk-cache.test.js new file mode 100644 index 000000000..50c1107ed --- /dev/null +++ b/utils/disk-cache.test.js @@ -0,0 +1,99 @@ +import fs from 'fs'; +import path from 'path'; +import os from 'os'; +import { readCache, writeCache } from './disk-cache.js'; + +let tmpDir; + +beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'disk-cache-test-')); +}); + +afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + delete process.env.BUST_CACHE; +}); + +describe('readCache', () => { + test('returns data when cache is fresh', () => { + const filePath = path.join(tmpDir, 'fresh.json'); + const data = [{ id: 1, title: 'test' }]; + fs.writeFileSync(filePath, JSON.stringify({ cachedAt: Date.now(), data })); + + const result = readCache(filePath, 60_000); + expect(result).toEqual(data); + }); + + test('returns null when cache is expired', () => { + const filePath = path.join(tmpDir, 'expired.json'); + const data = [{ id: 1 }]; + fs.writeFileSync( + filePath, + JSON.stringify({ cachedAt: Date.now() - 120_000, data }) + ); + + const result = readCache(filePath, 60_000); + expect(result).toBeNull(); + }); + + test('returns null when file does not exist', () => { + const filePath = path.join(tmpDir, 'nonexistent.json'); + + const result = readCache(filePath, 60_000); + expect(result).toBeNull(); + }); + + test('returns null when BUST_CACHE=true even if fresh', () => { + const filePath = path.join(tmpDir, 'bust.json'); + const data = [{ id: 1 }]; + fs.writeFileSync(filePath, JSON.stringify({ cachedAt: Date.now(), data })); + + process.env.BUST_CACHE = 'true'; + const result = readCache(filePath, 60_000); + expect(result).toBeNull(); + }); + + test('returns null when cachedAt is missing from JSON', () => { + const filePath = path.join(tmpDir, 'no-timestamp.json'); + fs.writeFileSync(filePath, JSON.stringify({ data: [{ id: 1 }] })); + + const result = readCache(filePath, 60_000); + expect(result).toBeNull(); + }); + + test('returns null on corrupt/invalid JSON', () => { + const filePath = path.join(tmpDir, 'corrupt.json'); + fs.writeFileSync(filePath, '{not valid json!!!'); + + const result = readCache(filePath, 60_000); + expect(result).toBeNull(); + }); +}); + +describe('writeCache', () => { + test('creates parent directory if missing', () => { + const filePath = path.join(tmpDir, '.cache', 'nested', 'data.json'); + + writeCache(filePath, [{ id: 1 }]); + + expect(fs.existsSync(filePath)).toBe(true); + const written = JSON.parse(fs.readFileSync(filePath, 'utf-8')); + expect(written).toHaveProperty('cachedAt'); + expect(written.data).toEqual([{ id: 1 }]); + }); + + test('overwrites existing cache file', () => { + const filePath = path.join(tmpDir, 'overwrite.json'); + fs.writeFileSync( + filePath, + JSON.stringify({ cachedAt: 1000, data: [{ old: true }] }) + ); + + const newData = [{ new: true }]; + writeCache(filePath, newData); + + const written = JSON.parse(fs.readFileSync(filePath, 'utf-8')); + expect(written.data).toEqual(newData); + expect(written.cachedAt).toBeGreaterThan(1000); + }); +}); diff --git a/utils/ghost/fetch-from-ghost.js b/utils/ghost/fetch-from-ghost.js index cbc232fd3..1222e80fa 100644 --- a/utils/ghost/fetch-from-ghost.js +++ b/utils/ghost/fetch-from-ghost.js @@ -1,6 +1,10 @@ +import { resolve } from 'path'; import { ghostAPI } from '../api.js'; +import { readCache, writeCache } from '../disk-cache.js'; import { wait } from '../wait.js'; +const CACHE_TTL = 12 * 60 * 60 * 1000; + export const fetchFromGhost = async endpoint => { let currPage = 1; let lastPage = 5; @@ -18,6 +22,17 @@ export const fetchFromGhost = async endpoint => { return []; } + const cacheFilePath = resolve( + import.meta.dirname, + '../../.cache', + `ghost-${endpoint}.json` + ); + const cached = readCache(cacheFilePath, CACHE_TTL); + if (cached) { + console.log(`Using cached Ghost ${endpoint} data`); + return cached; + } + while (currPage && currPage <= lastPage) { const ghostRes = await ghostAPI[endpoint] .browse({ @@ -39,5 +54,9 @@ export const fetchFromGhost = async endpoint => { await wait(200); } + if (data.length > 0) { + writeCache(cacheFilePath, data); + } + return data; }; diff --git a/utils/hashnode/fetch-from-hashnode.js b/utils/hashnode/fetch-from-hashnode.js index f62136c9a..370195e6f 100644 --- a/utils/hashnode/fetch-from-hashnode.js +++ b/utils/hashnode/fetch-from-hashnode.js @@ -1,15 +1,33 @@ import { gql, request } from 'graphql-request'; -import { join } from 'path'; +import { join, resolve } from 'path'; import { hashnodeHost } from '../api.js'; import { wait } from '../wait.js'; import { loadJSON } from '../load-json.js'; +import { readCache, writeCache } from '../disk-cache.js'; import { config } from '../../config/index.js'; const { eleventyEnv, currentLocale_i18n, hashnodeAPIURL } = config; +const CACHE_TTL = 12 * 60 * 60 * 1000; + export const fetchFromHashnode = async contentType => { if (!hashnodeHost) return []; + + const cacheFilePath = resolve( + import.meta.dirname, + '../../.cache', + `hashnode-${contentType}.json` + ); + + if (eleventyEnv !== 'ci') { + const cached = readCache(cacheFilePath, CACHE_TTL); + if (cached) { + console.log(`Using cached Hashnode ${contentType} data`); + return cached; + } + } + const fieldName = contentType === 'posts' ? 'posts' : 'staticPages'; const postFieldsFragment = gql` @@ -159,5 +177,9 @@ export const fetchFromHashnode = async contentType => { await wait(200); } + if (eleventyEnv !== 'ci' && data.length > 0) { + writeCache(cacheFilePath, data); + } + return data; };