feat: validation and consumer overhaull

compose.dev.yaml

@@ -7,6 +7,7 @@ services:
             POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
             POSTGRES_USER: ${POSTGRES_USER}
             POSTGRES_DB: ${POSTGRES_DB}
+            TZ: Etc/UTC
         ports:
             - '127.0.0.1:${POSTGRES_PORT}:5432'
         healthcheck:

compose.yaml

@@ -7,6 +7,7 @@ services:
             POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
             POSTGRES_USER: ${POSTGRES_USER}
             POSTGRES_DB: ${POSTGRES_DB}
+            TZ: Etc/UTC
         ports:
             - '127.0.0.1:${POSTGRES_PORT}:5432'
         healthcheck:

experiments/verify-validate/src/main.ts

@@ -1,7 +1,8 @@
-import { PackumentSchema } from '@npm.rest/validate/packument';
-import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
+import { PackumentSchema, type License } from '@npm.rest/validate/packument';
+import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises';
 import { packumentTable } from '@npm.rest/db/schema';
 import { db } from '@npm.rest/db/server';
+import { uniqueDeep } from './unique';
 import { existsSync } from 'node:fs';
 import { join } from 'node:path';
 import * as v from 'valibot';
@@ -28,19 +29,23 @@ const ISSUES_DIR = join(OUTPUT_DIR, './issues');
 if (!existsSync(ISSUES_DIR)) {
 	await mkdir(ISSUES_DIR, { recursive: true });
 } else {
-	const clear = await confirm({
-		message: 'clean issues directory',
-		initialValue: false,
-	});
-
-	if (isCancel(clear)) {
-		cancel('exiting');
-		process.exit(1);
-	}
+	const files = await readdir(ISSUES_DIR);
+
+	if (files.length) {
+		const clear = await confirm({
+			message: 'clean issues directory',
+			initialValue: false,
+		});
+
+		if (isCancel(clear)) {
+			cancel('exiting');
+			process.exit(1);
+		}
 
-	if (clear) {
-		await rm(ISSUES_DIR, { recursive: true });
-		await mkdir(ISSUES_DIR, { recursive: true });
+		if (clear) {
+			await rm(ISSUES_DIR, { recursive: true });
+			await mkdir(ISSUES_DIR, { recursive: true });
+		}
 	}
 }
 
@@ -59,6 +64,47 @@ async function saveCheckpoint(offset: number) {
 	await writeFile(CHECKPOINT_FILE, JSON.stringify({ offset }));
 }
 
+const ALL_REPO_TYPES_FILE = join(OUTPUT_DIR, './all-repository-types.json');
+const ALL_FUNDING_TYPES_FILE = join(OUTPUT_DIR, './all-funding-types.json');
+const REPO_TYPE_COUNTS_FILE = join(OUTPUT_DIR, './repo-type-counts.json');
+const REPO_GIT_HOSTNAME_COUNTS_FILE = join(
+	OUTPUT_DIR,
+	'./repo-git-hostname-counts.json',
+);
+const FUNDING_TYPE_COUNTS_FILE = join(OUTPUT_DIR, './funding-type-counts.json');
+const LICENSES_FILE = join(OUTPUT_DIR, './licenses.json');
+
+async function getThingy(path: string) {
+	if (!existsSync(path)) {
+		return {} as Record<string, unknown[]>;
+	}
+
+	const raw = await readFile(path, 'utf-8');
+	return JSON.parse(raw) as Record<string, unknown[]>;
+}
+
+async function saveThingy(path: string, data: Record<string, unknown[]>) {
+	for (const value of Object.values(data)) {
+		uniqueDeep(value);
+	}
+
+	await writeFile(path, JSON.stringify(data, null, 2));
+}
+
+let allFundingTypes = await getThingy(ALL_FUNDING_TYPES_FILE);
+let allRepoTypes = await getThingy(ALL_REPO_TYPES_FILE);
+
+// Keep only 'unknown' types for the saved unknown lists
+allFundingTypes = { unknown: allFundingTypes.unknown ?? [] };
+allRepoTypes = { unknown: allRepoTypes.unknown ?? [] };
+
+// Prepare map for git repo hostname counts
+const gitHostnameCounts: Record<string, number> = {};
+
+const licenses = (await getThingy(LICENSES_FILE)) as {
+	l?: (string | License)[];
+};
+
 let { offset } = await getCheckpoint();
 let processed = 0;
 let issues = 0;
@@ -71,13 +117,27 @@ function msg() {
 
 s.start(msg());
 
+// function tryExtractType(rawPkg: unknown, version: string, key: string) {
+// 	// @ts-expect-error necessary evil
+// 	// oxlint-disable-next-line typescript-eslint(no-unsafe-assignment), typescript-eslint(no-unsafe-member-access)
+// 	const f = rawPkg?.versions?.[version]?.[key];
+// 	const a = Array.isArray(f) ? f : [f];
+// 	const t = a
+// 		// oxlint-disable-next-line typescript-eslint(no-unsafe-return), typescript-eslint(no-unsafe-member-access)
+// 		.map((f) => (f && typeof f === 'object' && 'type' in f ? f : null))
+// 		.filter((t) => t !== null);
+
+// 	// oxlint-disable-next-line typescript-eslint(no-unsafe-return)
+// 	return t;
+// }
+
 while (true) {
 	const packuments = await db
 		.select({ data: packumentTable.data })
 		.from(packumentTable)
 		.orderBy(packumentTable.id)
 		.offset(offset)
-		.limit(100);
+		.limit(1000);
 
 	if (packuments.length === 0) {
 		break;
@@ -91,6 +151,86 @@ while (true) {
 			const result = await v.safeParseAsync(PackumentSchema, pkg.data);
 
 			if (result.success) {
+				const { versions } = result.output;
+
+				if (versions) {
+					for (const [_version, pkv] of Object.entries(versions)) {
+						// if (pkv.funding?.some((f) => f.type === 'unknown')) {
+						// 	const types = tryExtractType(
+						// 		pkg.data,
+						// 		version,
+						// 		'funding',
+						// 	);
+
+						// 	if (types.length === 0) {
+						// 		unknownFundingType.unknown ??= [];
+						// 		unknownFundingType.unknown.push(`pkg:${name}`);
+						// 	} else {
+						// 		for (const item of types) {
+						// 			// oxlint-disable-next-linetypescript-eslint(no-unsafe-member-access)
+						// 			(unknownFundingType[item.type] ??= []).push(
+						// 				item,
+						// 			);
+						// 		}
+						// 	}
+						// }
+
+						// if (pkv.repository?.some((r) => r.type === 'unknown')) {
+						// 	const types = tryExtractType(
+						// 		pkg.data,
+						// 		version,
+						// 		'repository',
+						// 	);
+
+						// 	if (types.length === 0) {
+						// 		unknownRepoType.unknown ??= [];
+						// 		unknownRepoType.unknown.push(`pkg:${name}`);
+						// 	} else {
+						// 		for (const item of types) {
+						// 			// oxlint-disable-next-linetypescript-eslint(no-unsafe-member-access)
+						// 			(unknownRepoType[item.type] ??= []).push(
+						// 				item,
+						// 			);
+						// 		}
+						// 	}
+						// }
+
+						for (const f of pkv.funding ?? []) {
+							if (f.type === 'unknown') {
+								allFundingTypes[f.type] ??= [];
+								allFundingTypes[f.type].push(f.url);
+							}
+						}
+
+						for (const r of pkv.repository ?? []) {
+							if (r.type === 'unknown') {
+								allRepoTypes[r.type] ??= [];
+								allRepoTypes[r.type].push(r.url);
+							}
+
+							if (r.type === 'git' && r.url) {
+								const hostname = new URL(r.url).hostname;
+								gitHostnameCounts[hostname] =
+									(gitHostnameCounts[hostname] ?? 0) + 1;
+							}
+						}
+
+						for (const l of pkv.license ?? []) {
+							licenses.l ??= [];
+
+							if (
+								!l.type ||
+								Object.entries(l).filter(([, v]) => v !== null)
+									.length > 1
+							) {
+								licenses.l.push(l);
+							} else {
+								licenses.l.push(l.type);
+							}
+						}
+					}
+				}
+
 				processed++;
 				return;
 			}
@@ -112,6 +252,32 @@ while (true) {
 
 	s.message(msg());
 	await saveCheckpoint(offset);
+	await saveThingy(ALL_FUNDING_TYPES_FILE, allFundingTypes);
+	await saveThingy(ALL_REPO_TYPES_FILE, allRepoTypes);
+	await saveThingy(LICENSES_FILE, licenses);
+
+	// Save counts of each unknown type
+	const fundingCounts = Object.fromEntries(
+		Object.entries(allFundingTypes).map(([type, arr]) => [
+			type,
+			arr.length,
+		]),
+	);
+	await writeFile(
+		FUNDING_TYPE_COUNTS_FILE,
+		JSON.stringify(fundingCounts, null, 2),
+	);
+
+	const repoCounts = Object.fromEntries(
+		Object.entries(allRepoTypes).map(([type, arr]) => [type, arr.length]),
+	);
+	await writeFile(REPO_TYPE_COUNTS_FILE, JSON.stringify(repoCounts, null, 2));
+
+	// Save git hostname counts
+	await writeFile(
+		REPO_GIT_HOSTNAME_COUNTS_FILE,
+		JSON.stringify(gitHostnameCounts, null, 2),
+	);
 
 	if (issues >= 10) {
 		s.error('too many issues, stopping');

experiments/verify-validate/src/repo-hostname-count.ts

@@ -0,0 +1,30 @@
+// Script to display top 10 Git hostname counts
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+
+const OUTPUT_DIR = join(import.meta.dirname, '../output');
+const COUNTS_FILE = join(OUTPUT_DIR, 'repo-git-hostname-counts.json');
+
+async function main() {
+	try {
+		const raw = await readFile(COUNTS_FILE, 'utf-8');
+		const data = JSON.parse(raw) as Record<string, number>;
+
+		const total = Object.values(data).reduce((sum, v) => sum + v, 0);
+		const top = Object.entries(data)
+			.toSorted((a, b) => b[1] - a[1])
+			.slice(0, 10);
+
+		console.log('Top 10 Git hostnames:');
+		for (const [hostname, count] of top) {
+			const percent = total ? ((count / total) * 100).toFixed(2) : '0.00';
+			console.log(`${hostname}: ${count} (${percent}%)`);
+		}
+		console.log(`Total repositories processed: ${total}`);
+	} catch (error) {
+		console.error('Unable to read hostname counts:', error);
+		process.exit(1);
+	}
+}
+
+await main();

experiments/verify-validate/src/unique.ts

@@ -0,0 +1,57 @@
+/**
+ * Deeply compares two values for equality
+ */
+function deepEqual(a: unknown, b: unknown): boolean {
+	if (a === b) return true;
+
+	if (a === null || b === null) {
+		return a === b;
+	}
+
+	if (typeof a !== typeof b) return false;
+
+	if (typeof a !== 'object') return false;
+
+	if (Array.isArray(a) && Array.isArray(b)) {
+		if (a.length !== b.length) return false;
+		for (let i = 0; i < a.length; i++) {
+			if (!deepEqual(a[i], b[i])) return false;
+		}
+		return true;
+	}
+
+	if (Array.isArray(a) || Array.isArray(b)) return false;
+
+	// Both are objects at this point
+	const objA = a as Record<string, unknown>;
+	const objB = b as Record<string, unknown>;
+
+	const keysA = Object.keys(objA);
+	const keysB = Object.keys(objB);
+
+	if (keysA.length !== keysB.length) return false;
+
+	for (const key of keysA) {
+		if (!keysB.includes(key)) return false;
+		if (!deepEqual(objA[key], objB[key])) return false;
+	}
+
+	return true;
+}
+
+/**
+ * Mutates an array to remove deeply duplicate items
+ * @param arr - The array to mutate
+ * @returns The mutated array (same reference)
+ */
+export function uniqueDeep<T>(arr: T[]): T[] {
+	for (let i = 0; i < arr.length; i++) {
+		for (let j = i + 1; j < arr.length; j++) {
+			if (deepEqual(arr[i], arr[j])) {
+				arr.splice(j, 1);
+				j--;
+			}
+		}
+	}
+	return arr;
+}

package.json

@@ -4,7 +4,7 @@
 	"version": "0.0.0",
 	"private": true,
 	"scripts": {
-		"format": "prettier --write .",
+		"fix": "prettier --write . && oxlint --type-aware --fix",
 		"lint": "prettier --check . && oxlint --type-aware",
 		"check": "pnpm -r run --if-present check",
 		"build": "pnpm -r run --if-present build",
@@ -13,7 +13,7 @@
 	},
 	"devDependencies": {
 		"@e18e/eslint-plugin": "^0.1.4",
-		"@rlly/pedantic": "^0.1.2",
+		"@rlly/pedantic": "^0.1.3",
 		"@types/node": "catalog:",
 		"@vitest/coverage-v8": "^4.0.18",
 		"@vitest/ui": "4.0.18",

packages/consumer/src/pkg/package.ts

@@ -11,15 +11,15 @@ export async function processPackage(packument: Packument, revId: string) {
 			id: generateId('pkg'),
 			revId: packument._rev ?? revId,
 			name: packument.name,
-			distTags: packument['dist-tags'],
+			distTags: packument['dist-tags'] ?? {},
 			createdAt: packument.time.created,
 			npmUpdatedAt: packument.time.modified,
 		})
 		.onConflictDoUpdate({
 			target: [packageTable.name],
 			set: {
 				revId: packument._rev ?? revId,
-				distTags: packument['dist-tags'],
+				distTags: packument['dist-tags'] ?? {},
 				npmUpdatedAt: packument.time.modified,
 				updatedAt: new Date(),
 			},