allow #-tag urls to be treated as distinct URLs for scope types other than custom:

- fix --allowHashUrls option being ignored
- if --allowHashUrls global or --allowHash per seed is set, don't reset to false if scopeType != 'custom'
- tests: update scope tests to ensure --allowHash and --allowHashUrls now work as expected

fixes #1023

Author: ikreymer

src/util/argParser.ts

@@ -172,6 +172,7 @@ class ArgParser {
         allowHashUrls: {
           describe:
             "Allow Hashtag URLs, useful for single-page-application crawling or when different hashtags load dynamic content",
+          type: "boolean",
         },
 
         selectLinks: {

src/util/seeds.ts

@@ -6,7 +6,7 @@ import { logger } from "./logger.js";
 import { type CrawlerArgs } from "./argParser.js";
 import { normalizeUrl } from "./normalize.js";
 
-type ScopeType =
+export type ScopeType =
   | "prefix"
   | "host"
   | "domain"
@@ -15,6 +15,18 @@ type ScopeType =
   | "any"
   | "custom";
 
+export type ScopeSeedInitOpts = {
+  url: string;
+  scopeType: ScopeType | undefined;
+  include: string[];
+  exclude: string[];
+  allowHash?: boolean;
+  depth?: number;
+  sitemap?: string | boolean | null;
+  extraHops?: number;
+  auth?: string | null;
+};
+
 export class ScopedSeed {
   url: string;
   normUrl: string;
@@ -43,17 +55,7 @@ export class ScopedSeed {
     sitemap = false,
     extraHops = 0,
     auth = null,
-  }: {
-    url: string;
-    scopeType: ScopeType | undefined;
-    include: string[];
-    exclude: string[];
-    allowHash?: boolean;
-    depth?: number;
-    sitemap?: string | boolean | null;
-    extraHops?: number;
-    auth?: string | null;
-  }) {
+  }: ScopeSeedInitOpts) {
     const parsedUrl = this.parseUrl(url);
     if (!parsedUrl) {
       throw new Error("Invalid URL");
@@ -84,7 +86,7 @@ export class ScopedSeed {
         parsedUrl,
       );
       this.include = [...includeNew, ...this.include];
-      allowHash = allowHashNew;
+      allowHash ||= allowHashNew;
     }
 
     // for page scope, the depth is set to extraHops, as no other
@@ -348,13 +350,14 @@ export async function parseSeeds(
     }
   }
 
-  const scopeOpts = {
+  const scopeOpts: Omit<ScopeSeedInitOpts, "url"> = {
     scopeType: params.scopeType as ScopeType | undefined,
     sitemap: params.sitemap,
     include: params.include,
     exclude: params.exclude,
     depth: params.depth,
     extraHops: params.extraHops,
+    allowHash: params.allowHashUrls,
   };
 
   for (const seed of seeds) {

tests/scopes.test.ts

@@ -34,6 +34,7 @@ seeds:
   expect(seeds[0].scopeType).toEqual("prefix");
   expect(seeds[0].include).toEqual([/^https?:\/\/example\.com\//]);
   expect(seeds[0].exclude).toEqual([]);
+  expect(seeds[0].allowHash).toEqual(false);
 });
 
 test("default scope + exclude", async () => {
@@ -405,3 +406,27 @@ scopeType: page
   expect(result).not.toBe(false);
   expect((result as Exclude<typeof result, false>).isOOS).toBe(false);
 });
+
+test("allowHashUrls global with scopeType prefix", async () => {
+  const seeds = await getSeeds(`
+allowHashUrls: true
+seeds:
+  - url: https://example.com/
+`);
+
+  expect(seeds[0].scopeType).toEqual("prefix");
+  expect(seeds[0].allowHash).toEqual(true);
+});
+
+test("allowHash with scopeType prefix", async () => {
+  const seeds = await getSeeds(`
+seeds:
+  - url: https://example.com/
+    allowHash: true
+
+scopeType: prefix
+`);
+
+  expect(seeds[0].scopeType).toEqual("prefix");
+  expect(seeds[0].allowHash).toEqual(true);
+});