fix hashtag in seed URLs for single page / non-spa scopes: (#1013)

- remove hashtag from normalized seed URL, fixing isIncluded() check
which already removes it
- consider crawls with all URLs excluded or failed as 'failed', add
numExcluded() check to isFailed()
- fixes #1012

---------

Co-authored-by: Tessa Walsh <tessa@bitarchivist.net>

Author: ikreymer

src/util/seeds.ts

@@ -67,8 +67,6 @@ export class ScopedSeed {
 
     this.url = parsedUrl.href;
 
-    // Normalize URL with sorted query parameters for consistent matching
-    this.normUrl = normalizeUrl(parsedUrl.href);
     this.include = parseRx(include);
     this.exclude = parseRx(exclude);
 
@@ -95,6 +93,14 @@ export class ScopedSeed {
       depth = extraHops;
     }
 
+    // normalize hash out if not distinguishing between hashes
+    if (!allowHash) {
+      parsedUrl.hash = "";
+    }
+
+    // normalize URL with sorted query parameters for consistent matching
+    this.normUrl = normalizeUrl(parsedUrl.href);
+
     this.sitemap = this.resolveSiteMap(sitemap);
     this.allowHash = allowHash;
     this.maxExtraHops = extraHops;

src/util/state.ts

@@ -1041,10 +1041,14 @@ return inx;
       (await this.numDone()) === 0 &&
       (await this.queueSize()) === 0 &&
       (await this.numPending()) === 0 &&
-      (await this.numFailed()) > 0
+      ((await this.numExcluded()) > 0 || (await this.numFailed()) > 0)
     );
   }
 
+  async numExcluded() {
+    return await this.redis.scard(this.exKey);
+  }
+
   async numFound() {
     return await this.redis.numfound(this.skey, this.esKey, this.exKey);
   }

tests/scopes.test.js

@@ -383,3 +383,19 @@ seeds:
   expect(result2).not.toBe(false);
   expect(result2.isOOS).toBe(false);
 });
+
+test("scopeType page includes single pages with hashtag", async () => {
+  const seeds = await getSeeds(`
+seeds:
+  - url: https://example.com/#hashtag
+
+scopeType: page
+`);
+
+  expect(seeds[0].scopeType).toEqual("page");
+
+  // Test with self (should match)
+  const result = seeds[0].isIncluded("https://example.com/#hashtag", 0, 0);
+  expect(result).not.toBe(false);
+  expect(result.isOOS).toBe(false);
+});