add reference to external WACZ per revisit record (#1009)

- store in `WARC-Refers-To-Container` with file://<WACZ filename> as per
discussions in iipc/warc-specifications#111
- wabac.js 2.26.0 will use this header for prioritizing the specified
WACZ for looking up the original.
- also clears the per-WACZ dependency key `...:duperef` after current
WACZ is finished, so future WACZ files don't use stale dependencies
- fixes #1008
- version: bump to 1.12.4

Author: ikreymer

package.json

@@ -1,6 +1,6 @@
 {
   "name": "browsertrix-crawler",
-  "version": "1.12.3",
+  "version": "1.12.4",
   "main": "browsertrix-crawler",
   "type": "module",
   "repository": "https://github.com/webrecorder/browsertrix-crawler",

src/util/constants.ts

@@ -115,3 +115,5 @@ export type CrawlStatus =
   | "interrupted"
   | "failed"
   | "canceled";
+
+export const WARC_REFERS_TO_CONTAINER = "WARC-Refers-To-Container";

src/util/recorder.ts

@@ -27,7 +27,7 @@ import { Crawler } from "../crawler.js";
 import { getProxyDispatcher } from "./proxy.js";
 import { ScopedSeed } from "./seeds.js";
 import EventEmitter from "events";
-import { DEFAULT_MAX_RETRIES } from "./constants.js";
+import { DEFAULT_MAX_RETRIES, WARC_REFERS_TO_CONTAINER } from "./constants.js";
 import { Readable } from "stream";
 
 const MAX_BROWSER_DEFAULT_FETCH_SIZE = 5_000_000;
@@ -1762,6 +1762,17 @@ export class Recorder extends EventEmitter {
         const { origUrl, origDate, crawlId, index, size } = res;
         origRecSize = size;
         const date = tsToDate(origDate).toISOString();
+
+        let externalWACZ = "";
+
+        // is external crawl
+        if (this.crawlState.isExternalCrawl(crawlId)) {
+          externalWACZ = await this.crawlState.lookupWACZFilename(
+            crawlId,
+            Number(index),
+          );
+        }
+
         // always write revisit here
         // duplicate URLs in same crawl filtered out separately
         serializer.externalBuffer?.purge();
@@ -1770,6 +1781,7 @@ export class Recorder extends EventEmitter {
           serializer,
           origUrl,
           date,
+          externalWACZ,
         ));
         await this.crawlState.addDupeCrawlDependency(crawlId, index);
       } else {
@@ -2200,6 +2212,7 @@ async function createRevisitForResponse(
   serializer: WARCSerializer,
   refersToUrl: string,
   refersToDate: string,
+  externalWACZ: string,
 ) {
   const payloadDigestForRevisit = responseRecord.warcPayloadDigest || "";
 
@@ -2213,6 +2226,10 @@ async function createRevisitForResponse(
     }
   }
 
+  if (externalWACZ) {
+    warcHeaders[WARC_REFERS_TO_CONTAINER] = `file://${externalWACZ}`;
+  }
+
   const revisitRecord = WARCRecord.create({
     url: responseRecord.warcTargetURI!,
     date: responseRecord.warcDate!,
@@ -2222,6 +2239,7 @@ async function createRevisitForResponse(
     refersToUrl,
     refersToDate,
   });
+
   revisitRecord.httpHeaders = responseRecord.httpHeaders;
 
   serializer = new WARCSerializer(revisitRecord, {

src/util/state.ts

@@ -276,6 +276,24 @@ export class RedisDedupeIndex {
     await pipe.exec();
   }
 
+  // LOOKUP WACZ FILENAME
+
+  async lookupWACZFilename(crawlId: string, index: number): Promise<string> {
+    try {
+      const waczdata = await this.dedupeRedis.lindex(
+        `c:${crawlId}:wacz`,
+        index,
+      );
+      if (!waczdata) {
+        return "";
+      }
+      const { filename } = JSON.parse(waczdata);
+      return filename;
+    } catch (_) {
+      return "";
+    }
+  }
+
   // COMMIT DEDUPE TO SHARED INDEX
 
   async commitDedupeDone(crawlId?: string, uncommitted_key = DUPE_UNCOMMITTED) {
@@ -313,12 +331,11 @@ export class RedisDedupeIndex {
     const numWacz = await this.dedupeRedis.llen(`c:${crawlId}:wacz`);
 
     for (let i = 0; i < numWacz; i++) {
-      const waczdata = await this.dedupeRedis.lindex(`c:${crawlId}:wacz`, i);
-      if (!waczdata) {
+      const filename = await this.lookupWACZFilename(crawlId, i);
+      if (!filename) {
         continue;
       }
       try {
-        const { filename } = JSON.parse(waczdata);
         await this.dedupeRedis.sadd(this.sourceDone, filename);
       } catch (e) {
         // ignore
@@ -1126,6 +1143,8 @@ return inx;
   async clearWACZFilename(): Promise<void> {
     await this.redis.hdel(`${this.crawlId}:nextWacz`, this.uid);
     this.waczFilename = null;
+
+    await this.redis.del(`${this.uid}:duperef`);
   }
 
   async setArchiveSize(size: number) {

tests/dedupe-basic.test.js

@@ -189,6 +189,7 @@ test("check revisit records written on duplicate crawl, different collections, w
 
     if (record.warcType === "revisit") {
       revisit++;
+      expect(record.warcHeader("WARC-Refers-To-Container")).toBe("file://dedupe-test-orig.wacz");
     }
   }
 
@@ -226,6 +227,7 @@ test("verify new crawl against imported dupe index has same dupes as dedupe agai
 
     if (record.warcType === "revisit") {
       revisit++;
+      expect(record.warcHeader("WARC-Refers-To-Container")).toBe("file://dedupe-test-orig.wacz");
     }
   }