revisit fixes: (#153)

- when using headers from revisit record, ensure content-length is set to original payload content-length, if any
cacheing:
- don't cache payloads >25MB
- possible fix for webrecorder/replayweb.page#260

package.json

@@ -27,7 +27,7 @@
     "path-parser": "^6.1.0",
     "process": "^0.11.10",
     "stream-browserify": "^3.0.0",
-    "warcio": "^2.2.0"
+    "warcio": "^2.2.1"
   },
   "devDependencies": {
     "ava": "^5.3.1",

src/remotearchivedb.js

@@ -4,6 +4,8 @@ import { BaseAsyncIterReader, AsyncIterReader, LimitReader, concatChunks } from
 
 import { createLoader } from "./blockloaders.js";
 
+const MAX_CACHE_SIZE = 25_000_000;
+
 
 // ===========================================================================
 class OnDemandPayloadArchiveDB extends ArchiveDB
@@ -106,7 +108,16 @@ class OnDemandPayloadArchiveDB extends ArchiveDB
       }
 
       // if revisit record has header, use those, otherwise use headers from original
-      cdx.respHeaders = remote.respHeaders ? remote.respHeaders : origResult.respHeaders;
+      if (remote.respHeaders) {
+        cdx.respHeaders = remote.respHeaders;
+        if (origResult.respHeaders["content-length"]) {
+          // ensure content-length is the original result content length always
+          cdx.respHeaders["content-length"] = origResult.respHeaders["content-length"];
+        }
+      } else {
+        cdx.respHeaders = origResult.respHeaders;
+      }
+
       cdx.mime = origResult.mime;
       // ensure digest is set to original (usually same but may have different prefix)
       // to ensure proper lookup from cache
@@ -138,10 +149,16 @@ class OnDemandPayloadArchiveDB extends ArchiveDB
 
     const digest = remote.digest;
 
-    if (!this.noCache && remote.reader && digest) {
+    const tooBigForCache = cdx.source.length >= MAX_CACHE_SIZE;
+
+    if (!this.noCache && !tooBigForCache && remote.reader && digest) {
       remote.reader = new PayloadBufferingReader(this, remote.reader, digest, cdx.url, this.streamMap, hasher, cdx.recordDigest, cdx.source);
     }
 
+    if (tooBigForCache) {
+      console.log("Not cacheing, too big: " + cdx.url);
+    }
+
     payload = remote.payload;
 
     if (!payload && !remote.reader) {
@@ -150,7 +167,7 @@ class OnDemandPayloadArchiveDB extends ArchiveDB
 
     // Update payload if cacheing
     try {
-      if (payload && !this.noCache) {
+      if (payload && !this.noCache && !tooBigForCache) {
         await this.commitPayload(payload, digest);
       }
     } catch(e) {
@@ -166,7 +183,7 @@ class OnDemandPayloadArchiveDB extends ArchiveDB
         cdx.extraOpts = remote.extraOpts;
       }
 
-      if (!this.noCache) {
+      if (!this.noCache && !tooBigForCache) {
         try {
           await this.db.put("resources", cdx);
         } catch (e) {

yarn.lock

@@ -2828,6 +2828,18 @@ warcio@^2.2.0:
     uuid-random "^1.3.2"
     yargs "^17.6.2"
 
+warcio@^2.2.1:
+  version "2.2.1"
+  resolved "https://registry.yarnpkg.com/warcio/-/warcio-2.2.1.tgz#3619728fde716291c9b364744c276362a94bacec"
+  integrity sha512-KPLoz3aFtdTjexG+QQaubMyuLiNANzvcadGMyNKdpcmhl0k6lBHQQVpxZw3Hx9+4pbyqDXyiF4cr/h2tS8kvcw==
+  dependencies:
+    base32-encode "^2.0.0"
+    hash-wasm "^4.9.0"
+    pako "^1.0.11"
+    tempy "^3.1.0"
+    uuid-random "^1.3.2"
+    yargs "^17.6.2"
+
 watchpack@^2.4.0:
   version "2.4.0"
   resolved "https://registry.yarnpkg.com/watchpack/-/watchpack-2.4.0.tgz#fa33032374962c78113f93c7f2fb4c54c9862a5d"