additional failure logic:

- logger.fatal() also sets crawl status to 'failed'
- add 'failOnFailedLimit' to set crawl status to 'failed' if number of failed pages exceeds limit, refactored from #393

crawler.js

@@ -329,15 +329,9 @@ export class Crawler {
       }
 
     } finally {
-      logger.info(`Final crawl status: ${status}`);
+      logger.info(`Crawl status: ${status}`);
 
-      if (this.crawlState) {
-        await this.crawlState.setStatus(status);
-      }
-
-      await this.closeLog();
-
-      await this.setEndTimeAndExit(exitCode);
+      await this.setEndTimeAndExit(exitCode, status);
     }
   }
 
@@ -698,6 +692,13 @@ self.__bx_behaviors.selectMainBehavior();
       }
     }
 
+    if (this.params.failOnFailedLimit) {
+      const numFailed = this.crawlState.numFailed();
+      if (numFailed >= this.params.failOnFailedLimit) {
+        logger.fatal(`Failed threshold reached ${numFailed} >= ${this.params.failedLimit}, failing crawl`);
+      }
+    }
+
     if (interrupt) {
       this.uploadAndDeleteLocal = true;
       this.gracefulFinishOnInterrupt();
@@ -712,18 +713,26 @@ self.__bx_behaviors.selectMainBehavior();
     }
   }
 
-  async setEndTimeAndExit(exitCode = 0) {
+  async setEndTimeAndExit(exitCode, status) {
+    await this.closeLog();
+
     if (this.crawlState) {
+      if (status) {
+        await this.crawlState.setStatus(status);
+      }
       await this.crawlState.setEndTime();
     }
     process.exit(exitCode);
   }
 
   async serializeAndExit() {
     await this.serializeConfig();
-    await this.closeLog();
 
-    await this.setEndTimeAndExit(this.interrupted ? 13 : 0);
+    if (this.interrupted) {
+      await this.setEndTimeAndExit(13, "interrupted");
+    } else {
+      await this.setEndTimeAndExit(0, "done");
+    }
   }
 
   async isCrawlRunning() {
@@ -948,10 +957,8 @@ self.__bx_behaviors.selectMainBehavior();
         if ((await this.crawlState.numDone()) > 0) {
           return;
         }
-        // stopped and no done pages, mark crawl as failed
-        await this.crawlState.setStatus("failed");
       }
-      // fail for now, may restart to try again
+      // fail crawl otherwise
       logger.fatal("No WARC Files, assuming crawl failed");
     }
 

util/argParser.js

@@ -398,6 +398,12 @@ class ArgParser {
         default: false
       },
 
+      "failOnFailedLimit": {
+        describe: "If set, save state and exit if number of failed pages exceeds this value",
+        type: "number",
+        default: 0,
+      },
+
       "customBehaviors": {
         describe: "injects a custom behavior file or set of behavior files in a directory",
         type: ["string"]

util/logger.js

@@ -104,8 +104,13 @@ class Logger
   fatal(message, data={}, context="general", exitCode=17) {
     this.logAsJSON(`${message}. Quitting`, data, context, "fatal");
 
+    async function markFailedAndEnd(crawlState) {
+      await crawlState.setStatus("failed");
+      await crawlState.setEndTime();
+    }
+
     if (this.crawlState) {
-      this.crawlState.setEndTime().finally(process.exit(exitCode));
+      markFailedAndEnd(this.crawlState).finally(process.exit(exitCode));
     } else {
       process.exit(exitCode);
     }