Merge pull request #406 from internetarchive/run-job-option

Fix and document the -r option which runs a named job on startup
internetarchive · Jun 25, 2021 · c263070 · c263070
2 parents 6835d09 + a38fda2
commit c263070
Showing 1 changed file with 14 additions and 1 deletion.
diff --git a/engine/src/main/java/org/archive/crawler/Heritrix.java b/engine/src/main/java/org/archive/crawler/Heritrix.java
@@ -46,6 +46,7 @@
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.output.TeeOutputStream;
 import org.apache.commons.lang.StringUtils;
+import org.archive.crawler.framework.CrawlJob;
 import org.archive.crawler.framework.Engine;
 import org.archive.crawler.restlet.EngineApplication;
 import org.archive.crawler.restlet.RateLimitGuard;
@@ -140,6 +141,8 @@ private static Options options() {
                 "web interface to bind to.");
         options.addOption("p", "web-port", true, "The port the web interface " +
                 "should listen on.");
+        options.addOption("r", "run-job", true, "Run a single job and then exit when it" +
+                "finishes.");
         options.addOption("s", "ssl-params", true,  "Specify a keystore " +
                 "path, keystore password, and key password for HTTPS use. " +
                 "Separate with commas, no whitespace.");
@@ -367,7 +370,17 @@ public void instanceMain(String[] args)
 "interface will be internet-accessible.");
             }
             if (cl.hasOption('r')) {
-                engine.requestLaunch(cl.getOptionValue('r'));
+                String jobName = cl.getOptionValue('r');
+                engine.requestLaunch(jobName);
+                CrawlJob job = engine.getJob(jobName);
+                if (job == null || job.getCrawlController() == null) {
+                    System.err.println("Failed to launch job: " + jobName);
+                    System.exit(1);
+                }
+                job.getCrawlController().requestCrawlResume();
+                engine.waitForNoRunningJobs(0);
+                engine.shutdown();
+                System.exit(0);
             } 
         } catch (Exception e) {
             // Show any exceptions in STARTLOG.