feat: Add API to download likelihood patchset archives

docs/api.rst

@@ -173,3 +173,14 @@ Utilities
    validate
    options_from_eqdelimstring
    digest
+
+Contrib
+-------
+
+.. currentmodule:: pyhf.contrib
+
+.. autosummary::
+   :toctree: _generated/
+
+   viz.brazil
+   utils.download

setup.py

@@ -19,7 +19,7 @@
         + extras_require['minuit']
     )
 )
-extras_require['contrib'] = sorted(set(['matplotlib']))
+extras_require['contrib'] = sorted(set(['matplotlib', 'requests']))
 extras_require['lint'] = sorted(set(['pyflakes', 'black']))
 
 extras_require['test'] = sorted(

src/pyhf/cli/__init__.py

@@ -4,5 +4,6 @@
 from .spec import cli as spec
 from .infer import cli as infer
 from .complete import cli as complete
+from .contrib import cli as contrib
 
-__all__ = ['cli', 'rootio', 'spec', 'infer', 'complete']
+__all__ = ['cli', 'rootio', 'spec', 'infer', 'complete', 'contrib']

src/pyhf/cli/cli.py

@@ -4,7 +4,7 @@
 import click
 
 from ..version import __version__
-from . import rootio, spec, infer, patchset, complete
+from . import rootio, spec, infer, patchset, complete, contrib
 
 logging.basicConfig()
 log = logging.getLogger(__name__)
@@ -35,3 +35,5 @@ def pyhf():
 pyhf.add_command(patchset.cli)
 
 pyhf.add_command(complete.cli)
+
+pyhf.add_command(contrib.cli)

src/pyhf/cli/contrib.py

@@ -0,0 +1,71 @@
+"""CLI for functionality that will get migrated out eventually."""
+import logging
+import click
+from pathlib import Path
+
+from ..contrib import utils
+
+logging.basicConfig()
+log = logging.getLogger(__name__)
+
+
+@click.group(name="contrib")
+def cli():
+    """
+    Contrib experimental operations.
+
+    .. note::
+
+        Requires installation of the ``contrib`` extra.
+
+        .. code-block:: shell
+
+            $ python -m pip install pyhf[contrib]
+    """
+
+
+@cli.command()
+@click.argument("archive-url", default="-")
+@click.argument("output-directory", default="-")
+@click.option("-v", "--verbose", is_flag=True, help="Enables verbose mode")
+@click.option(
+    "-f", "--force", is_flag=True, help="Force download from non-approved host"
+)
+@click.option(
+    "-c",
+    "--compress",
+    is_flag=True,
+    help="Keep the archive in a compressed tar.gz form",
+)
+def download(archive_url, output_directory, verbose, force, compress):
+    """
+    Download the patchset archive from the remote URL and extract it in a
+    directory at the path given.
+
+    Example:
+
+    .. code-block:: shell
+
+        $ pyhf contrib download --verbose https://www.hepdata.net/record/resource/1408476?view=true 1Lbb-likelihoods
+
+        \b
+        1Lbb-likelihoods/patchset.json
+        1Lbb-likelihoods/README.md
+        1Lbb-likelihoods/BkgOnly.json
+
+    Raises:
+        :class:`~pyhf.exceptions.InvalidArchiveHost`: if the provided archive host name is not known to be valid
+    """
+    try:
+        utils.download(archive_url, output_directory, force, compress)
+
+        if verbose:
+            file_list = [str(file) for file in list(Path(output_directory).glob("*"))]
+            print("\n".join(file_list))
+    except AttributeError as excep:
+        exception_info = (
+            str(excep)
+            + "\nInstallation of the contrib extra is required to use the contrib CLI API"
+            + "\nPlease install with: python -m pip install pyhf[contrib]\n"
+        )
+        log.error(exception_info)

src/pyhf/contrib/utils.py

@@ -0,0 +1,66 @@
+"""Helper utilities for common tasks."""
+
+from urllib.parse import urlparse
+import tarfile
+from io import BytesIO
+import logging
+from .. import exceptions
+
+logging.basicConfig()
+log = logging.getLogger(__name__)
+
+try:
+    import requests
+
+    def download(archive_url, output_directory, force=False, compress=False):
+        """
+        Download the patchset archive from the remote URL and extract it in a
+        directory at the path given.
+
+        Example:
+
+            >>> from pyhf.contrib.utils import download
+            >>> download("https://www.hepdata.net/record/resource/1408476?view=true", "1Lbb-likelihoods")
+            >>> import os
+            >>> sorted(os.listdir("1Lbb-likelihoods"))
+            ['BkgOnly.json', 'README.md', 'patchset.json']
+            >>> download("https://www.hepdata.net/record/resource/1408476?view=true", "1Lbb-likelihoods.tar.gz", compress=True)
+            >>> import glob
+            >>> glob.glob("1Lbb-likelihoods.tar.gz")
+            ['1Lbb-likelihoods.tar.gz']
+
+        Args:
+            archive_url (`str`): The URL of the :class:`~pyhf.patchset.PatchSet` archive to download.
+            output_directory (`str`): Name of the directory to unpack the archive into.
+            force (`Bool`): Force download from non-approved host. Default is ``False``.
+            compress (`Bool`): Keep the archive in a compressed ``tar.gz`` form. Default is ``False``.
+
+        Raises:
+            :class:`~pyhf.exceptions.InvalidArchiveHost`: if the provided archive host name is not known to be valid
+        """
+        if not force:
+            valid_hosts = ["www.hepdata.net", "doi.org"]
+            netloc = urlparse(archive_url).netloc
+            if netloc not in valid_hosts:
+                raise exceptions.InvalidArchiveHost(
+                    f"{netloc} is not an approved archive host: {', '.join(str(host) for host in valid_hosts)}\n"
+                    + "To download an archive from this host use the --force option."
+                )
+
+        with requests.get(archive_url) as response:
+            if compress:
+                with open(output_directory, "wb") as archive:
+                    archive.write(response.content)
+            else:
+                with tarfile.open(
+                    mode="r|gz", fileobj=BytesIO(response.content)
+                ) as archive:
+                    archive.extractall(output_directory)
+
+
+except ModuleNotFoundError as excep:
+    log.error(
+        str(excep)
+        + "\nInstallation of the contrib extra is required to use pyhf.contrib.utils.download"
+        + "\nPlease install with: python -m pip install pyhf[contrib]\n"
+    )

src/pyhf/exceptions/__init__.py

@@ -40,6 +40,10 @@ def __init__(self, ValidationError):
         super().__init__(message)
 
 
+class InvalidArchiveHost(Exception):
+    """InvalidArchiveHost is raised when a given patchset archive url is not an approved host."""
+
+
 class InvalidPatchSet(Exception):
     """InvalidPatchSet is raised when a given patchset object does not have the right configuration, even though it validates correctly against the schema."""
 

tests/test_scripts.py

@@ -2,7 +2,13 @@
 import shlex
 import pyhf
 import time
+import sys
+import logging
 import pytest
+from click.testing import CliRunner
+from unittest import mock
+from importlib import reload
+from importlib import import_module
 
 
 def test_version(script_runner):
@@ -561,6 +567,100 @@ def test_workspace_digest(tmpdir, script_runner, algorithms, do_json):
         }
 
 
+@pytest.mark.parametrize(
+    "archive",
+    [
+        "https://www.hepdata.net/record/resource/1408476?view=true",
+        "https://doi.org/10.17182/hepdata.89408.v1/r2",
+    ],
+)
+def test_patchset_download(datadir, script_runner, archive):
+    command = f'pyhf contrib download {archive} {datadir.join("likelihoods").strpath}'
+    ret = script_runner.run(*shlex.split(command))
+    assert ret.success
+
+    # Run with all optional flags
+    command = f'pyhf contrib download --verbose --force {archive} {datadir.join("likelihoods").strpath}'
+    ret = script_runner.run(*shlex.split(command))
+    assert ret.success
+
+    command = f'pyhf contrib download --verbose https://www.fail.org/record/resource/1234567 {datadir.join("likelihoods").strpath}'
+    ret = script_runner.run(*shlex.split(command))
+    assert not ret.success
+    assert (
+        "pyhf.exceptions.InvalidArchiveHost: www.fail.org is not an approved archive host"
+        in ret.stderr
+    )
+    command = f'pyhf contrib download --verbose --force https://www.fail.org/record/resource/1234567 {datadir.join("likelihoods").strpath}'
+    ret = script_runner.run(*shlex.split(command))
+    assert not ret.success
+    # TODO: https://github.com/scikit-hep/pyhf/issues/1075
+    # Python 3.6 has different return error than 3.7, 3.8
+    assert (
+        "ssl.CertificateError: hostname 'www.fail.org' doesn't match"
+        or "certificate verify failed: Hostname mismatch, certificate is not valid for 'www.fail.org'."
+        in ret.stderr
+    )
+
+
+def test_missing_contrib_extra(caplog):
+    with mock.patch.dict(sys.modules):
+        sys.modules["requests"] = None
+        if "pyhf.contrib.utils" in sys.modules:
+            reload(sys.modules["pyhf.contrib.utils"])
+        else:
+            import_module("pyhf.cli")
+
+    with caplog.at_level(logging.ERROR):
+        for line in [
+            "import of requests halted; None in sys.modules",
+            "Installation of the contrib extra is required to use pyhf.contrib.utils.download",
+            "Please install with: python -m pip install pyhf[contrib]",
+        ]:
+            assert line in caplog.text
+        caplog.clear()
+
+
+def test_missing_contrib_download(caplog):
+    with mock.patch.dict(sys.modules):
+        sys.modules["requests"] = None
+        if "pyhf.cli" in sys.modules:
+            reload(sys.modules["pyhf.cli"])
+        else:
+            import_module("pyhf.cli")
+
+        # Force environment for runner
+        for module in [
+            "pyhf.cli.contrib",
+            "pyhf.contrib",
+            "pyhf.contrib.utils",
+        ]:
+            if module in sys.modules:
+                del sys.modules[module]
+
+        from pyhf.cli.contrib import download
+
+        runner = CliRunner(mix_stderr=False)
+        result = runner.invoke(
+            download,
+            [
+                "--verbose",
+                "https://www.hepdata.net/record/resource/1408476?view=true",
+                "1Lbb-likelihoods",
+            ],
+        )
+        assert result.exit_code == 0
+
+        with caplog.at_level(logging.ERROR):
+            for line in [
+                "module 'pyhf.contrib.utils' has no attribute 'download'",
+                "Installation of the contrib extra is required to use the contrib CLI API",
+                "Please install with: python -m pip install pyhf[contrib]",
+            ]:
+                assert line in caplog.text
+            caplog.clear()
+
+
 @pytest.mark.parametrize('output_file', [False, True])
 @pytest.mark.parametrize('with_metadata', [False, True])
 def test_patchset_extract(datadir, tmpdir, script_runner, output_file, with_metadata):