Add support for TOML configuration file. Closes #66.

Replace tomlkit dependency by tomli.
Other projects are switching too: nedbat/coveragepy#1180

changelog.rst

@@ -7,12 +7,14 @@ Changelog
 
 .. note:: This version is not yet released and is under active development.
 
+* [mpm] Add support for TOML configuration file. Closes #66.
 * [mpm] Add ``-C`` / ``--config`` option to point to specific configuration file.
 * [mpm] Upgrade to Click 8.x.
 * [mpm] Add support for ``psql_unicode`` and ``minimal`` table format.
 * [mpm] Set default table format to ``psql_unicode`` instead of ``fancy_grid`` to
   reduce visual noise.
 * [mpm] Let Click produce default values in help screen.
+* [mpm] Replace ``tomlkit`` dependency by ``tomli``.
 
 
 `4.1.0 (2021-05-01) <https://github.com/kdeldycke/meta-package-manager/compare/v4.0.0...v4.1.0>`_

docs/conf.py

@@ -1,12 +1,11 @@
-import io
 import os
+from pathlib import Path
 
-import tomlkit
+import tomli
 
 # Fetch general information about the project from pyproject.toml.
-toml_path = os.path.join(os.path.dirname(__file__), "..", "pyproject.toml")
-with io.open(toml_path, "r") as toml_file:
-    toml_config = tomlkit.loads(toml_file.read())
+toml_path = Path(__file__).parent.joinpath("../pyproject.toml").resolve()
+toml_config = tomli.loads(toml_path.read_text())
 
 # Redistribute pyproject.toml config to Sphinx.
 project_id = toml_config["tool"]["poetry"]["name"]

docs/configuration.rst

@@ -11,9 +11,7 @@ Here is a sample:
 
     [mpm]
     verbosity = "DEBUG"
-    manager = [
-      "brew",
-      "cask",
-    ]
+    manager = ["brew", "cask"]
 
-    [backup]
+    [mpm.search]
+    exact = True

meta_package_manager/cli.py

@@ -18,7 +18,6 @@
 import functools
 import logging
 import re
-import sys
 from datetime import datetime
 from functools import partial
 from io import TextIOWrapper
@@ -29,7 +28,7 @@
 
 import click
 import click_log
-import tomlkit
+import tomli
 from boltons.cacheutils import LRI, cached
 from boltons.strutils import complement_int_list, int_ranges_from_int_list, strip_ansi
 from cli_helpers.tabular_output import TabularOutputFormatter
@@ -39,7 +38,7 @@
 
 from . import CLI_NAME, __version__, env_data, logger
 from .base import CLI_FORMATS, CLIError, PackageManager
-from .config import DEFAULT_CONFIG_FILE, ConfigurationFileError, read_config
+from .config import DEFAULT_CONFIG_FILE, ConfigurationFileError, read_conf
 from .managers import pool
 from .platform import CURRENT_OS_ID, WINDOWS, os_label
 from .version import TokenizedString
@@ -195,18 +194,27 @@ def __exit__(self, *args, **kwargs):
 
 def load_config(ctx, param, config_file):
     # Fetch option from configuration file.
+    conf = {}
     try:
-        conf = read_config(config_file)
+        conf = read_conf(config_file)
     except ConfigurationFileError as excpt:
         # Exit the CLI if the user-provided config file is bad.
         if ctx.get_parameter_source("config") != ParameterSource.DEFAULT:
             logger.fatal(excpt)
-            sys.exit()
+            ctx.exit()
         else:
-            # TODO: merge CLI parameters and config file here.
-            # See: https://github.com/kdeldycke/meta-package-manager/issues/66
             logger.debug(excpt)
             logger.debug("Ignore configuration file.")
+
+    # Merge user config to the context default_map. See:
+    # https://click.palletsprojects.com/en/8.0.x/commands/#context-defaults
+    # This allow user's config to only overrides defaults. Values sets from direct
+    # command line calls, environment variables or interactive prompts takes precedence
+    # over any parameters from the config file.
+    if ctx.default_map is None:
+        ctx.default_map = dict()
+    ctx.default_map.update(conf.get(CLI_NAME, {}))
+
     return config_file
 
 
@@ -275,6 +283,9 @@ def load_config(ctx, param, config_file):
     metavar="CONFIG_PATH",
     type=click.Path(path_type=Path, resolve_path=True),
     default=DEFAULT_CONFIG_FILE,
+    # Force eagerness so the config option's callback gets the oportunity to set the
+    # default_map values before the other options use them.
+    is_eager=True,
     callback=load_config,
     help="Location of the configuration file.",
 )
@@ -778,11 +789,9 @@ def backup(ctx, toml_output):
             logger.error("Target file is not a TOML file.")
             return
 
-    # Initialize the TOML structure.
-    doc = tomlkit.document()
     # Leave some metadata as comment.
-    doc.add(tomlkit.comment(f"Generated by {CLI_NAME} {__version__}."))
-    doc.add(tomlkit.comment("Timestamp: {}.".format(datetime.now().isoformat())))
+    doc = f"# Generated by {CLI_NAME} {__version__}.\n"
+    doc += "# Timestamp: {}.\n".format(datetime.now().isoformat())
 
     installed_data = {}
 
@@ -796,17 +805,19 @@ def backup(ctx, toml_output):
             "packages": manager.installed.values(),
         }
 
-        manager_section = tomlkit.table()
-        pkg_data = sorted(
-            [(p["id"], p["installed_version"]) for p in manager.installed.values()]
+        pkg_data = dict(
+            **sorted(
+                # Version specifier is inspired by Poetry.
+                [
+                    (p["id"], "^" + p["installed_version"])
+                    for p in manager.installed.values()
+                ]
+            )
         )
-        for package_id, package_version in pkg_data:
-            # Version specifier is inspired by Poetry.
-            manager_section.add(package_id, f"^{package_version}")
         if pkg_data:
-            doc.add(manager.id, manager_section)
+            doc += "\n" + tomli.dumps({manager.id: pkg_data})
 
-    toml_output.write(tomlkit.dumps(doc, sort_keys=True))
+    toml_output.write_text(doc)
 
     if stats:
         print_stats(installed_data)
@@ -831,7 +842,7 @@ def restore(ctx, toml_files):
         )
         logger.info(f"Load package list from {toml_filepath}")
 
-        doc = tomlkit.parse(toml_input.read())
+        doc = tomli.loads(toml_input.read())
 
         # List unrecognized sections.
         ignored_sections = [f"[{section}]" for section in doc if section not in pool()]

meta_package_manager/config.py

@@ -20,7 +20,8 @@
 from pathlib import Path
 
 import click
-import tomlkit
+import tomli
+from boltons.iterutils import remap
 
 from . import CLI_NAME, logger
 
@@ -45,10 +46,12 @@ class ConfigurationFileError(Exception):
     pass
 
 
-def config_structure():
+def conf_structure():
     """Returns the supported configuration structure.
 
     Derives TOML structure from CLI definition.
+
+    Sections are dicts. All options have their defaults value to None.
     """
     # Imported here to avoid circular imports.
     from .cli import cli
@@ -63,61 +66,66 @@ def config_structure():
 
     # Global, top-level options shared by all subcommands are placed under the
     # cli name's section.
-    config = {CLI_NAME: {p.name for p in cli.params if p.name not in ignored_options}}
+    conf = {
+        CLI_NAME: {p.name: None for p in cli.params if p.name not in ignored_options}
+    }
 
     # Subcommand-specific options.
     for cmd_id, cmd in cli.commands.items():
-        cmd_options = {p.name for p in cmd.params if p.name not in ignored_options}
+        cmd_options = {
+            p.name: None for p in cmd.params if p.name not in ignored_options
+        }
         if cmd_options:
-            config[cmd_id] = cmd_options
+            conf[CLI_NAME][cmd_id] = cmd_options
 
-    return config
+    return conf
 
 
-def read_config(cfg_filepath):
-    """Loads a configuration files and returns recognized options and their values.
+def read_conf(conf_filepath):
+    """Loads a configuration files and only returns recognized options and their values.
 
-    Invalid parameters are ignored and log messages are emitted.
+    Invalid parameters are ignored.
     """
-    # The recognized configuration extracted from the file.
-    valid_config = {}
-
-    # Check config file.
-    if not cfg_filepath.exists():
-        raise ConfigurationFileError(f"Configuration not found at {cfg_filepath}")
-    if not cfg_filepath.is_file():
-        raise ConfigurationFileError(f"Configuration {cfg_filepath} is not a file.")
+    # Check conf file.
+    if not conf_filepath.exists():
+        raise ConfigurationFileError(f"Configuration not found at {conf_filepath}")
+    if not conf_filepath.is_file():
+        raise ConfigurationFileError(f"Configuration {conf_filepath} is not a file.")
 
     # Parse TOML content.
-    logger.info(f"Load configuration from {cfg_filepath}")
-    doc = tomlkit.parse(cfg_filepath.read_bytes())
+    logger.info(f"Load configuration from {conf_filepath}")
+    user_conf = tomli.loads(conf_filepath.read_text())
+
+    # Merge configuration file's content into the canonical reference structure, but
+    # ignore all unrecognized options.
 
-    # Filters-out configuration file's content against the reference structure and
-    # only keep recognized options.
-    structure = config_structure()
+    def recursive_update(a, b):
+        """Like standard ``dict.update()``, but recursive so sub-dict gets updated.
 
-    for section, options in doc.items():
+        Ignore elements present in ``b`` but not in ``a``.
+        """
+        for k, v in b.items():
+            if isinstance(v, dict) and isinstance(a.get(k), dict):
+                a[k] = recursive_update(a[k], v)
+            # Ignore elements unregistered in the canonical structure.
+            elif k in a:
+                a[k] = b[k]
+        return a
 
-        # Ignore unrecognized section.
-        if section not in structure:
-            logger.warning(f"Ignore [{section}] section.")
-            continue
+    valid_conf = recursive_update(conf_structure(), user_conf)
 
-        # Dive into section and look at options.
-        valid_options = {}
-        for opt_id, opt_value in options.items():
+    # Clean-up blank values left-over by the canonical reference structure.
 
-            # Ignore unrecognized option.
-            if opt_id not in structure[section]:
-                logger.warning(f"Ignore [{section}].{opt_id} option.")
-                continue
+    def visit(path, key, value):
+        """Skip None values and empty dicts."""
+        if value is None:
+            return False
+        if isinstance(value, dict) and not len(value):
+            return False
+        return True
 
-            # Keep option.
-            valid_options[opt_id] = opt_value
+    clean_conf = remap(valid_conf, visit=visit)
 
-        if valid_options:
-            valid_config[section] = valid_options
-        else:
-            logger.warning(f"Ignore empty [{section}] section.")
+    logger.debug(f"Configuration loaded: {clean_conf}")
 
-    return valid_config
+    return clean_conf

meta_package_manager/tests/test_cli.py

@@ -49,13 +49,18 @@ def test_temporary_fs(runner):
 DUMMY_CONFIG_FILE = """
         # Comment
 
+        top_level_param = "to_ignore"
+
         [mpm]
         verbosity = "DEBUG"
         blahblah = 234
+        manager = ["pip", "npm", "gem"]
 
         [garbage]
 
-        [backup]
+        [mpm.search]
+        exact = true
+        dummy_parameter = 3
         """
 
 
@@ -130,8 +135,6 @@ def test_read_default_conf_file(self, invoke):
         result = invoke("managers")
         assert result.exit_code == 0
         assert f"Load configuration from {DEFAULT_CONFIG_FILE}" in result.stderr
-        assert "warning: Ignore [mpm].blahblah option." in result.stderr
-        assert "warning: Ignore [garbage] section." in result.stderr
 
     def test_read_specific_conf_file(self, invoke, tmp_path):
         specific_conf = tmp_path / "configuration.extension"
@@ -143,6 +146,33 @@ def test_read_specific_conf_file(self, invoke, tmp_path):
         assert "warning: Ignore [mpm].blahblah option." in result.stderr
         assert "warning: Ignore [garbage] section." in result.stderr
 
+    def test_conf_file_overrides_defaults(self, invoke):
+        create_toml(DEFAULT_CONFIG_FILE, DUMMY_CONFIG_FILE)
+        result = invoke("managers")
+        assert result.exit_code == 0
+        assert logger.level == getattr(logging, "DEBUG")
+        assert " │ pip │ " in result.stdout
+        assert " │ npm │ " in result.stdout
+        assert " │ gem │ " in result.stdout
+        assert "brew" not in result.stdout
+        assert "cask" not in result.stdout
+        assert "debug: " in result.stderr
+
+    def test_conf_file_cli_override(self, invoke):
+        create_toml(DEFAULT_CONFIG_FILE, DUMMY_CONFIG_FILE)
+        result = invoke("--verbosity", "CRITICAL", "managers")
+        assert result.exit_code == 0
+        assert logger.level == getattr(logging, "CRITICAL")
+        assert " │ pip │ " in result.stdout
+        assert " │ npm │ " in result.stdout
+        assert " │ gem │ " in result.stdout
+        assert "brew" not in result.stdout
+        assert "cask" not in result.stdout
+        assert "error: " not in result.stderr
+        assert "warning: " not in result.stderr
+        assert "info: " not in result.stderr
+        assert "debug: " not in result.stderr
+
     @unless_windows
     @pytest.mark.parametrize("mode", WINDOWS_MODE_BLACKLIST)
     def test_check_failing_unicode_rendering(self, mode):

pyproject.toml

@@ -46,7 +46,7 @@ click-log = "^0.3.2"
 click-help-colors = "^0.9"
 cli-helpers = "^2.2.0"
 simplejson = "^3.17.5"
-tomlkit = "^0.7.2"
+tomli = "^1.2.1"
 # Should have been put in dev-dependencies but extras only works from main
 # section.
 sphinx = {version = "^4.2.0", optional = true}