diff --git a/changelog.rst b/changelog.rst index 0b08e03a1..7d9318ce2 100644 --- a/changelog.rst +++ b/changelog.rst @@ -7,12 +7,14 @@ Changelog .. note:: This version is not yet released and is under active development. +* [mpm] Add support for TOML configuration file. Closes #66. * [mpm] Add ``-C`` / ``--config`` option to point to specific configuration file. * [mpm] Upgrade to Click 8.x. * [mpm] Add support for ``psql_unicode`` and ``minimal`` table format. * [mpm] Set default table format to ``psql_unicode`` instead of ``fancy_grid`` to reduce visual noise. * [mpm] Let Click produce default values in help screen. +* [mpm] Replace ``tomlkit`` dependency by ``tomli``. `4.1.0 (2021-05-01) `_ diff --git a/docs/conf.py b/docs/conf.py index 488e68aff..783c974d9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,12 +1,11 @@ -import io import os +from pathlib import Path -import tomlkit +import tomli # Fetch general information about the project from pyproject.toml. -toml_path = os.path.join(os.path.dirname(__file__), "..", "pyproject.toml") -with io.open(toml_path, "r") as toml_file: - toml_config = tomlkit.loads(toml_file.read()) +toml_path = Path(__file__).parent.joinpath("../pyproject.toml").resolve() +toml_config = tomli.loads(toml_path.read_text()) # Redistribute pyproject.toml config to Sphinx. project_id = toml_config["tool"]["poetry"]["name"] diff --git a/docs/configuration.rst b/docs/configuration.rst index 407c2fc6c..3a1b70010 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -11,9 +11,7 @@ Here is a sample: [mpm] verbosity = "DEBUG" - manager = [ - "brew", - "cask", - ] + manager = ["brew", "cask"] - [backup] \ No newline at end of file + [mpm.search] + exact = True \ No newline at end of file diff --git a/meta_package_manager/cli.py b/meta_package_manager/cli.py index 6d3a144f8..34c1fe3ea 100644 --- a/meta_package_manager/cli.py +++ b/meta_package_manager/cli.py @@ -18,7 +18,6 @@ import functools import logging import re -import sys from datetime import datetime from functools import partial from io import TextIOWrapper @@ -29,7 +28,7 @@ import click import click_log -import tomlkit +import tomli from boltons.cacheutils import LRI, cached from boltons.strutils import complement_int_list, int_ranges_from_int_list, strip_ansi from cli_helpers.tabular_output import TabularOutputFormatter @@ -39,7 +38,7 @@ from . import CLI_NAME, __version__, env_data, logger from .base import CLI_FORMATS, CLIError, PackageManager -from .config import DEFAULT_CONFIG_FILE, ConfigurationFileError, read_config +from .config import DEFAULT_CONFIG_FILE, ConfigurationFileError, read_conf from .managers import pool from .platform import CURRENT_OS_ID, WINDOWS, os_label from .version import TokenizedString @@ -195,18 +194,27 @@ def __exit__(self, *args, **kwargs): def load_config(ctx, param, config_file): # Fetch option from configuration file. + conf = {} try: - conf = read_config(config_file) + conf = read_conf(config_file) except ConfigurationFileError as excpt: # Exit the CLI if the user-provided config file is bad. if ctx.get_parameter_source("config") != ParameterSource.DEFAULT: logger.fatal(excpt) - sys.exit() + ctx.exit() else: - # TODO: merge CLI parameters and config file here. - # See: https://github.com/kdeldycke/meta-package-manager/issues/66 logger.debug(excpt) logger.debug("Ignore configuration file.") + + # Merge user config to the context default_map. See: + # https://click.palletsprojects.com/en/8.0.x/commands/#context-defaults + # This allow user's config to only overrides defaults. Values sets from direct + # command line calls, environment variables or interactive prompts takes precedence + # over any parameters from the config file. + if ctx.default_map is None: + ctx.default_map = dict() + ctx.default_map.update(conf.get(CLI_NAME, {})) + return config_file @@ -275,6 +283,9 @@ def load_config(ctx, param, config_file): metavar="CONFIG_PATH", type=click.Path(path_type=Path, resolve_path=True), default=DEFAULT_CONFIG_FILE, + # Force eagerness so the config option's callback gets the oportunity to set the + # default_map values before the other options use them. + is_eager=True, callback=load_config, help="Location of the configuration file.", ) @@ -778,11 +789,9 @@ def backup(ctx, toml_output): logger.error("Target file is not a TOML file.") return - # Initialize the TOML structure. - doc = tomlkit.document() # Leave some metadata as comment. - doc.add(tomlkit.comment(f"Generated by {CLI_NAME} {__version__}.")) - doc.add(tomlkit.comment("Timestamp: {}.".format(datetime.now().isoformat()))) + doc = f"# Generated by {CLI_NAME} {__version__}.\n" + doc += "# Timestamp: {}.\n".format(datetime.now().isoformat()) installed_data = {} @@ -796,17 +805,19 @@ def backup(ctx, toml_output): "packages": manager.installed.values(), } - manager_section = tomlkit.table() - pkg_data = sorted( - [(p["id"], p["installed_version"]) for p in manager.installed.values()] + pkg_data = dict( + **sorted( + # Version specifier is inspired by Poetry. + [ + (p["id"], "^" + p["installed_version"]) + for p in manager.installed.values() + ] + ) ) - for package_id, package_version in pkg_data: - # Version specifier is inspired by Poetry. - manager_section.add(package_id, f"^{package_version}") if pkg_data: - doc.add(manager.id, manager_section) + doc += "\n" + tomli.dumps({manager.id: pkg_data}) - toml_output.write(tomlkit.dumps(doc, sort_keys=True)) + toml_output.write_text(doc) if stats: print_stats(installed_data) @@ -831,7 +842,7 @@ def restore(ctx, toml_files): ) logger.info(f"Load package list from {toml_filepath}") - doc = tomlkit.parse(toml_input.read()) + doc = tomli.loads(toml_input.read()) # List unrecognized sections. ignored_sections = [f"[{section}]" for section in doc if section not in pool()] diff --git a/meta_package_manager/config.py b/meta_package_manager/config.py index be92ffe90..2ef4e720f 100644 --- a/meta_package_manager/config.py +++ b/meta_package_manager/config.py @@ -20,7 +20,8 @@ from pathlib import Path import click -import tomlkit +import tomli +from boltons.iterutils import remap from . import CLI_NAME, logger @@ -45,10 +46,12 @@ class ConfigurationFileError(Exception): pass -def config_structure(): +def conf_structure(): """Returns the supported configuration structure. Derives TOML structure from CLI definition. + + Sections are dicts. All options have their defaults value to None. """ # Imported here to avoid circular imports. from .cli import cli @@ -63,61 +66,66 @@ def config_structure(): # Global, top-level options shared by all subcommands are placed under the # cli name's section. - config = {CLI_NAME: {p.name for p in cli.params if p.name not in ignored_options}} + conf = { + CLI_NAME: {p.name: None for p in cli.params if p.name not in ignored_options} + } # Subcommand-specific options. for cmd_id, cmd in cli.commands.items(): - cmd_options = {p.name for p in cmd.params if p.name not in ignored_options} + cmd_options = { + p.name: None for p in cmd.params if p.name not in ignored_options + } if cmd_options: - config[cmd_id] = cmd_options + conf[CLI_NAME][cmd_id] = cmd_options - return config + return conf -def read_config(cfg_filepath): - """Loads a configuration files and returns recognized options and their values. +def read_conf(conf_filepath): + """Loads a configuration files and only returns recognized options and their values. - Invalid parameters are ignored and log messages are emitted. + Invalid parameters are ignored. """ - # The recognized configuration extracted from the file. - valid_config = {} - - # Check config file. - if not cfg_filepath.exists(): - raise ConfigurationFileError(f"Configuration not found at {cfg_filepath}") - if not cfg_filepath.is_file(): - raise ConfigurationFileError(f"Configuration {cfg_filepath} is not a file.") + # Check conf file. + if not conf_filepath.exists(): + raise ConfigurationFileError(f"Configuration not found at {conf_filepath}") + if not conf_filepath.is_file(): + raise ConfigurationFileError(f"Configuration {conf_filepath} is not a file.") # Parse TOML content. - logger.info(f"Load configuration from {cfg_filepath}") - doc = tomlkit.parse(cfg_filepath.read_bytes()) + logger.info(f"Load configuration from {conf_filepath}") + user_conf = tomli.loads(conf_filepath.read_text()) + + # Merge configuration file's content into the canonical reference structure, but + # ignore all unrecognized options. - # Filters-out configuration file's content against the reference structure and - # only keep recognized options. - structure = config_structure() + def recursive_update(a, b): + """Like standard ``dict.update()``, but recursive so sub-dict gets updated. - for section, options in doc.items(): + Ignore elements present in ``b`` but not in ``a``. + """ + for k, v in b.items(): + if isinstance(v, dict) and isinstance(a.get(k), dict): + a[k] = recursive_update(a[k], v) + # Ignore elements unregistered in the canonical structure. + elif k in a: + a[k] = b[k] + return a - # Ignore unrecognized section. - if section not in structure: - logger.warning(f"Ignore [{section}] section.") - continue + valid_conf = recursive_update(conf_structure(), user_conf) - # Dive into section and look at options. - valid_options = {} - for opt_id, opt_value in options.items(): + # Clean-up blank values left-over by the canonical reference structure. - # Ignore unrecognized option. - if opt_id not in structure[section]: - logger.warning(f"Ignore [{section}].{opt_id} option.") - continue + def visit(path, key, value): + """Skip None values and empty dicts.""" + if value is None: + return False + if isinstance(value, dict) and not len(value): + return False + return True - # Keep option. - valid_options[opt_id] = opt_value + clean_conf = remap(valid_conf, visit=visit) - if valid_options: - valid_config[section] = valid_options - else: - logger.warning(f"Ignore empty [{section}] section.") + logger.debug(f"Configuration loaded: {clean_conf}") - return valid_config + return clean_conf diff --git a/meta_package_manager/tests/test_cli.py b/meta_package_manager/tests/test_cli.py index ac853b611..b6cc9d5ea 100644 --- a/meta_package_manager/tests/test_cli.py +++ b/meta_package_manager/tests/test_cli.py @@ -49,13 +49,18 @@ def test_temporary_fs(runner): DUMMY_CONFIG_FILE = """ # Comment + top_level_param = "to_ignore" + [mpm] verbosity = "DEBUG" blahblah = 234 + manager = ["pip", "npm", "gem"] [garbage] - [backup] + [mpm.search] + exact = true + dummy_parameter = 3 """ @@ -130,8 +135,6 @@ def test_read_default_conf_file(self, invoke): result = invoke("managers") assert result.exit_code == 0 assert f"Load configuration from {DEFAULT_CONFIG_FILE}" in result.stderr - assert "warning: Ignore [mpm].blahblah option." in result.stderr - assert "warning: Ignore [garbage] section." in result.stderr def test_read_specific_conf_file(self, invoke, tmp_path): specific_conf = tmp_path / "configuration.extension" @@ -143,6 +146,33 @@ def test_read_specific_conf_file(self, invoke, tmp_path): assert "warning: Ignore [mpm].blahblah option." in result.stderr assert "warning: Ignore [garbage] section." in result.stderr + def test_conf_file_overrides_defaults(self, invoke): + create_toml(DEFAULT_CONFIG_FILE, DUMMY_CONFIG_FILE) + result = invoke("managers") + assert result.exit_code == 0 + assert logger.level == getattr(logging, "DEBUG") + assert " │ pip │ " in result.stdout + assert " │ npm │ " in result.stdout + assert " │ gem │ " in result.stdout + assert "brew" not in result.stdout + assert "cask" not in result.stdout + assert "debug: " in result.stderr + + def test_conf_file_cli_override(self, invoke): + create_toml(DEFAULT_CONFIG_FILE, DUMMY_CONFIG_FILE) + result = invoke("--verbosity", "CRITICAL", "managers") + assert result.exit_code == 0 + assert logger.level == getattr(logging, "CRITICAL") + assert " │ pip │ " in result.stdout + assert " │ npm │ " in result.stdout + assert " │ gem │ " in result.stdout + assert "brew" not in result.stdout + assert "cask" not in result.stdout + assert "error: " not in result.stderr + assert "warning: " not in result.stderr + assert "info: " not in result.stderr + assert "debug: " not in result.stderr + @unless_windows @pytest.mark.parametrize("mode", WINDOWS_MODE_BLACKLIST) def test_check_failing_unicode_rendering(self, mode): diff --git a/pyproject.toml b/pyproject.toml index b65667fa9..3e3a570a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ click-log = "^0.3.2" click-help-colors = "^0.9" cli-helpers = "^2.2.0" simplejson = "^3.17.5" -tomlkit = "^0.7.2" +tomli = "^1.2.1" # Should have been put in dev-dependencies but extras only works from main # section. sphinx = {version = "^4.2.0", optional = true}