WIP: Enable *some* parallelizable sequences for Molecule

molecule/command/base.py

@@ -110,6 +110,8 @@ def execute_cmdline_scenarios(scenario_name,
                 execute_subcommand(scenario.config, 'destroy')
                 # always prune ephemeral dir if destroying on failure
                 scenario.prune()
+                if scenario.config.is_parallel_mode:
+                    scenario._remove_parallel_directory()
                 util.sysexit()
             else:
                 raise
@@ -144,6 +146,9 @@ def execute_scenario(scenario):
     if 'destroy' in scenario.sequence:
         scenario.prune()
 
+    if scenario.config.is_parallel_mode:
+        scenario._remove_parallel_directory()
+
 
 def get_configs(args, command_args, ansible_args=()):
     """

molecule/command/test.py

@@ -23,6 +23,7 @@
 from molecule import config
 from molecule import logger
 from molecule.command import base
+from molecule import util
 
 LOG = logger.get_logger(__name__)
 
@@ -71,6 +72,10 @@ class Test(base.Base):
 
         Load an env file to read variables from when rendering
         molecule.yml.
+
+    .. option:: molecule --parallel create
+
+       Run in parallelizable mode.
     """
 
     def execute(self):
@@ -106,20 +111,28 @@ def execute(self):
     default='always',
     help=('The destroy strategy used at the conclusion of a '
           'Molecule run (always).'))
-def test(ctx, scenario_name, driver_name, __all, destroy):  # pragma: no cover
+@click.option(
+    '--parallel/--no-parallel',
+    default=False,
+    help='Enable or disable parallel mode. Default is disabled.')
+def test(ctx, scenario_name, driver_name, __all, destroy,
+         parallel):  # pragma: no cover
     """
     Test (lint, cleanup, destroy, dependency, syntax, create, prepare,
           converge, idempotence, side_effect, verify, cleanup, destroy).
     """
-
     args = ctx.obj.get('args')
+
     subcommand = base._get_subcommand(__name__)
     command_args = {
         'destroy': destroy,
+        'parallel': parallel,
         'subcommand': subcommand,
         'driver_name': driver_name,
     }
 
+    util.validate_args_with_parallel(command_args)
+
     if __all:
         scenario_name = None
 

molecule/config.py

@@ -261,6 +261,10 @@ def verifier(self):
     def verifiers(self):
         return molecule_verifiers()
 
+    @property
+    def is_parallel_mode(self):
+        return self.command_args.get('parallel', False)
+
     def _get_driver_name(self):
         driver_from_state_file = self.state.driver
         driver_from_cli = self.command_args.get('driver_name')

molecule/platforms.py

@@ -19,6 +19,7 @@
 #  DEALINGS IN THE SOFTWARE.
 
 from molecule import logger
+from molecule import util
 
 LOG = logger.get_logger(__name__)
 
@@ -72,8 +73,19 @@ def __init__(self, config):
         :param config: An instance of a Molecule config.
         :return: None
         """
+        config.config['platforms'] = self._maybe_parallelize_instance_names(
+            config)
         self._config = config
 
+    def _maybe_parallelize_instance_names(self, config):
+        if not config.is_parallel_mode:
+            return config
+
+        return util.parallelize_instance_names(
+            config.config['platforms'],
+            config.scenario.parallel_run_uuid,
+        )
+
     @property
     def instances(self):
         return self._config.config['platforms']

molecule/provisioner/ansible.py

@@ -569,7 +569,11 @@ def inventory(self):
                     "{{ lookup('env', 'MOLECULE_INSTANCE_CONFIG') }}",
                     'molecule_no_log':
                     "{{ lookup('env', 'MOLECULE_NO_LOG') or not "
-                    "molecule_yml.provisioner.log|default(False) | bool }}"
+                    "molecule_yml.provisioner.log|default(False) | bool }}",
+                    'molecule_parallel_mode':
+                    self._config.is_parallel_mode,
+                    'molecule_parallel_run_uuid':
+                    self._config.scenario.parallel_run_uuid,
                 }
 
                 # All group

molecule/provisioner/ansible/playbooks/docker/create.yml

@@ -121,7 +121,7 @@
 
     - name: Create molecule instance(s)
       docker_container:
-        name: "{{ item.name }}"
+        name: "{{ item.name | molecule_parallelize_instance_name(molecule_parallel_run_uuid) }}"
         docker_host: "{{ item.docker_host | default(lookup('env', 'DOCKER_HOST') or 'unix://var/run/docker.sock') }}"
         cacert_path: "{{ item.cacert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/ca.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}"
         cert_path: "{{ item.cert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/cert.pem')  if lookup('env', 'DOCKER_CERT_PATH') else omit) }}"

molecule/provisioner/ansible/playbooks/docker/destroy.yml

@@ -7,7 +7,7 @@
   tasks:
     - name: Destroy molecule instance(s)
       docker_container:
-        name: "{{ item.name }}"
+        name: "{{ item.name | molecule_parallelize_instance_name(molecule_parallel_run_uuid) }}"
         docker_host: "{{ item.docker_host | default(lookup('env', 'DOCKER_HOST') or 'unix://var/run/docker.sock') }}"
         cacert_path: "{{ item.cacert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/ca.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}"
         cert_path: "{{ item.cert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/cert.pem')  if lookup('env', 'DOCKER_CERT_PATH') else omit) }}"

molecule/provisioner/ansible/plugins/filters/molecule_core.py

@@ -65,6 +65,10 @@ def get_docker_networks(data):
     return network_list
 
 
+def parallelize_instance_name(name, parallel_run_uuid):
+    return '{}-{}'.format(name, parallel_run_uuid)
+
+
 class FilterModule(object):
     """ Core Molecule filter plugins. """
 
@@ -74,4 +78,5 @@ def filters(self):
             'molecule_to_yaml': to_yaml,
             'molecule_header': header,
             'molecule_get_docker_networks': get_docker_networks,
+            'molecule_parallelize_instance_name': parallelize_instance_name,
         }

molecule/scenario.py

@@ -20,6 +20,8 @@
 
 import os
 import fnmatch
+from uuid import uuid4
+import shutil
 try:
     from pathlib import Path
 except ImportError:
@@ -138,10 +140,17 @@ def directory(self):
     def ephemeral_directory(self):
         project_directory = os.path.basename(self.config.project_directory)
         scenario_name = self.name
-        project_scenario_directory = os.path.join(
-            'molecule', project_directory, scenario_name)
-        path = ephemeral_directory(project_scenario_directory)
 
+        if self.is_parallel_mode:
+            project_directory = '{}-{}'.format(project_directory,
+                                               self.parallel_run_uuid)
+            project_scenario_directory = os.path.join(
+                'molecule_parallel', project_directory, scenario_name)
+        else:
+            project_scenario_directory = os.path.join(
+                'molecule', project_directory, scenario_name)
+
+        path = ephemeral_directory(project_scenario_directory)
         return ephemeral_directory(path)
 
     @property
@@ -219,12 +228,30 @@ def sequence(self):
             # TODO(retr0h): May change this handling in the future.
             return []
 
+    @property
+    def is_parallel_mode(self):
+        return self.config.is_parallel_mode
+
+    @property
+    def parallel_run_uuid(self):
+        return self._parallel_run_uuid
+
+    def _remove_parallel_directory(self):
+        msg = ('Removing ephemeral directory because '
+               'parallel execution mode has been configured')
+        LOG.info(msg)
+
+        shutil.rmtree(Path(self.ephemeral_directory).parent)
+
     def _setup(self):
         """
          Prepare the scenario for Molecule and returns None.
 
          :return: None
          """
+        if self.is_parallel_mode:
+            self._parallel_run_uuid = str(uuid4())
+
         if not os.path.isdir(self.inventory_directory):
             os.makedirs(self.inventory_directory)
 

molecule/util.py

@@ -30,6 +30,8 @@
 import colorama
 import yaml
 
+from uuid import uuid4
+
 from molecule.logger import get_logger
 
 LOG = get_logger(__name__)
@@ -311,3 +313,17 @@ def wrapper(*args, **kwargs):
         return memo[args]
 
     return wrapper
+
+
+def validate_args_with_parallel(cmd_args):
+    if cmd_args.get('parallel') and cmd_args.get('destroy') == 'never':
+        msg = '"--parallel" and "--destroy=never" are not supported'
+        sysexit_with_message(msg)
+
+
+def parallelize_instance_names(platforms, uuid):
+    def transform(platform):
+        platform['name'] = '{}-{}'.format(platform['name'], uuid)
+        return platform
+
+    return [transform(platform) for platform in platforms]