Beanchmark grid search

examples/pipelines_grid/CSP_grid.yml

@@ -0,0 +1,29 @@
+name: CSP + LDA
+paradigms:
+  - LeftRightImagery
+
+citations:
+  - https://doi.org/10.1007/BF01129656
+  - https://doi.org/10.1109/MSP.2008.4408441
+
+pipeline:
+  - name: Covariances
+    from: pyriemann.estimation
+    parameters:
+      estimator: oas
+
+  - name: CSP
+    from: pyriemann.spatialfilters
+    parameters:
+      nfilter: 6
+
+  - name: LinearDiscriminantAnalysis
+    from: sklearn.discriminant_analysis
+    parameters:
+      solver: svd
+
+param_grid:
+  csp__nfilter:
+    - 4
+    - 5
+    - 6

examples/pipelines_grid/EN.yml

@@ -0,0 +1,27 @@
+name: EN
+paradigms:
+  - LeftRightImagery
+
+citations:
+
+pipeline:
+  - name: Covariances
+    from: pyriemann.estimation
+    parameters:
+      estimator: oas
+
+  - name: TangentSpace
+    from: pyriemann.tangentspace
+    parameters:
+      metric: "riemann"
+
+  - name: LogisticRegression
+    from: sklearn.linear_model
+    parameters:
+      penalty: "elasticnet"
+      l1_ratio: 0.15
+      intercept_scaling: 1000.0
+      solver: "saga"
+      max_iter: 1000
+
+param_grid: None

examples/pipelines_grid/EN_grid.yml

@@ -0,0 +1,33 @@
+name: EN Grid
+paradigms:
+  - LeftRightImagery
+
+citations:
+
+pipeline:
+  - name: Covariances
+    from: pyriemann.estimation
+    parameters:
+      estimator: oas
+
+  - name: TangentSpace
+    from: pyriemann.tangentspace
+    parameters:
+      metric: "riemann"
+
+  - name: LogisticRegression
+    from: sklearn.linear_model
+    parameters:
+      penalty: "elasticnet"
+      l1_ratio: 0.70
+      intercept_scaling: 1000.0
+      solver: "saga"
+      max_iter: 1000
+
+param_grid:
+  logisticregression__l1_ratio:
+    - 0.20
+    - 0.30
+    - 0.45
+    - 0.65
+    - 0.75

examples/plot_benchmark_grid_search.py

@@ -0,0 +1,67 @@
+"""
+=======================
+Benchmarking with MOABB with Grid Search
+=======================
+
+This example shows how to use MOABB to benchmark a set of pipelines
+on all available datasets. In particular we run the Gridsearch to select the best hyperparameter of some pipelines
+and save the gridsearch.
+For this example, we will use only one dataset to keep the computation time low, but this benchmark is designed
+to easily scale to many datasets.
+"""
+# Authors: Igor Carrara <[email protected]>
+#
+# License: BSD (3-clause)
+
+import matplotlib.pyplot as plt
+
+from moabb import benchmark, set_log_level
+from moabb.analysis.plotting import score_plot
+
+
+set_log_level("info")
+
+
+###############################################################################
+# In this example, we will use only the dataset 'Zhou 2016'.
+#
+# Running the benchmark
+# ---------------------
+#
+# The benchmark is run using the ``benchmark`` function. You need to specify the
+# folder containing the pipelines to use, the kind of evaluation and the paradigm
+# to use. By default, the benchmark will use all available datasets for all
+# paradigms listed in the pipelines. You could restrict to specific evaluation and
+# paradigm using the ``evaluations`` and ``paradigms`` arguments.
+#
+# To save computation time, the results are cached. If you want to re-run the
+# benchmark, you can set the ``overwrite`` argument to ``True``.
+#
+# It is possible to indicate the folder to cache the results and the one to save
+# the analysis & figures. By default, the results are saved in the ``results``
+# folder, and the analysis & figures are saved in the ``benchmark`` folder.
+
+# In the results folder we will save the gridsearch evaluation
+# When write the pipeline in ylm file we need to specify the parameter that we want to test, in format
+# name__name_parameter. Note that name must be NOT in capital letter.
+# If the grid search is already implemented it will load the previous results
+
+results = benchmark(
+    # pipelines="../pipelines_grid/",
+    pipelines="./pipelines_grid/",
+    evaluations=["WithinSession"],
+    paradigms=["LeftRightImagery"],
+    include_datasets=["Zhou 2016"],
+    results="./results/",
+    overwrite=False,
+    plot=False,
+    output="./benchmark/",
+)
+
+###############################################################################
+# Benchmark prints a summary of the results. Detailed results are saved in a
+# pandas dataframe, and can be used to generate figures. The analysis & figures
+# are saved in the ``benchmark`` folder.
+
+score_plot(results)
+plt.show()

examples/sample_pipelines/CSP_SVM.yml

@@ -21,3 +21,5 @@ pipeline:
     from: sklearn.svm
     parameters:
       kernel: "linear"
+
+param_grid: None

examples/sample_pipelines/TSLR.yml

@@ -21,3 +21,5 @@ pipeline:
     from: sklearn.linear_model
     parameters:
       C: 1.0
+
+param_grid: None

moabb/benchmark.py

@@ -14,7 +14,11 @@
     CrossSubjectEvaluation,
     WithinSessionEvaluation,
 )
-from moabb.pipelines.utils import generate_paradigms, parse_pipelines_from_directory
+from moabb.pipelines.utils import (
+    generate_paradigms,
+    generate_param_grid,
+    parse_pipelines_from_directory,
+)
 
 
 log = logging.getLogger(__name__)
@@ -116,6 +120,8 @@ def benchmark(
     if paradigms is not None:
         prdgms = {p: prdgms[p] for p in paradigms}
 
+    param_grid = generate_param_grid(pipeline_configs, context_params, log)
+
     log.debug(f"The paradigms being run are {prdgms.keys()}")
 
     if len(context_params) == 0:
@@ -145,7 +151,9 @@ def benchmark(
                 n_jobs=n_jobs,
                 overwrite=overwrite,
             )
-            paradigm_results = context.process(pipelines=prdgms[paradigm])
+            paradigm_results = context.process(
+                pipelines=prdgms[paradigm], param_grid=param_grid
+            )
             paradigm_results["paradigm"] = f"{paradigm}"
             paradigm_results["evaluation"] = f"{evaluation}"
             eval_results[f"{paradigm}"] = paradigm_results

moabb/pipelines/utils.py

@@ -82,15 +82,26 @@ def parse_pipelines_from_directory(dir_path):
             # load config
             config_dict = yaml.load(content, Loader=yaml.FullLoader)
             ppl = create_pipeline_from_config(config_dict["pipeline"])
-            pipeline_configs.append(
-                {
-                    "paradigms": config_dict["paradigms"],
-                    "pipeline": ppl,
-                    "name": config_dict["name"],
-                }
-            )
+            if config_dict["param_grid"] is not None:
+                pipeline_configs.append(
+                    {
+                        "paradigms": config_dict["paradigms"],
+                        "pipeline": ppl,
+                        "name": config_dict["name"],
+                        "param_grid": config_dict["param_grid"],
+                    }
+                )
+            else:
+                pipeline_configs.append(
+                    {
+                        "paradigms": config_dict["paradigms"],
+                        "pipeline": ppl,
+                        "name": config_dict["name"],
+                    }
+                )
 
     # we can do the same for python defined pipeline
+    # TODO for python pipelines
     python_files = glob(os.path.join(dir_path, "*.py"))
 
     for python_file in python_files:
@@ -162,6 +173,24 @@ def generate_paradigms(pipeline_configs, context=None, logger=log):
     return paradigms
 
 
+def generate_param_grid(pipeline_configs, context=None, logger=log):
+
+    context = context or {}
+    param_grid = {}
+    for config in pipeline_configs:
+
+        if "paradigms" not in config.keys():
+            logger.error("{} must have a 'paradigms' key.".format(config))
+            continue
+
+        # iterate over paradigms
+        if config["param_grid"] != "None":
+            param_grid[config["name"]] = config["param_grid"]
+            # param_grid[config["name"]] = config["param_grid"].get(config["name"])
+
+    return param_grid
+
+
 class FilterBank(BaseEstimator, TransformerMixin):
     """Apply a given indentical pipeline over a bank of filter.
 

moabb/tests/test_pipelines/CSP.yml

@@ -13,3 +13,5 @@ pipeline:
     from: sklearn.discriminant_analysis
     parameters:
       solver: svd
+
+param_grid: None

moabb/tests/test_pipelines/LogVar.yml

@@ -12,3 +12,5 @@ pipeline:
     from: sklearn.discriminant_analysis
     parameters:
       solver: svd
+
+param_grid: None

moabb/tests/test_pipelines/SSVEP_CCA.yml

@@ -10,3 +10,5 @@ pipeline:
       n_harmonics: 3
       interval: [1, 3]
       freqs: { "13": 0, "17": 1 }
+
+param_grid: None

pipelines/CCA-SSVEP.yml

@@ -10,3 +10,5 @@ pipeline:
       n_harmonics: 3
       interval: [2, 4]
       freqs: { "13": 2, "17": 3, "21": 4 }
+
+param_grid: None

pipelines/CSP.yml

@@ -21,3 +21,5 @@ pipeline:
     from: sklearn.discriminant_analysis
     parameters:
       solver: svd
+
+param_grid: None

pipelines/CSP_SVM.yml

@@ -21,3 +21,5 @@ pipeline:
     from: sklearn.svm
     parameters:
       kernel: "linear"
+
+param_grid: None

pipelines/LogVar.yml

@@ -11,3 +11,5 @@ pipeline:
     from: sklearn.discriminant_analysis
     parameters:
       solver: svd
+
+param_grid: None

pipelines/LogVarSVM.yml

@@ -11,3 +11,5 @@ pipeline:
     from: sklearn.svm
     parameters:
       kernel: "linear"
+
+param_grid: None

pipelines/MDM-SSVEP.yml

@@ -18,3 +18,5 @@ pipeline:
     from: pyriemann.classification
     parameters:
       metric: "riemann"
+
+param_grid: None

pipelines/MDM.yml

@@ -15,3 +15,5 @@ pipeline:
     from: pyriemann.classification
     parameters:
       metric: "riemann"
+
+param_grid: None

pipelines/TSLR-SSVEP.yml

@@ -21,3 +21,5 @@ pipeline:
     from: sklearn.linear_model
     parameters:
       C: 1.0
+
+param_grid: None

pipelines/TSLR.yml

@@ -21,3 +21,5 @@ pipeline:
     from: sklearn.linear_model
     parameters:
       C: 1.0
+
+param_grid: None

pipelines/TSSVM-SSVEP.yml

@@ -21,3 +21,5 @@ pipeline:
     from: sklearn.svm
     parameters:
       kernel: "linear"
+
+param_grid: None

pipelines/TSSVM.yml

@@ -21,3 +21,5 @@ pipeline:
     from: sklearn.svm
     parameters:
       kernel: "linear"
+
+param_grid: None

pipelines/regCSP+shLDA.yml

@@ -22,3 +22,5 @@ pipeline:
     parameters:
       solver: lsqr
       shrinkage: auto
+
+param_grid: None