Example CodeCarbon

docs/source/whats_new.rst

@@ -28,6 +28,7 @@ Enhancements
 - Add BrainDecode example(:gh:`340` by `Igor Carrara`_ and `Bruno Aristimunha`_)
 - Add Google Analytics to the documentation (:gh:`335` by `Bruno Aristimunha`_)
 - Add CodeCarbon to track emission CO₂ (:gh:`350` by `Igor Carrara`_, `Bruno Aristimunha`_ and `Sylvain Chevallier`_)
+- Add CodeCarbon example (:gh:`356` by `Igor Carrara`_ and `Bruno Aristimunha`_)
 
 Bugs
 ~~~~

examples/example_codecarbon.py

@@ -0,0 +1,117 @@
+"""
+=================================================
+Benchmarking with MOABB showing the CO2 footprint
+=================================================
+
+This example shows how to use MOABB to track the CO2 footprint
+using `CodeCarbon library<https://codecarbon.io/>`_.
+For this example, we will use only one
+dataset to keep the computation time low, but this benchmark is designed
+to easily scale to many datasets. Due to limitation of online documentation
+generation, the results is computed on a local cluster but could be easily
+replicated on your infrastructure.
+"""
+# Authors: Igor Carrara <[email protected]>
+#          Bruno Aristimunha <[email protected]>
+#
+# License: BSD (3-clause)
+
+###############################################################################
+from moabb import benchmark, set_log_level
+from moabb.analysis.plotting import codecarbon_plot
+from moabb.datasets import BNCI2014001, Zhou2016
+from moabb.paradigms import LeftRightImagery
+
+
+set_log_level("info")
+
+###############################################################################
+# Loading the pipelines
+# ---------------------
+#
+# To run this example we use several pipelines, ML and DL (Keras) and also
+# pipelines that need an optimization of the hyperparameter.
+# All this different pipelines are stored in ``pipelines_codecarbon``
+
+###############################################################################
+# Selecting the datasets (optional)
+# ---------------------------------
+#
+# If you want to limit your benchmark on a subset of datasets, you can use the
+# ``include_datasets`` and ``exclude_datasets`` arguments. You will need either
+# to provide the dataset's object, or a dataset's code. To get the list of
+# available dataset's code for a given paradigm, you can use the following command:
+
+paradigm = LeftRightImagery()
+for d in paradigm.datasets:
+    print(d.code)
+
+###############################################################################
+# In this example, we will use only the last dataset, 'Zhou 2016', considering only the first subject.
+#
+# Running the benchmark
+# ---------------------
+#
+# The benchmark is run using the ``benchmark`` function. You need to specify the
+# folder containing the pipelines to use, the kind of evaluation and the paradigm
+# to use. By default, the benchmark will use all available datasets for all
+# paradigms listed in the pipelines. You could restrict to specific evaluation and
+# paradigm using the ``evaluations`` and ``paradigms`` arguments.
+#
+# To save computation time, the results are cached. If you want to re-run the
+# benchmark, you can set the ``overwrite`` argument to ``True``.
+#
+# It is possible to indicate the folder to cache the results and the one to save
+# the analysis & figures. By default, the results are saved in the ``results``
+# folder, and the analysis & figures are saved in the ``benchmark`` folder.
+
+dataset = Zhou2016()
+dataset2 = BNCI2014001()
+dataset.subject_list = dataset.subject_list[:1]
+dataset2.subject_list = dataset2.subject_list[:1]
+datasets = [dataset, dataset2]
+
+results = benchmark(
+    pipelines="./pipelines_codecarbon/",
+    evaluations=["WithinSession"],
+    paradigms=["LeftRightImagery"],
+    include_datasets=datasets,
+    results="./results/",
+    overwrite=False,
+    plot=False,
+    output="./benchmark/",
+)
+
+###############################################################################
+# Benchmark prints a summary of the results. Detailed results are saved in a
+# pandas dataframe, and can be used to generate figures. The analysis & figures
+# are saved in the ``benchmark`` folder.
+results.head()
+
+order_list = [
+    "CSP + SVM",
+    "Tangent Space LR",
+    "EN Grid",
+    "CSP + LDA Grid",
+    "Keras_EEGNet_8_2",
+]
+
+###############################################################################
+# Plotting the results
+# --------------------
+# We can plot the results using the ``codecarbon_plot`` function, generated bellow. This function takes the
+# dataframe returned by the ``benchmark`` function as input, and returns a pyplot figure.
+# The ``order_list`` argument is used to specify the order of the pipelines in
+# the plot.
+
+codecarbon_plot(results, order_list, country="(France)")
+
+###############################################################################
+# The result expected will be the following image, but varying depending on the machine
+# and the country used to run the example.
+#
+# .. image:: ../images/example_codecarbon.png
+#    :align: center
+#    :alt: carbon_example
+#
+###############################################################################

examples/pipelines_codecarbon/CSP_SVM.yml

@@ -0,0 +1,23 @@
+name: CSP + SVM
+paradigms:
+  - LeftRightImagery
+
+citations:
+  - https://doi.org/10.1007/BF01129656
+  - https://doi.org/10.1109/MSP.2008.4408441
+
+pipeline:
+  - name: Covariances
+    from: pyriemann.estimation
+    parameters:
+      estimator: oas
+
+  - name: CSP
+    from: pyriemann.spatialfilters
+    parameters:
+      nfilter: 6
+
+  - name: SVC
+    from: sklearn.svm
+    parameters:
+      kernel: "linear"

examples/pipelines_codecarbon/CSP_grid.yml

@@ -0,0 +1,32 @@
+name: CSP + LDA Grid
+paradigms:
+  - LeftRightImagery
+
+citations:
+  - https://doi.org/10.1007/BF01129656
+  - https://doi.org/10.1109/MSP.2008.4408441
+
+pipeline:
+  - name: Covariances
+    from: pyriemann.estimation
+    parameters:
+      estimator: oas
+
+  - name: CSP
+    from: pyriemann.spatialfilters
+    parameters:
+      nfilter: 6
+
+  - name: LinearDiscriminantAnalysis
+    from: sklearn.discriminant_analysis
+    parameters:
+      solver: svd
+
+param_grid:
+  csp__nfilter:
+    - 2
+    - 3
+    - 4
+    - 5
+    - 6
+    - 7

examples/pipelines_codecarbon/EN_grid.yml

@@ -0,0 +1,34 @@
+name: EN Grid
+paradigms:
+  - LeftRightImagery
+
+citations:
+  - https://doi.org/10.1109/TBME.2022.3154885
+
+pipeline:
+  - name: Covariances
+    from: pyriemann.estimation
+    parameters:
+      estimator: oas
+
+  - name: TangentSpace
+    from: pyriemann.tangentspace
+    parameters:
+      metric: "riemann"
+
+  - name: LogisticRegression
+    from: sklearn.linear_model
+    parameters:
+      penalty: "elasticnet"
+      l1_ratio: 0.70
+      intercept_scaling: 1000.0
+      solver: "saga"
+      max_iter: 1000
+
+param_grid:
+  logisticregression__l1_ratio:
+    - 0.20
+    - 0.30
+    - 0.45
+    - 0.65
+    - 0.75

examples/pipelines_codecarbon/Keras_EEGNet_8_2.yml

@@ -0,0 +1,40 @@
+name: Keras_EEGNet_8_2
+
+paradigms:
+  - LeftRightImagery
+  - MotorImagery
+
+citations:
+  - https://doi.org/10.1088/1741-2552/aace8c
+
+pipeline:
+  - name: StandardScaler_Epoch
+    from: moabb.pipelines.features
+
+  - name: KerasEEGNet_8_2
+    from: moabb.pipelines.deep_learning
+    parameters:
+      loss: "sparse_categorical_crossentropy"
+      optimizer:
+        - name: Adam
+          from: tensorflow.keras.optimizers.legacy
+          parameters:
+            learning_rate: 0.0009
+      epochs: 10
+      batch_size: 64
+      verbose: 0
+      random_state: 42
+      validation_split: 0.2
+      callbacks:
+        - name: EarlyStopping
+          from: tensorflow.keras.callbacks
+          parameters:
+            monitor: "val_loss"
+            patience: 300
+
+        - name: ReduceLROnPlateau
+          from: tensorflow.keras.callbacks
+          parameters:
+            monitor: "val_loss"
+            patience: 300
+            factor: 0.5

examples/pipelines_codecarbon/TSLR.yml

@@ -0,0 +1,23 @@
+name: Tangent Space LR
+
+paradigms:
+  - LeftRightImagery
+
+citations:
+  - https://doi.org/10.1016/j.neucom.2012.12.039
+
+pipeline:
+  - name: Covariances
+    from: pyriemann.estimation
+    parameters:
+      estimator: oas
+
+  - name: TangentSpace
+    from: pyriemann.tangentspace
+    parameters:
+      metric: "riemann"
+
+  - name: LogisticRegression
+    from: sklearn.linear_model
+    parameters:
+      C: 1.0

moabb/analysis/__init__.py

@@ -8,6 +8,7 @@
     compute_dataset_statistics,
     find_significant_differences,
 )
+from moabb.analysis.plotting import codecarbon_plot  # noqa: F401
 from moabb.analysis.results import Results  # noqa: F401
 
 

moabb/analysis/plotting.py

@@ -76,6 +76,57 @@ def score_plot(data, pipelines=None):
     return fig, color_dict
 
 
+def codecarbon_plot(data, order_list=None, pipelines=None, country=""):
+    """Plot code carbon consume for the results from the benchmark
+
+    Parameters
+    ----------
+    data: output of Results.to_dataframe()
+        results on datasets
+    order_list: list of str | None
+        order of pipelines to include in this plot
+    pipelines: list of str | None
+        pipelines to include in this plot
+    country: str
+        country to include in the title
+    pipelines: list of str | None
+        pipelines to include in this plot
+
+    Returns
+    -------
+    fig: Figure
+        Pyplot handle
+    """
+    data = collapse_session_scores(data)
+    unique_ids = data["dataset"].apply(_simplify_names)
+    if len(unique_ids) != len(set(unique_ids)):
+        log.warning("Dataset names are too similar, turning off name shortening")
+    else:
+        data["dataset"] = unique_ids
+
+    if pipelines is not None:
+        data = data[data.pipeline.isin(pipelines)]
+
+    data = data.rename(columns={"carbon emission": "carbon_emission"})
+
+    fig = sea.catplot(
+        kind="bar",
+        data=data,
+        x="dataset",
+        y="carbon_emission",
+        hue="pipeline",
+        palette=PIPELINE_PALETTE,
+        height=8.5,
+        hue_order=order_list,
+    ).set(title=r"$CO_2$ emission per dataset and algorithm" + country)
+    fig.set(yscale="log")
+    fig.tight_layout()
+    fig.set_ylabels(r"$CO_2$ emission (Log Scale)")
+    fig.set_xlabels("Dataset")
+
+    return fig
+
+
 def paired_plot(data, alg1, alg2):
     """Generate a figure with a paired plot