notebooks/naive.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Quick Start: Running Naive model on gift-eval benchmark\n",
    "\n",
    "This notebook shows how to run the Naive model on the gift-eval benchmark.\n",
    "\n",
    "Make sure you download the gift-eval benchmark and set the `GIFT-EVAL` environment variable correctly before running this notebook.\n",
    "\n",
    "We will use the `Dataset` class to load the data and run the model. If you have not already please check out the [dataset.ipynb](./dataset.ipynb) notebook to learn more about the `Dataset` class. We are going to just run the model on two datasets for brevity. But feel free to run on any dataset by changing the `short_datasets` and `med_long_datasets` variables below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "# Load environment variables\n",
    "load_dotenv()\n",
    "\n",
    "# short_datasets = \"m4_yearly m4_quarterly m4_monthly m4_weekly m4_daily m4_hourly electricity/15T electricity/H electricity/D electricity/W solar/10T solar/H solar/D solar/W hospital covid_deaths us_births/D us_births/M us_births/W saugeenday/D saugeenday/M saugeenday/W temperature_rain_with_missing kdd_cup_2018_with_missing/H kdd_cup_2018_with_missing/D car_parts_with_missing restaurant hierarchical_sales/D hierarchical_sales/W LOOP_SEATTLE/5T LOOP_SEATTLE/H LOOP_SEATTLE/D SZ_TAXI/15T SZ_TAXI/H M_DENSE/H M_DENSE/D ett1/15T ett1/H ett1/D ett1/W ett2/15T ett2/H ett2/D ett2/W jena_weather/10T jena_weather/H jena_weather/D bitbrains_fast_storage/5T bitbrains_fast_storage/H bitbrains_rnd/5T bitbrains_rnd/H bizitobs_application bizitobs_service bizitobs_l2c/5T bizitobs_l2c/H\"\n",
    "short_datasets = \"m4_weekly\"\n",
    "\n",
    "# med_long_datasets = \"electricity/15T electricity/H solar/10T solar/H kdd_cup_2018_with_missing/H LOOP_SEATTLE/5T LOOP_SEATTLE/H SZ_TAXI/15T M_DENSE/H ett1/15T ett1/H ett2/15T ett2/H jena_weather/10T jena_weather/H bitbrains_fast_storage/5T bitbrains_rnd/5T bizitobs_application bizitobs_service bizitobs_l2c/5T bizitobs_l2c/H\"\n",
    "med_long_datasets = \"bizitobs_l2c/H\"\n",
    "\n",
    "# Get union of short and med_long datasets\n",
    "all_datasets = list(set(short_datasets.split() + med_long_datasets.split()))\n",
    "\n",
    "dataset_properties_map = json.load(open(\"dataset_properties.json\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gluonts.ev.metrics import (\n",
    "    MSE,\n",
    "    MAE,\n",
    "    MASE,\n",
    "    MAPE,\n",
    "    SMAPE,\n",
    "    MSIS,\n",
    "    RMSE,\n",
    "    NRMSE,\n",
    "    ND,\n",
    "    MeanWeightedSumQuantileLoss,\n",
    ")\n",
    "\n",
    "# Instantiate the metrics\n",
    "metrics = [\n",
    "    MSE(forecast_type=\"mean\"),\n",
    "    MSE(forecast_type=0.5),\n",
    "    MAE(),\n",
    "    MASE(),\n",
    "    MAPE(),\n",
    "    SMAPE(),\n",
    "    MSIS(),\n",
    "    RMSE(),\n",
    "    NRMSE(),\n",
    "    ND(),\n",
    "    MeanWeightedSumQuantileLoss(\n",
    "        quantile_levels=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]\n",
    "    ),\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## StatsForecast Predictor\n",
    "\n",
    "We will use the `StatsForecastPredictor` class to wrap the Naive model. This class is a wrapper around the `StatsForecast` library and is used to make it compatible with the `gluonts` interface. Note that `StatsForecastPredictor` is compatible with any model from the `statsforecast` library, however for brevity we will just use the `Naive` model in this notebook.\n",
    "\n",
    "This is just meant to be a simple wrapper to get you started, feel free to use your own custom implementation to wrap any model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/iaksu/miniforge3/envs/gifteval/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import inspect\n",
    "from dataclasses import dataclass, field\n",
    "from typing import Iterator, List, Optional, Type\n",
    "import logging\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from gluonts.core.component import validated\n",
    "from gluonts.dataset import Dataset\n",
    "from gluonts.dataset.util import forecast_start\n",
    "from gluonts.model import Forecast\n",
    "from gluonts.model.forecast import QuantileForecast\n",
    "from gluonts.model.predictor import RepresentablePredictor\n",
    "from gluonts.transform.feature import LastValueImputation, MissingValueImputation\n",
    "from statsforecast import StatsForecast\n",
    "from statsforecast.models import (\n",
    "    Naive,\n",
    "    SeasonalNaive,\n",
    ")\n",
    "\n",
    "\n",
    "@dataclass\n",
    "class ModelConfig:\n",
    "    quantile_levels: Optional[List[float]] = None\n",
    "    forecast_keys: List[str] = field(init=False)\n",
    "    statsforecast_keys: List[str] = field(init=False)\n",
    "    intervals: Optional[List[int]] = field(init=False)\n",
    "\n",
    "    def __post_init__(self):\n",
    "        self.forecast_keys = [\"mean\"]\n",
    "        self.statsforecast_keys = [\"mean\"]\n",
    "        if self.quantile_levels is None:\n",
    "            self.intervals = None\n",
    "            return\n",
    "\n",
    "        intervals = set()\n",
    "\n",
    "        for quantile_level in self.quantile_levels:\n",
    "            interval = round(200 * (max(quantile_level, 1 - quantile_level) - 0.5))\n",
    "            intervals.add(interval)\n",
    "            side = \"hi\" if quantile_level > 0.5 else \"lo\"\n",
    "            self.forecast_keys.append(str(quantile_level))\n",
    "            self.statsforecast_keys.append(f\"{side}-{interval}\")\n",
    "\n",
    "        self.intervals = sorted(intervals)\n",
    "\n",
    "\n",
    "class StatsForecastPredictor(RepresentablePredictor):\n",
    "    \"\"\"\n",
    "    A predictor type that wraps models from the `statsforecast`_ package.\n",
    "\n",
    "    This class is used via subclassing and setting the ``ModelType`` class\n",
    "    attribute to specify the ``statsforecast`` model type to use.\n",
    "\n",
    "    .. _statsforecast: https://github.com/Nixtla/statsforecast\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    prediction_length\n",
    "        Prediction length for the model to use.\n",
    "    quantile_levels\n",
    "        Optional list of quantile levels that we want predictions for.\n",
    "        Note: this is only supported by specific types of models, such as\n",
    "        ``AutoARIMA``. By default this is ``None``, giving only the mean\n",
    "        prediction.\n",
    "    **model_params\n",
    "        Keyword arguments to be passed to the model type for construction.\n",
    "        The specific arguments accepted or required depend on the\n",
    "        ``ModelType``; please refer to the documentation of ``statsforecast``\n",
    "        for details.\n",
    "    \"\"\"\n",
    "\n",
    "    ModelType: Type\n",
    "\n",
    "    @validated()\n",
    "    def __init__(\n",
    "        self,\n",
    "        prediction_length: int,\n",
    "        season_length: int,\n",
    "        freq: str,\n",
    "        quantile_levels: Optional[List[float]] = None,\n",
    "        imputation_method: MissingValueImputation = LastValueImputation(),\n",
    "        max_length: Optional[int] = None,\n",
    "        batch_size: int = 1,\n",
    "        parallel: bool = False,\n",
    "        **model_params,\n",
    "    ) -> None:\n",
    "        super().__init__(prediction_length=prediction_length)\n",
    "\n",
    "        if \"season_length\" in inspect.signature(self.ModelType.__init__).parameters:\n",
    "            model_params[\"season_length\"] = season_length\n",
    "\n",
    "        self.freq = freq\n",
    "        self.model = StatsForecast(\n",
    "            models=[self.ModelType(**model_params)],\n",
    "            freq=freq,\n",
    "            fallback_model=SeasonalNaive(season_length=season_length),\n",
    "            n_jobs=-1 if parallel else 1,\n",
    "        )\n",
    "        self.fallback_model = StatsForecast(\n",
    "            # Fallback model when main model returns NaNs\n",
    "            models=[SeasonalNaive(season_length=season_length)],\n",
    "            freq=freq,\n",
    "            n_jobs=-1 if parallel else 1,\n",
    "        )\n",
    "        self.config = ModelConfig(quantile_levels=quantile_levels)\n",
    "        self.imputation_method = imputation_method\n",
    "        self.batch_size = batch_size\n",
    "        self.max_length = max_length\n",
    "\n",
    "        # Set up the logger\n",
    "        self.logger = logging.getLogger(__name__)\n",
    "        logging.basicConfig(level=logging.INFO)\n",
    "\n",
    "    def predict(self, dataset: Dataset, **kwargs) -> Iterator[Forecast]:\n",
    "        batch = {}\n",
    "        total_series = len(dataset)\n",
    "        self.logger.info(f\"Starting prediction on {total_series} series.\")\n",
    "\n",
    "        for idx, entry in enumerate(dataset):\n",
    "            assert entry[\"target\"].ndim == 1, \"only for univariate time series\"\n",
    "            assert (\n",
    "                len(entry[\"target\"]) >= 1\n",
    "            ), \"all time series should have at least one data point\"\n",
    "\n",
    "            if self.max_length is not None:\n",
    "                entry[\"start\"] += len(entry[\"target\"][: -self.max_length])\n",
    "                entry[\"target\"] = entry[\"target\"][-self.max_length :]\n",
    "\n",
    "            target = np.asarray(entry[\"target\"], np.float32)\n",
    "            if np.isnan(target).any():\n",
    "                target = target.copy()\n",
    "                target = self.imputation_method(target)\n",
    "\n",
    "            unique_id = (\n",
    "                f\"{entry['item_id']}_{str(forecast_start(entry))}_{str(len(batch))}\"\n",
    "            )\n",
    "            start = entry[\"start\"]\n",
    "            batch[unique_id] = pd.DataFrame(\n",
    "                {\n",
    "                    \"unique_id\": unique_id,\n",
    "                    \"ds\": pd.date_range(\n",
    "                        start=start.to_timestamp(),\n",
    "                        periods=len(target),\n",
    "                        freq=start.freq,\n",
    "                    ).to_numpy(),\n",
    "                    \"y\": target,\n",
    "                }\n",
    "            )\n",
    "\n",
    "            if len(batch) == self.batch_size:\n",
    "                self.logger.info(f\"Processing batch {idx // self.batch_size + 1}.\")\n",
    "                results = self.sf_predict(pd.concat(batch.values()))\n",
    "                yield from self.yield_forecast(batch.keys(), results)\n",
    "                batch = {}\n",
    "\n",
    "        if len(batch) > 0:\n",
    "            self.logger.info(f\"Processing final batch.\")\n",
    "            results = self.sf_predict(pd.concat(batch.values()))\n",
    "            yield from self.yield_forecast(batch.keys(), results)\n",
    "\n",
    "        self.logger.info(\"Prediction completed.\")\n",
    "\n",
    "    def sf_predict(self, Y_df: pd.DataFrame) -> pd.DataFrame:\n",
    "        kwargs = {}\n",
    "        if self.config.intervals is not None:\n",
    "            kwargs[\"level\"] = self.config.intervals\n",
    "        results = self.model.forecast(\n",
    "            df=Y_df,\n",
    "            h=self.prediction_length,\n",
    "            **kwargs,\n",
    "        )\n",
    "        # replace nan results with fallback\n",
    "        row_nan = results.isnull().values.any(axis=-1)\n",
    "        if row_nan.any():\n",
    "            nan_ids = results[row_nan].index.values\n",
    "            nan_df = Y_df[Y_df[\"unique_id\"].isin(nan_ids)]\n",
    "            fallback_results = self.fallback_model.forecast(\n",
    "                df=nan_df,\n",
    "                h=self.prediction_length,\n",
    "                **kwargs,\n",
    "            )\n",
    "            results = pd.concat(\n",
    "                [\n",
    "                    results[~results.index.isin(nan_ids)],\n",
    "                    fallback_results,\n",
    "                ]\n",
    "            )\n",
    "\n",
    "        return results\n",
    "\n",
    "    def yield_forecast(\n",
    "        self, item_ids, results: pd.DataFrame\n",
    "    ) -> Iterator[QuantileForecast]:\n",
    "        results.set_index('unique_id',inplace=True)\n",
    "        for idx in item_ids:\n",
    "            prediction = results.loc[idx]\n",
    "            forecast_arrays = []\n",
    "            model_name = self.ModelType.__name__\n",
    "            for key in self.config.statsforecast_keys:\n",
    "                if key == \"mean\":\n",
    "                    forecast_arrays.append(prediction.loc[:, model_name].to_numpy())\n",
    "                else:\n",
    "                    forecast_arrays.append(\n",
    "                        prediction.loc[:, f\"{model_name}-{key}\"].to_numpy()\n",
    "                    )\n",
    "\n",
    "            yield QuantileForecast(\n",
    "                forecast_arrays=np.stack(forecast_arrays, axis=0),\n",
    "                forecast_keys=self.config.forecast_keys,\n",
    "                start_date=prediction.ds.iloc[0].to_period(freq=self.freq),\n",
    "                item_id=idx,\n",
    "            )\n",
    "\n",
    "\n",
    "class NaivePredictor(StatsForecastPredictor):\n",
    "    \"\"\"\n",
    "    A predictor wrapping the ``Naive`` model from `statsforecast`_.\n",
    "\n",
    "    See :class:`StatsForecastPredictor` for the list of arguments.\n",
    "\n",
    "    .. _statsforecast: https://github.com/Nixtla/statsforecast\n",
    "    \"\"\"\n",
    "\n",
    "    ModelType = Naive"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation\n",
    "\n",
    "Now that we have our predictor class, we can use it to predict on the gift-eval benchmark datasets. We will use the `evaluate_model` function to evaluate the model. This function is a helper function to evaluate the model on the test data and return the results in a dictionary. We are going to follow the naming conventions explained in the [README](../README.md) file to store the results in a csv file called `all_results.csv` under the `results/naive` folder.\n",
    "\n",
    "The first column in the csv file is the dataset config name which is a combination of the dataset name, frequency and the term:\n",
    "\n",
    "```python\n",
    "f\"{dataset_name}/{freq}/{term}\"\n",
    "```\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "tags": [
     "naive_eval"
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing dataset: m4_weekly\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]INFO:__main__:Starting prediction on 359 series.\n",
      "INFO:__main__:Processing final batch.\n",
      "/Users/iaksu/miniforge3/envs/gifteval/lib/python3.10/site-packages/statsforecast/core.py:494: FutureWarning: In a future version the predictions will have the id as a column. You can set the `NIXTLA_ID_AS_COL` environment variable to adopt the new behavior and to suppress this warning.\n",
      "  warnings.warn(\n",
      "INFO:__main__:Prediction completed.\n",
      "359it [00:02, 169.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results for m4_weekly have been written to ../results/naive/all_results.csv\n",
      "Processing dataset: bizitobs_l2c/H\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]INFO:__main__:Starting prediction on 42 series.\n",
      "INFO:__main__:Processing final batch.\n",
      "/Users/iaksu/miniforge3/envs/gifteval/lib/python3.10/site-packages/statsforecast/core.py:494: FutureWarning: In a future version the predictions will have the id as a column. You can set the `NIXTLA_ID_AS_COL` environment variable to adopt the new behavior and to suppress this warning.\n",
      "  warnings.warn(\n",
      "INFO:__main__:Prediction completed.\n",
      "42it [00:00, 541.78it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results for bizitobs_l2c/H have been written to ../results/naive/all_results.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]INFO:__main__:Starting prediction on 7 series.\n",
      "INFO:__main__:Processing final batch.\n",
      "/Users/iaksu/miniforge3/envs/gifteval/lib/python3.10/site-packages/statsforecast/core.py:494: FutureWarning: In a future version the predictions will have the id as a column. You can set the `NIXTLA_ID_AS_COL` environment variable to adopt the new behavior and to suppress this warning.\n",
      "  warnings.warn(\n",
      "INFO:__main__:Prediction completed.\n",
      "7it [00:00, 168.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results for bizitobs_l2c/H have been written to ../results/naive/all_results.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]INFO:__main__:Starting prediction on 7 series.\n",
      "INFO:__main__:Processing final batch.\n",
      "/Users/iaksu/miniforge3/envs/gifteval/lib/python3.10/site-packages/statsforecast/core.py:494: FutureWarning: In a future version the predictions will have the id as a column. You can set the `NIXTLA_ID_AS_COL` environment variable to adopt the new behavior and to suppress this warning.\n",
      "  warnings.warn(\n",
      "INFO:__main__:Prediction completed.\n",
      "7it [00:00, 149.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results for bizitobs_l2c/H have been written to ../results/naive/all_results.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from gluonts.model import evaluate_model\n",
    "import csv\n",
    "import os\n",
    "import time\n",
    "from gluonts.time_feature import get_seasonality\n",
    "from gift_eval.data import Dataset\n",
    "\n",
    "# Iterate over all available datasets\n",
    "\n",
    "output_dir = \"../results/naive\"\n",
    "# Ensure the output directory exists\n",
    "os.makedirs(output_dir, exist_ok=True)\n",
    "\n",
    "# Define the path for the CSV file\n",
    "csv_file_path = os.path.join(output_dir, \"all_results.csv\")\n",
    "\n",
    "pretty_names = {\n",
    "    \"saugeenday\": \"saugeen\",\n",
    "    \"temperature_rain_with_missing\": \"temperature_rain\",\n",
    "    \"kdd_cup_2018_with_missing\": \"kdd_cup_2018\",\n",
    "    \"car_parts_with_missing\": \"car_parts\",\n",
    "}\n",
    "\n",
    "with open(csv_file_path, \"w\", newline=\"\") as csvfile:\n",
    "    writer = csv.writer(csvfile)\n",
    "\n",
    "    # Write the header\n",
    "    writer.writerow(\n",
    "        [\n",
    "            \"dataset\",\n",
    "            \"model\",\n",
    "            \"eval_metrics/MSE[mean]\",\n",
    "            \"eval_metrics/MSE[0.5]\",\n",
    "            \"eval_metrics/MAE[0.5]\",\n",
    "            \"eval_metrics/MASE[0.5]\",\n",
    "            \"eval_metrics/MAPE[0.5]\",\n",
    "            \"eval_metrics/sMAPE[0.5]\",\n",
    "            \"eval_metrics/MSIS\",\n",
    "            \"eval_metrics/RMSE[mean]\",\n",
    "            \"eval_metrics/NRMSE[mean]\",\n",
    "            \"eval_metrics/ND[0.5]\",\n",
    "            \"eval_metrics/mean_weighted_sum_quantile_loss\",\n",
    "            \"domain\",\n",
    "            \"num_variates\",\n",
    "        ]\n",
    "    )\n",
    "\n",
    "for ds_name in all_datasets:\n",
    "    print(f\"Processing dataset: {ds_name}\")\n",
    "\n",
    "    terms = [\"short\", \"medium\", \"long\"]\n",
    "    for term in terms:\n",
    "        if (\n",
    "            term == \"medium\" or term == \"long\"\n",
    "        ) and ds_name not in med_long_datasets.split():\n",
    "            continue\n",
    "\n",
    "        if \"/\" in ds_name:\n",
    "            ds_key = ds_name.split(\"/\")[0]\n",
    "            ds_freq = ds_name.split(\"/\")[1]\n",
    "            ds_key = ds_key.lower()\n",
    "            ds_key = pretty_names.get(ds_key, ds_key)\n",
    "        else:\n",
    "            ds_key = ds_name.lower()\n",
    "            ds_key = pretty_names.get(ds_key, ds_key)\n",
    "            ds_freq = dataset_properties_map[ds_key][\"frequency\"]\n",
    "        ds_config = f\"{ds_key}/{ds_freq}/{term}\"\n",
    "\n",
    "        # Initialize the dataset\n",
    "        to_univariate = (\n",
    "            False\n",
    "            if Dataset(name=ds_name, term=term, to_univariate=False).target_dim == 1\n",
    "            else True\n",
    "        )\n",
    "        dataset = Dataset(name=ds_name, term=term, to_univariate=to_univariate)\n",
    "        season_length = get_seasonality(dataset.freq)\n",
    "\n",
    "        # Initialize the predictor\n",
    "        predictor = NaivePredictor(\n",
    "            dataset.prediction_length,\n",
    "            season_length=season_length,\n",
    "            freq=dataset.freq,\n",
    "            quantile_levels=[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9],\n",
    "            batch_size=512,\n",
    "        )\n",
    "\n",
    "        # Measure the time taken for evaluation\n",
    "        res = evaluate_model(\n",
    "            predictor,\n",
    "            test_data=dataset.test_data,\n",
    "            metrics=metrics,\n",
    "            batch_size=512,\n",
    "            axis=None,\n",
    "            mask_invalid_label=True,\n",
    "            allow_nan_forecast=False,\n",
    "            seasonality=season_length,\n",
    "        )\n",
    "\n",
    "        # Append the results to the CSV file\n",
    "        with open(csv_file_path, \"a\", newline=\"\") as csvfile:\n",
    "            writer = csv.writer(csvfile)\n",
    "            writer.writerow(\n",
    "                [\n",
    "                    ds_config,\n",
    "                    \"naive\",\n",
    "                    res[\"MSE[mean]\"][0],\n",
    "                    res[\"MSE[0.5]\"][0],\n",
    "                    res[\"MAE[0.5]\"][0],\n",
    "                    res[\"MASE[0.5]\"][0],\n",
    "                    res[\"MAPE[0.5]\"][0],\n",
    "                    res[\"sMAPE[0.5]\"][0],\n",
    "                    res[\"MSIS\"][0],\n",
    "                    res[\"RMSE[mean]\"][0],\n",
    "                    res[\"NRMSE[mean]\"][0],\n",
    "                    res[\"ND[0.5]\"][0],\n",
    "                    res[\"mean_weighted_sum_quantile_loss\"][0],\n",
    "                    dataset_properties_map[ds_key][\"domain\"],\n",
    "                    dataset_properties_map[ds_key][\"num_variates\"],\n",
    "                ]\n",
    "            )\n",
    "\n",
    "        print(f\"Results for {ds_name} have been written to {csv_file_path}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Results\n",
    "\n",
    "Running the above cell will generate a csv file called `all_results.csv` under the `results/naive` folder containing the results for the Naive model on the gift-eval benchmark. The csv file will look like this:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>model</th>\n",
       "      <th>eval_metrics/MSE[mean]</th>\n",
       "      <th>eval_metrics/MSE[0.5]</th>\n",
       "      <th>eval_metrics/MAE[0.5]</th>\n",
       "      <th>eval_metrics/MASE[0.5]</th>\n",
       "      <th>eval_metrics/MAPE[0.5]</th>\n",
       "      <th>eval_metrics/sMAPE[0.5]</th>\n",
       "      <th>eval_metrics/MSIS</th>\n",
       "      <th>eval_metrics/RMSE[mean]</th>\n",
       "      <th>eval_metrics/NRMSE[mean]</th>\n",
       "      <th>eval_metrics/ND[0.5]</th>\n",
       "      <th>eval_metrics/mean_weighted_sum_quantile_loss</th>\n",
       "      <th>domain</th>\n",
       "      <th>num_variates</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>m4_weekly/W/short</td>\n",
       "      <td>naive</td>\n",
       "      <td>453525.145918</td>\n",
       "      <td>453525.145918</td>\n",
       "      <td>347.991483</td>\n",
       "      <td>2.777295</td>\n",
       "      <td>0.089373</td>\n",
       "      <td>0.091613</td>\n",
       "      <td>26.631225</td>\n",
       "      <td>673.442756</td>\n",
       "      <td>0.122691</td>\n",
       "      <td>0.063399</td>\n",
       "      <td>0.060870</td>\n",
       "      <td>Econ/Fin</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>bizitobs_l2c/H/short</td>\n",
       "      <td>naive</td>\n",
       "      <td>263.305494</td>\n",
       "      <td>263.305494</td>\n",
       "      <td>11.038768</td>\n",
       "      <td>1.086094</td>\n",
       "      <td>1.288994</td>\n",
       "      <td>1.012692</td>\n",
       "      <td>9.958253</td>\n",
       "      <td>16.226691</td>\n",
       "      <td>0.874660</td>\n",
       "      <td>0.595018</td>\n",
       "      <td>0.486325</td>\n",
       "      <td>Web/CloudOps</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bizitobs_l2c/H/medium</td>\n",
       "      <td>naive</td>\n",
       "      <td>538.646763</td>\n",
       "      <td>538.646763</td>\n",
       "      <td>15.811579</td>\n",
       "      <td>1.485090</td>\n",
       "      <td>1.147788</td>\n",
       "      <td>1.761468</td>\n",
       "      <td>15.525791</td>\n",
       "      <td>23.208765</td>\n",
       "      <td>1.405326</td>\n",
       "      <td>0.957415</td>\n",
       "      <td>0.863952</td>\n",
       "      <td>Web/CloudOps</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>bizitobs_l2c/H/long</td>\n",
       "      <td>naive</td>\n",
       "      <td>326.644048</td>\n",
       "      <td>326.644048</td>\n",
       "      <td>13.937010</td>\n",
       "      <td>1.427895</td>\n",
       "      <td>2.597595</td>\n",
       "      <td>0.906076</td>\n",
       "      <td>17.133984</td>\n",
       "      <td>18.073297</td>\n",
       "      <td>1.103969</td>\n",
       "      <td>0.851313</td>\n",
       "      <td>0.817819</td>\n",
       "      <td>Web/CloudOps</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 dataset  model  eval_metrics/MSE[mean]   \n",
       "0      m4_weekly/W/short  naive           453525.145918  \\\n",
       "1   bizitobs_l2c/H/short  naive              263.305494   \n",
       "2  bizitobs_l2c/H/medium  naive              538.646763   \n",
       "3    bizitobs_l2c/H/long  naive              326.644048   \n",
       "\n",
       "   eval_metrics/MSE[0.5]  eval_metrics/MAE[0.5]  eval_metrics/MASE[0.5]   \n",
       "0          453525.145918             347.991483                2.777295  \\\n",
       "1             263.305494              11.038768                1.086094   \n",
       "2             538.646763              15.811579                1.485090   \n",
       "3             326.644048              13.937010                1.427895   \n",
       "\n",
       "   eval_metrics/MAPE[0.5]  eval_metrics/sMAPE[0.5]  eval_metrics/MSIS   \n",
       "0                0.089373                 0.091613          26.631225  \\\n",
       "1                1.288994                 1.012692           9.958253   \n",
       "2                1.147788                 1.761468          15.525791   \n",
       "3                2.597595                 0.906076          17.133984   \n",
       "\n",
       "   eval_metrics/RMSE[mean]  eval_metrics/NRMSE[mean]  eval_metrics/ND[0.5]   \n",
       "0               673.442756                  0.122691              0.063399  \\\n",
       "1                16.226691                  0.874660              0.595018   \n",
       "2                23.208765                  1.405326              0.957415   \n",
       "3                18.073297                  1.103969              0.851313   \n",
       "\n",
       "   eval_metrics/mean_weighted_sum_quantile_loss        domain  num_variates  \n",
       "0                                      0.060870      Econ/Fin             1  \n",
       "1                                      0.486325  Web/CloudOps             7  \n",
       "2                                      0.863952  Web/CloudOps             7  \n",
       "3                                      0.817819  Web/CloudOps             7  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv(\"../results/naive/all_results.csv\")\n",
    "df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "bench_oss",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}