chore: pin binder version for users (#1607)

* chore: pin binder compatible version * fix dbfs/databricks case escapes in notebooks Co-authored-by: Puneet Pruthi <[email protected]>
microsoft · Aug 10, 2022 · 93d7ccf · 93d7ccf
1 parent c7a61ec
commit 93d7ccf
Show file tree

Hide file tree

Showing 9 changed files with 45 additions and 16 deletions.
diff --git a/notebooks/features/onnx/ONNX - Inference on Spark.ipynb b/notebooks/features/onnx/ONNX - Inference on Spark.ipynb
@@ -119,6 +119,7 @@
     "from synapse.ml.core.platform import running_on_binder\n",
     "\n",
     "if running_on_binder():\n",
+    "    !pip install lightgbm==3.2.1\n",
     "    from IPython import get_ipython\n",
     "import lightgbm as lgb\n",
     "from lightgbm import Booster, LGBMClassifier\n",

diff --git a/notebooks/features/other/CyberML - Anomalous Access Detection.ipynb b/notebooks/features/other/CyberML - Anomalous Access Detection.ipynb
@@ -87,7 +87,12 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "spark.sparkContext.setCheckpointDir(\"dbfs:/checkpoint_path/\")\n",
+    "from synapse.ml.core.platform import running_on_databricks\n",
+    "\n",
+    "if running_on_databricks():\n",
+    "    spark.sparkContext.setCheckpointDir(\"dbfs:/checkpoint_path/\")\n",
+    "else:\n",
+    "    spark.sparkContext.setCheckpointDir(\"/tmp/checkpoint_path/\")\n",
     "\n",
     "factory = DataFactory(\n",
     "    num_hr_users=25,\n",
@@ -445,7 +450,12 @@
     "\n",
     "p = plot(fig, output_type=\"div\")\n",
     "\n",
-    "displayHTML(p)"
+    "if running_on_databricks():\n",
+    "    displayHTML(p)\n",
+    "else:\n",
+    "    import IPython\n",
+    "\n",
+    "    IPython.display.HTML(p)"
    ]
   },
   {

diff --git a/notebooks/features/other/DeepLearning - BiLSTM Medical Entity Extraction.ipynb b/notebooks/features/other/DeepLearning - BiLSTM Medical Entity Extraction.ipynb
@@ -45,7 +45,7 @@
     "# Bootstrap Spark Session\n",
     "spark = SparkSession.builder.getOrCreate()\n",
     "\n",
-    "from synapse.ml.core.platform import running_on_synapse\n",
+    "from synapse.ml.core.platform import running_on_synapse, running_on_databricks\n",
     "\n",
     "if running_on_synapse():\n",
     "    from notebookutils.visualization import display"
@@ -71,9 +71,12 @@
     "if running_on_synapse():\n",
     "    modelDir = \"abfss://[email protected]/models/\"\n",
     "    dataDir = \"./nltkdata\"\n",
-    "else:\n",
+    "elif running_on_databricks():\n",
     "    modelDir = \"dbfs:/models/\"\n",
     "    dataDir = \"/dbfs/nltkdata\"\n",
+    "else:\n",
+    "    modelDir = \"/tmp/models/\"\n",
+    "    dataDir = \"/tmp/nltkdata\"\n",
     "\n",
     "d = ModelDownloader(spark, modelDir)\n",
     "modelSchema = d.downloadByName(modelName)\n",

diff --git a/notebooks/features/other/DeepLearning - CIFAR10 Convolutional Network.ipynb b/notebooks/features/other/DeepLearning - CIFAR10 Convolutional Network.ipynb
@@ -23,7 +23,7 @@
     "# Bootstrap Spark Session\n",
     "spark = SparkSession.builder.getOrCreate()\n",
     "\n",
-    "from synapse.ml.core.platform import running_on_synapse\n",
+    "from synapse.ml.core.platform import running_on_synapse, running_on_databricks\n",
     "\n",
     "if running_on_synapse():\n",
     "    from notebookutils.visualization import display"
@@ -63,8 +63,10 @@
     "modelName = \"ConvNet\"\n",
     "if running_on_synapse():\n",
     "    modelDir = \"abfss://[email protected]/models/\"\n",
+    "elif running_on_databricks():\n",
+    "    modelDir = \"dbfs:/models/\"\n",
     "else:\n",
-    "    modelDir = \"dbfs:/models/\""
+    "    modelDir = \"/tmp/models/\""
    ]
   },
   {

diff --git a/notebooks/features/other/DeepLearning - Flower Image Classification.ipynb b/notebooks/features/other/DeepLearning - Flower Image Classification.ipynb
@@ -22,7 +22,7 @@
     "# Bootstrap Spark Session\n",
     "spark = SparkSession.builder.getOrCreate()\n",
     "\n",
-    "from synapse.ml.core.platform import running_on_synapse\n",
+    "from synapse.ml.core.platform import running_on_synapse, running_on_databricks\n",
     "\n",
     "if running_on_synapse():\n",
     "    from notebookutils.visualization import display"
@@ -38,8 +38,10 @@
    "source": [
     "if running_on_synapse():\n",
     "    modelDir = \"abfss://[email protected]/models/\"\n",
-    "else:\n",
+    "elif running_on_databricks():\n",
     "    modelDir = \"dbfs:/models/\"\n",
+    "else:\n",
+    "    modelDir = \"/tmp/models/\"\n",
     "\n",
     "model = ModelDownloader(spark, modelDir).downloadByName(\"ResNet50\")"
    ]

diff --git a/notebooks/features/other/DeepLearning - Transfer Learning.ipynb b/notebooks/features/other/DeepLearning - Transfer Learning.ipynb
@@ -35,13 +35,15 @@
     "# Bootstrap Spark Session\n",
     "spark = SparkSession.builder.getOrCreate()\n",
     "\n",
-    "from synapse.ml.core.platform import running_on_synapse\n",
+    "from synapse.ml.core.platform import running_on_synapse, running_on_databricks\n",
     "\n",
     "modelName = \"ConvNet\"\n",
     "if running_on_synapse():\n",
     "    modelDir = \"abfss://[email protected]/models/\"\n",
-    "else:\n",
+    "elif running_on_databricks():\n",
     "    modelDir = \"dbfs:/models/\"\n",
+    "else:\n",
+    "    modelDir = \"/tmp/models/\"\n",
     "\n",
     "d = ModelDownloader(spark, modelDir)\n",
     "model = d.downloadByName(modelName)\n",

diff --git a/notebooks/features/regression/Regression - Flight Delays.ipynb b/notebooks/features/regression/Regression - Flight Delays.ipynb
@@ -125,12 +125,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from synapse.ml.core.platform import running_on_synapse\n",
+    "from synapse.ml.core.platform import *\n",
     "\n",
     "if running_on_synapse():\n",
     "    model_name = \"/models/flightDelayModel.mml\"\n",
-    "else:\n",
+    "elif running_on_databricks():\n",
     "    model_name = \"dbfs:/flightDelayModel.mml\"\n",
+    "else:\n",
+    "    model_name = \"/tmp/flightDelayModel.mml\"\n",
     "\n",
     "model.write().overwrite().save(model_name)\n",
     "flightDelayModel = TrainedRegressorModel.load(model_name)\n",

diff --git a/notebooks/features/responsible_ai/Interpretability - Snow Leopard Detection.ipynb b/notebooks/features/responsible_ai/Interpretability - Snow Leopard Detection.ipynb
@@ -15,7 +15,7 @@
    "source": [
     "import os\n",
     "from pyspark.sql import SparkSession\n",
-    "from synapse.ml.core.platform import running_on_synapse, find_secret\n",
+    "from synapse.ml.core.platform import *\n",
     "\n",
     "# Bootstrap Spark Session\n",
     "spark = SparkSession.builder.getOrCreate()\n",
@@ -133,7 +133,12 @@
     "        header, tableHTML\n",
     "    )\n",
     "    try:\n",
-    "        displayHTML(style + body)\n",
+    "        if running_on_databricks():\n",
+    "            displayHTML(style + body)\n",
+    "        else:\n",
+    "            import IPython\n",
+    "\n",
+    "            IPython.display.HTML(style + body)\n",
     "    except:\n",
     "        pass"
    ],
@@ -232,8 +237,10 @@
     "    network = ModelDownloader(\n",
     "        spark, \"abfss://[email protected]/models/\"\n",
     "    ).downloadByName(\"ResNet50\")\n",
-    "else:\n",
+    "elif running_on_databricks():\n",
     "    network = ModelDownloader(spark, \"dbfs:/Models/\").downloadByName(\"ResNet50\")\n",
+    "else:\n",
+    "    network = ModelDownloader(spark, \"/tmp/Models/\").downloadByName(\"ResNet50\")\n",
     "\n",
     "model = Pipeline(\n",
     "    stages=[\n",

diff --git a/start b/start
@@ -3,7 +3,7 @@
 export OPENMPI_VERSION="3.1.2"
 export SPARK_VERSION="3.2.2"
 export HADOOP_VERSION="2.7"
-export MMLSPARK_VERSION="0.10.0"
+export SYNAPSEML_VERSION="0.10.0-26-c7a61ecd-SNAPSHOT" # Binder compatibility version
 
 echo "Beginning Spark Session..."
 exec "$@"