From 1f6aa504570297a6b872cf313ac7cc89bf3634a5 Mon Sep 17 00:00:00 2001 From: Xinyue Ruan Date: Tue, 21 Jun 2022 12:18:11 +0800 Subject: [PATCH 01/21] fix: update library to use netstandard2.1 as target framework --- core/src/main/dotnet/src/dotnetBase.csproj | 2 +- core/src/main/dotnet/test/dotnetTestBase.csproj | 2 +- .../com/microsoft/azure/synapse/ml/codegen/DotnetCodegen.scala | 2 +- .../com/microsoft/azure/synapse/ml/codegen/DotnetTestGen.scala | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/dotnet/src/dotnetBase.csproj b/core/src/main/dotnet/src/dotnetBase.csproj index b783a9b7c1..18d3460091 100644 --- a/core/src/main/dotnet/src/dotnetBase.csproj +++ b/core/src/main/dotnet/src/dotnetBase.csproj @@ -1,7 +1,7 @@ - net5.0 + netstandard2.1 9.0 SynapseML.DotnetBase true diff --git a/core/src/main/dotnet/test/dotnetTestBase.csproj b/core/src/main/dotnet/test/dotnetTestBase.csproj index 94690c43c2..73c5e7e070 100644 --- a/core/src/main/dotnet/test/dotnetTestBase.csproj +++ b/core/src/main/dotnet/test/dotnetTestBase.csproj @@ -1,7 +1,7 @@ - net5.0 + netstandard2.1 9.0 SynapseML.DotnetE2ETest true diff --git a/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/DotnetCodegen.scala b/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/DotnetCodegen.scala index 676226d5a6..fe8a73f220 100644 --- a/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/DotnetCodegen.scala +++ b/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/DotnetCodegen.scala @@ -35,7 +35,7 @@ object DotnetCodegen { s""" | | - | net5.0 + | netstandard2.1 | 9.0 | SynapseML.$curProject | true diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/codegen/DotnetTestGen.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/codegen/DotnetTestGen.scala index d587b047cc..e8c3622770 100644 --- a/core/src/test/scala/com/microsoft/azure/synapse/ml/codegen/DotnetTestGen.scala +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/codegen/DotnetTestGen.scala @@ -73,7 +73,7 @@ object DotnetTestGen { s""" | | - | net5.0 + | netstandard2.1 | 9.0 | SynapseML.$curProject.Test | From 1612473e2972462ae456e85b1540882288bc0d42 Mon Sep 17 00:00:00 2001 From: Xinyue Ruan Date: Tue, 21 Jun 2022 16:22:16 +0800 Subject: [PATCH 02/21] fix target framework and refactor some tasks --- build.sbt | 22 +++++++------------ .../src/Internal/Dotnet/CompilerServices.cs | 8 +++++++ .../apache/spark/ml/feature/StringIndexer.cs | 1 - .../synapse/ml/codegen/DotnetTestGen.scala | 2 +- project/CodegenPlugin.scala | 9 ++------ project/build.scala | 11 ++++++++++ 6 files changed, 30 insertions(+), 23 deletions(-) create mode 100644 core/src/main/dotnet/src/Internal/Dotnet/CompilerServices.cs diff --git a/build.sbt b/build.sbt index 5b4feb357c..d4fc63e89a 100644 --- a/build.sbt +++ b/build.sbt @@ -99,6 +99,12 @@ genBuildInfo := { FileUtils.writeStringToFile(infoFile, buildInfo, "utf-8") } +val rootGenDir = SettingKey[File]("rootGenDir") +rootGenDir := { + val targetDir = (root / Compile / packageBin / artifactPath).value.getParentFile + join(targetDir, "generated") +} + // scalastyle:off line.size.limit val genSleetConfig = TaskKey[Unit]("genSleetConfig", "generate sleet.json file for sleet configuration so we can push nuget package to the blob") @@ -148,21 +154,9 @@ publishDotnetTestBase := { val dotnetHelperFile = join(dotnetTestBaseDir, "SynapseMLVersion.cs") if (dotnetHelperFile.exists()) FileUtils.forceDelete(dotnetHelperFile) FileUtils.writeStringToFile(dotnetHelperFile, fileContent, "utf-8") - runCmd( - Seq("dotnet", "pack", "--output", join(dotnetTestBaseDir, "target").getAbsolutePath), - dotnetTestBaseDir - ) + packDotnetAssemblyCmd(join(dotnetTestBaseDir, "target").getAbsolutePath, dotnetTestBaseDir) val packagePath = join(dotnetTestBaseDir, "target", s"SynapseML.DotnetE2ETest.0.9.1.nupkg").getAbsolutePath - runCmd( - Seq("sleet", "push", packagePath, "--config", join(rootGenDir.value, "sleet.json").getAbsolutePath, - "--source", "SynapseMLNuget", "--force") - ) -} - -val rootGenDir = SettingKey[File]("rootGenDir") -rootGenDir := { - val targetDir = (root / Compile / packageBin / artifactPath).value.getParentFile - join(targetDir, "generated") + publishDotnetAssemblyCmd(packagePath, rootGenDir.value) } def runTaskForAllInCompile(task: TaskKey[Unit]): Def.Initialize[Task[Seq[Unit]]] = { diff --git a/core/src/main/dotnet/src/Internal/Dotnet/CompilerServices.cs b/core/src/main/dotnet/src/Internal/Dotnet/CompilerServices.cs new file mode 100644 index 0000000000..2e00473c25 --- /dev/null +++ b/core/src/main/dotnet/src/Internal/Dotnet/CompilerServices.cs @@ -0,0 +1,8 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace System.Runtime.CompilerServices +{ + internal static class IsExternalInit {} +} diff --git a/core/src/main/dotnet/src/org/apache/spark/ml/feature/StringIndexer.cs b/core/src/main/dotnet/src/org/apache/spark/ml/feature/StringIndexer.cs index 919f10896f..4b1f0830d3 100644 --- a/core/src/main/dotnet/src/org/apache/spark/ml/feature/StringIndexer.cs +++ b/core/src/main/dotnet/src/org/apache/spark/ml/feature/StringIndexer.cs @@ -16,7 +16,6 @@ using Microsoft.Spark.Utils; using SynapseML.Dotnet.Utils; using Synapse.ML.LightGBM.Param; -using Microsoft.Spark.ML.Feature; namespace Microsoft.Spark.ML.Feature { diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/codegen/DotnetTestGen.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/codegen/DotnetTestGen.scala index e8c3622770..2780cc8ff0 100644 --- a/core/src/test/scala/com/microsoft/azure/synapse/ml/codegen/DotnetTestGen.scala +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/codegen/DotnetTestGen.scala @@ -73,7 +73,7 @@ object DotnetTestGen { s""" | | - | netstandard2.1 + | netcoreapp3.1 | 9.0 | SynapseML.$curProject.Test | diff --git a/project/CodegenPlugin.scala b/project/CodegenPlugin.scala index 1328da0db4..c8b0d93c49 100644 --- a/project/CodegenPlugin.scala +++ b/project/CodegenPlugin.scala @@ -298,19 +298,14 @@ object CodegenPlugin extends AutoPlugin { val sourceDotnetDir = join(dotnetSrcDir.getAbsolutePath, genPackageNamespace.value) FileUtils.copyDirectory(sourceDotnetDir, destDotnetDir) val packageDir = join(codegenDir.value, "package", "dotnet").absolutePath - runCmd( - Seq("dotnet", "pack", "--output", packageDir), - join(dotnetSrcDir, "synapse", "ml")) + packDotnetAssemblyCmd(packageDir, join(dotnetSrcDir, "synapse", "ml")) }, publishDotnet := { packageDotnet.value val dotnetPackageName = name.value.split("-").drop(1).map(s => s.capitalize).mkString("") val packagePath = join(codegenDir.value, "package", "dotnet", s"SynapseML.$dotnetPackageName.${dotnetVersion.value}.nupkg").absolutePath - val sleetConfigFile = join(mergePyCodeDir.value, "sleet.json").getAbsolutePath - runCmd( - Seq("sleet", "push", packagePath, "--config", sleetConfigFile, "--source", "SynapseMLNuget", "--force") - ) + publishDotnetAssemblyCmd(packagePath, mergePyCodeDir.value) }, targetDir := { (Compile / packageBin / artifactPath).value.getParentFile diff --git a/project/build.scala b/project/build.scala index 833627f869..48cfe5bdb0 100644 --- a/project/build.scala +++ b/project/build.scala @@ -63,6 +63,17 @@ object BuildUtils { workDir) } + def packDotnetAssemblyCmd(outputDir: String, + workDir: File): Unit = + runCmd(Seq("dotnet", "pack", "--output", outputDir), workDir) + + def publishDotnetAssemblyCmd(packagePath: String, + sleetConfigDir: File): Unit = + runCmd( + Seq("sleet", "push", packagePath, "--config", join(sleetConfigDir, "sleet.json").getAbsolutePath, + "--source", "SynapseMLNuget", "--force") + ) + def uploadToBlob(source: String, dest: String, container: String, From 360eb21e6167f9b8bcc2f266c6883bc8c7b6d3a7 Mon Sep 17 00:00:00 2001 From: Xinyue Ruan Date: Thu, 30 Jun 2022 17:30:00 +0800 Subject: [PATCH 03/21] update some documentation --- .../ml/cognitive/CognitiveServiceBase.scala | 16 ++++++++++++++-- .../synapse/ml/cognitive/FormRecognizer.scala | 2 +- .../synapse/ml/cognitive/TextTranslator.scala | 9 +++++---- .../azure/synapse/ml/codegen/DotnetCodegen.scala | 2 +- .../synapse/ml/codegen/DotnetWrappable.scala | 15 +++++++++++---- .../synapse/ml/param/JsonEncodableParam.scala | 11 +++++++---- .../ml/recommendation/RecommendationHelper.scala | 2 +- 7 files changed, 40 insertions(+), 17 deletions(-) diff --git a/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/CognitiveServiceBase.scala b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/CognitiveServiceBase.scala index 670b6fa656..41d7c6f688 100644 --- a/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/CognitiveServiceBase.scala +++ b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/CognitiveServiceBase.scala @@ -239,7 +239,13 @@ trait HasSetLinkedService extends Wrappable with HasURL with HasSubscriptionKey } override def dotnetAdditionalMethods: String = super.dotnetAdditionalMethods + { - s""" + s"""/// + |/// Sets value for linkedService + |/// + |/// + |/// linkedService name + |/// + |/// New $dotnetClassName object |public $dotnetClassName SetLinkedService(string value) => | $dotnetClassWrapperName(Reference.Invoke(\"setLinkedService\", value)); |""".stripMargin @@ -280,7 +286,13 @@ trait HasSetLocation extends Wrappable with HasURL with HasUrlPath { } override def dotnetAdditionalMethods: String = super.dotnetAdditionalMethods + { - s""" + s"""/// + |/// Sets value for location + |/// + |/// + |/// Location of the cognitive service + |/// + |/// New $dotnetClassName object |public $dotnetClassName SetLocation(string value) => | $dotnetClassWrapperName(Reference.Invoke(\"setLocation\", value)); |""".stripMargin diff --git a/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala index db627682ad..f345057591 100644 --- a/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala +++ b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala @@ -37,7 +37,7 @@ trait HasPages extends HasServiceParams { val pages = new ServiceParam[String](this, "pages", "The page selection only leveraged for" + " multi-page PDF and TIFF documents. Accepted input include single pages (e.g.'1, 2' -> pages 1 and 2 will be " + "processed), finite (e.g. '2-5' -> pages 2 to 5 will be processed) and open-ended ranges (e.g. '5-' -> all the" + - " pages from page 5 will be processed & e.g. '-10' -> pages 1 to 10 will be processed). All of these can be mixed" + + " pages from page 5 will be processed; e.g. '-10' -> pages 1 to 10 will be processed). All of these can be mixed" + " together and ranges are allowed to overlap (eg. '-5, 1, 3, 5-10' - pages 1 to 10 will be processed). The" + " service will accept the request if it can process at least one page of the document (e.g. using '5-100' on a " + "5 page document is a valid input where page 5 will be processed). If no page range is provided, the entire" + diff --git a/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala index 3bf3647fe2..c848f79a7b 100644 --- a/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala +++ b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala @@ -256,10 +256,11 @@ class Translate(override val uid: String) extends TextTranslatorBase(uid) override protected def getInternalTransformer(schema: StructType): PipelineModel = customGetInternalTransformer(schema, Seq("text", "toLanguage")) - val toLanguage = new ServiceParam[Seq[String]](this, "toLanguage", "Specifies the language of the output" + - " text. The target language must be one of the supported languages included in the translation scope." + - " For example, use to=de to translate to German. It's possible to translate to multiple languages simultaneously" + - " by repeating the parameter in the query string. For example, use to=de&to=it to translate to German and Italian.", + val toLanguage = new ServiceParam[Seq[String]](this, "toLanguage", + "Specifies the language of the output text. The target language must be one of the supported languages" + + " included in the translation scope. For example, use to=de to translate to German. It's possible to translate" + + " to multiple languages simultaneously by repeating the parameter in the query string. For example, use " + + "to=de and to=it to translate to German and Italian.", isRequired = true, isURLParam = true, toValueString = { seq => seq.mkString(",") }) diff --git a/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/DotnetCodegen.scala b/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/DotnetCodegen.scala index 7d8f3d0590..fc10c61b84 100644 --- a/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/DotnetCodegen.scala +++ b/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/DotnetCodegen.scala @@ -39,7 +39,7 @@ object DotnetCodegen { | 9.0 | SynapseML.$curProject | true - | + | true | .NET for SynapseML.$curProject | ${conf.dotnetVersion} | diff --git a/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/DotnetWrappable.scala b/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/DotnetWrappable.scala index 2604f4721f..1cc16958b3 100644 --- a/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/DotnetWrappable.scala +++ b/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/DotnetWrappable.scala @@ -4,7 +4,7 @@ package com.microsoft.azure.synapse.ml.codegen import com.microsoft.azure.synapse.ml.core.env.FileUtilities -import com.microsoft.azure.synapse.ml.param.WrappableParam +import com.microsoft.azure.synapse.ml.param.{ServiceParam, WrappableParam} import org.apache.commons.lang.StringUtils.capitalize import org.apache.spark.ml._ import org.apache.spark.ml.evaluation.Evaluator @@ -120,14 +120,21 @@ trait DotnetWrappable extends BaseWrappable { val capName = p.name.capitalize val docString = s"""|/// - |/// Sets ${p.name} value for + |/// Sets value for ${p.name} |/// - |/// + |/// |/// ${p.doc} |/// |/// New $dotnetClassName object """.stripMargin p match { // TODO: Fix UDF & UDPyF confusion; ParamSpaceParam, BallTreeParam, ConditionalBallTreeParam type + case sp: ServiceParam[_] => + s"""|$docString + |${sp.dotnetSetter(dotnetClassName, capName, dotnetClassWrapperName)} + | + |${docString.replaceFirst(sp.name, s"${sp.name} column")} + |${sp.dotnetSetterForSrvParamCol(dotnetClassName, capName, dotnetClassWrapperName)} + |""".stripMargin case wp: WrappableParam[_] => s"""|$docString |${wp.dotnetSetter(dotnetClassName, capName, dotnetClassWrapperName)} @@ -148,7 +155,7 @@ trait DotnetWrappable extends BaseWrappable { val capName = p.name.capitalize val docString = s"""|/// - |/// Gets ${p.name} value for + |/// Gets ${p.name} value |/// |/// |/// ${p.name}: ${p.doc} diff --git a/core/src/main/scala/com/microsoft/azure/synapse/ml/param/JsonEncodableParam.scala b/core/src/main/scala/com/microsoft/azure/synapse/ml/param/JsonEncodableParam.scala index 9dabe1a57c..f4e18e5e9c 100644 --- a/core/src/main/scala/com/microsoft/azure/synapse/ml/param/JsonEncodableParam.scala +++ b/core/src/main/scala/com/microsoft/azure/synapse/ml/param/JsonEncodableParam.scala @@ -130,14 +130,17 @@ class ServiceParam[T: TypeTag](parent: Params, override private[ml] def dotnetSetter(dotnetClassName: String, capName: String, - dotnetClassWrapperName: String): String = { + dotnetClassWrapperName: String): String = s"""|public $dotnetClassName Set$capName($dotnetType value) => | $dotnetClassWrapperName(Reference.Invoke(\"set$capName\", (object)value)); - | - |public $dotnetClassName Set${capName}Col(string value) => + |""".stripMargin + + private[ml] def dotnetSetterForSrvParamCol(dotnetClassName: String, + capName: String, + dotnetClassWrapperName: String): String = + s"""|public $dotnetClassName Set${capName}Col(string value) => | $dotnetClassWrapperName(Reference.Invoke(\"set${capName}Col\", value)); |""".stripMargin - } override private[ml] def dotnetGetter(capName: String): String = { dotnetType match { diff --git a/core/src/main/scala/org/apache/spark/ml/recommendation/RecommendationHelper.scala b/core/src/main/scala/org/apache/spark/ml/recommendation/RecommendationHelper.scala index ae2879f6d6..1acbf92ffd 100644 --- a/core/src/main/scala/org/apache/spark/ml/recommendation/RecommendationHelper.scala +++ b/core/src/main/scala/org/apache/spark/ml/recommendation/RecommendationHelper.scala @@ -136,7 +136,7 @@ trait RankingTrainValidationSplitParams extends Wrappable with HasSeed { * @group param */ val trainRatio: DoubleParam = new DoubleParam(this, "trainRatio", - "ratio between training set and validation set (>= 0 && <= 1)", ParamValidators.inRange(0, 1)) + "ratio between training set and validation set (>= 0 and <= 1)", ParamValidators.inRange(0, 1)) /** @group getParam */ def getTrainRatio: Double = $(trainRatio) From b4ec6a5c1405f8be6ecf8f6320d0c6752f46d0d7 Mon Sep 17 00:00:00 2001 From: Xinyue Ruan Date: Thu, 30 Jun 2022 18:13:12 +0800 Subject: [PATCH 04/21] add documentation generation for dotnet --- build.sbt | 6 +++++- pipeline.yaml | 1 + project/CodegenPlugin.scala | 16 +++++++++++++++- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index d4fc63e89a..3a81255aeb 100644 --- a/build.sbt +++ b/build.sbt @@ -244,10 +244,11 @@ publishPypi := { ) } -val publishDocs = TaskKey[Unit]("publishDocs", "publish docs for scala and python") +val publishDocs = TaskKey[Unit]("publishDocs", "publish docs for scala, python and dotnet") publishDocs := { generatePythonDoc.value (root / Compile / unidoc).value + generateDotnetDoc.value val html = """ |
@@ -262,6 +263,9 @@ publishDocs := {
   if (scalaDir.exists()) FileUtils.forceDelete(scalaDir)
   FileUtils.copyDirectory(join(targetDir, "unidoc"), scalaDir)
   FileUtils.writeStringToFile(join(unifiedDocDir.toString, "index.html"), html, "utf-8")
+  val dotnetDir = join(unifiedDocDir.toString, "dotnet")
+  if (dotnetDir.exists()) FileUtils.forceDelete(dotnetDir)
+  FileUtils.copyDirectory(join(codegenDir, "src", "dotnet", "html"), dotnetDir)
   uploadToBlob(unifiedDocDir.toString, version.value, "docs")
 }
 
diff --git a/pipeline.yaml b/pipeline.yaml
index 74c0c4ba4b..5a9a4082a3 100644
--- a/pipeline.yaml
+++ b/pipeline.yaml
@@ -62,6 +62,7 @@ jobs:
     - template: templates/kv.yml
     - bash: |
         set -e
+        sudo apt-get install doxygen
         source activate synapseml
         sbt packagePython
         sbt publishBlob publishDocs publishR publishPython
diff --git a/project/CodegenPlugin.scala b/project/CodegenPlugin.scala
index ad6f2a3d91..9eac0b5dc0 100644
--- a/project/CodegenPlugin.scala
+++ b/project/CodegenPlugin.scala
@@ -76,6 +76,8 @@ object CodegenPlugin extends AutoPlugin {
     val packageDotnet = TaskKey[Unit]("packageDotnet", "Generate dotnet nuget package")
     val publishDotnet = TaskKey[Unit]("publishDotnet", "publish dotnet nuget package")
     val testDotnet = TaskKey[Unit]("testDotnet", "test dotnet nuget package")
+    val generateDotnetDoc = TaskKey[Unit]("generateDotnetDoc",
+      "Generate documentation for dotnet classes")
 
     val mergePyCodeDir = SettingKey[File]("mergePyCodeDir")
     val mergePyCode = TaskKey[Unit]("mergePyCode", "copy python code to a destination")
@@ -218,7 +220,7 @@ object CodegenPlugin extends AutoPlugin {
         val versionArray = version.value.split("-".toCharArray)
         versionArray.head + "-rc" + versionArray.drop(1).dropRight(1).mkString("")
       } else {
-       version.value
+        version.value
       }
     },
     packageR := {
@@ -307,6 +309,18 @@ object CodegenPlugin extends AutoPlugin {
         s"SynapseML.$dotnetPackageName.${dotnetVersion.value}.nupkg").absolutePath
       publishDotnetAssemblyCmd(packagePath, mergePyCodeDir.value)
     },
+    generateDotnetDoc := {
+      val dotnetSrcDir = join(codegenDir.value, "src", "dotnet")
+      runCmd(Seq("doxygen", "-g"), dotnetSrcDir)
+      FileUtils.copyFile(join(baseDirectory.value.getParent, "README.md"), join(dotnetSrcDir, "README.md"))
+      runCmd(Seq("sed", "-i", "\'s/img width=\"800\"/img width=\"300\"/g\'", "README.md"), dotnetSrcDir)
+      runCmd(Seq(
+        "echo", s"PROJECT_NAME = \"SynapseML Core\"\nPROJECT_NUMBER = \"${dotnetVersion.value}\"\n" +
+          "USE_MDFILE_AS_MAINPAGE = \"README.md\"\nRECURSIVE = YES",
+        "|", "tee", "-a", "Doxyfile"
+      ), dotnetSrcDir)
+      runCmd(Seq("doxygen"), dotnetSrcDir)
+    },
     targetDir := {
       (Compile / packageBin / artifactPath).value.getParentFile
     },

From d8b5493f916e76432b708c472542d81a6c9dbe47 Mon Sep 17 00:00:00 2001
From: Xinyue Ruan 
Date: Thu, 30 Jun 2022 18:16:13 +0800
Subject: [PATCH 05/21] fix pipeline

---
 pipeline.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pipeline.yaml b/pipeline.yaml
index 5a9a4082a3..10c6a787e6 100644
--- a/pipeline.yaml
+++ b/pipeline.yaml
@@ -64,7 +64,7 @@ jobs:
         set -e
         sudo apt-get install doxygen
         source activate synapseml
-        sbt packagePython
+        sbt packagePython packageDotnet
         sbt publishBlob publishDocs publishR publishPython
         sbt publishSigned
         sbt genBuildInfo

From 4b6e4ce540b540af8ff159e95697dd2f8748401a Mon Sep 17 00:00:00 2001
From: Xinyue Ruan 
Date: Thu, 30 Jun 2022 18:29:36 +0800
Subject: [PATCH 06/21] fix generateDotnetDoc

---
 project/CodegenPlugin.scala | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/project/CodegenPlugin.scala b/project/CodegenPlugin.scala
index 9eac0b5dc0..6adc3b7549 100644
--- a/project/CodegenPlugin.scala
+++ b/project/CodegenPlugin.scala
@@ -314,9 +314,13 @@ object CodegenPlugin extends AutoPlugin {
       runCmd(Seq("doxygen", "-g"), dotnetSrcDir)
       FileUtils.copyFile(join(baseDirectory.value.getParent, "README.md"), join(dotnetSrcDir, "README.md"))
       runCmd(Seq("sed", "-i", "\'s/img width=\"800\"/img width=\"300\"/g\'", "README.md"), dotnetSrcDir)
+      val packageName = name.value.split("-").map(_.capitalize).mkString(" ")
       runCmd(Seq(
-        "echo", s"PROJECT_NAME = \"SynapseML Core\"\nPROJECT_NUMBER = \"${dotnetVersion.value}\"\n" +
-          "USE_MDFILE_AS_MAINPAGE = \"README.md\"\nRECURSIVE = YES",
+        "echo",
+        s"""PROJECT_NAME = \"$packageName\"
+PROJECT_NUMBER = \"${dotnetVersion.value}\"
+USE_MDFILE_AS_MAINPAGE = \"README.md\"
+RECURSIVE = YES""".stripMargin,
         "|", "tee", "-a", "Doxyfile"
       ), dotnetSrcDir)
       runCmd(Seq("doxygen"), dotnetSrcDir)

From c330462254d570925a6ede7aedde24266dc4c728 Mon Sep 17 00:00:00 2001
From: Xinyue Ruan 
Date: Thu, 30 Jun 2022 18:59:41 +0800
Subject: [PATCH 07/21] fix publishDocs

---
 build.sbt                   | 20 ++++++++++++++++++++
 project/CodegenPlugin.scala | 32 ++++++++++----------------------
 2 files changed, 30 insertions(+), 22 deletions(-)

diff --git a/build.sbt b/build.sbt
index 3a81255aeb..1aa356f13d 100644
--- a/build.sbt
+++ b/build.sbt
@@ -177,6 +177,26 @@ generatePythonDoc := {
   runCmd(activateCondaEnv ++ Seq("sphinx-build", "-b", "html", "doc", "../../../doc/pyspark"), dir)
 }
 
+val generateDotnetDoc = TaskKey[Unit]("generateDotnetDoc", "Generate documentation for dotnet classes")
+generateDotnetDoc := {
+  runTaskForAllInCompile(packageDotnet).value
+  runTaskForAllInCompile(mergeDotnetCode).value
+  val dotnetSrcDir = join(rootGenDir.value, "src", "dotnet")
+  runCmd(Seq("doxygen", "-g"), dotnetSrcDir)
+  FileUtils.copyFile(join(baseDirectory.value, "README.md"), join(dotnetSrcDir, "README.md"))
+  runCmd(Seq("sed", "-i", "\'s/img width=\"800\"/img width=\"300\"/g\'", "README.md"), dotnetSrcDir)
+  val packageName = name.value.split("-").map(_.capitalize).mkString(" ")
+  runCmd(Seq(
+    "echo",
+    s"""PROJECT_NAME = \"$packageName\"
+PROJECT_NUMBER = \"${dotnetVersion.value}\"
+USE_MDFILE_AS_MAINPAGE = \"README.md\"
+RECURSIVE = YES""".stripMargin,
+    "|", "tee", "-a", "Doxyfile"
+  ), dotnetSrcDir)
+  runCmd(Seq("doxygen"), dotnetSrcDir)
+}
+
 val packageSynapseML = TaskKey[Unit]("packageSynapseML", "package all projects into SynapseML")
 packageSynapseML := {
   def writeSetupFileToTarget(dir: File): Unit = {
diff --git a/project/CodegenPlugin.scala b/project/CodegenPlugin.scala
index 6adc3b7549..34625dcb85 100644
--- a/project/CodegenPlugin.scala
+++ b/project/CodegenPlugin.scala
@@ -76,11 +76,10 @@ object CodegenPlugin extends AutoPlugin {
     val packageDotnet = TaskKey[Unit]("packageDotnet", "Generate dotnet nuget package")
     val publishDotnet = TaskKey[Unit]("publishDotnet", "publish dotnet nuget package")
     val testDotnet = TaskKey[Unit]("testDotnet", "test dotnet nuget package")
-    val generateDotnetDoc = TaskKey[Unit]("generateDotnetDoc",
-      "Generate documentation for dotnet classes")
 
-    val mergePyCodeDir = SettingKey[File]("mergePyCodeDir")
+    val mergeCodeDir = SettingKey[File]("mergeCodeDir")
     val mergePyCode = TaskKey[Unit]("mergePyCode", "copy python code to a destination")
+    val mergeDotnetCode = TaskKey[Unit]("mergeDotnetCode", "copy dotnet code to a destination")
   }
 
   import autoImport._
@@ -270,7 +269,12 @@ object CodegenPlugin extends AutoPlugin {
     },
     mergePyCode := {
       val srcDir = join(codegenDir.value, "src", "python", genPackageNamespace.value)
-      val destDir = join(mergePyCodeDir.value, "src", "python", genPackageNamespace.value)
+      val destDir = join(mergeCodeDir.value, "src", "python", genPackageNamespace.value)
+      FileUtils.copyDirectory(srcDir, destDir)
+    },
+    mergeDotnetCode := {
+      val srcDir = join(codegenDir.value, "src", "dotnet", genPackageNamespace.value)
+      val destDir = join(mergeCodeDir.value, "src", "dotnet", genPackageNamespace.value)
       FileUtils.copyDirectory(srcDir, destDir)
     },
     testPython := {
@@ -307,28 +311,12 @@ object CodegenPlugin extends AutoPlugin {
       val dotnetPackageName = name.value.split("-").drop(1).map(s => s.capitalize).mkString("")
       val packagePath = join(codegenDir.value, "package", "dotnet",
         s"SynapseML.$dotnetPackageName.${dotnetVersion.value}.nupkg").absolutePath
-      publishDotnetAssemblyCmd(packagePath, mergePyCodeDir.value)
-    },
-    generateDotnetDoc := {
-      val dotnetSrcDir = join(codegenDir.value, "src", "dotnet")
-      runCmd(Seq("doxygen", "-g"), dotnetSrcDir)
-      FileUtils.copyFile(join(baseDirectory.value.getParent, "README.md"), join(dotnetSrcDir, "README.md"))
-      runCmd(Seq("sed", "-i", "\'s/img width=\"800\"/img width=\"300\"/g\'", "README.md"), dotnetSrcDir)
-      val packageName = name.value.split("-").map(_.capitalize).mkString(" ")
-      runCmd(Seq(
-        "echo",
-        s"""PROJECT_NAME = \"$packageName\"
-PROJECT_NUMBER = \"${dotnetVersion.value}\"
-USE_MDFILE_AS_MAINPAGE = \"README.md\"
-RECURSIVE = YES""".stripMargin,
-        "|", "tee", "-a", "Doxyfile"
-      ), dotnetSrcDir)
-      runCmd(Seq("doxygen"), dotnetSrcDir)
+      publishDotnetAssemblyCmd(packagePath, mergeCodeDir.value)
     },
     targetDir := {
       (Compile / packageBin / artifactPath).value.getParentFile
     },
-    mergePyCodeDir := {
+    mergeCodeDir := {
       join(baseDirectory.value.getParent, "target", "scala-2.12", "generated")
     },
     codegenDir := {

From 81d947656b721537cbd3e882e80917ea9473328a Mon Sep 17 00:00:00 2001
From: Xinyue Ruan 
Date: Thu, 30 Jun 2022 19:07:37 +0800
Subject: [PATCH 08/21] fix dotnet version in generateDotnetDoc

---
 build.sbt           |  2 +-
 project/build.scala | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/build.sbt b/build.sbt
index 1aa356f13d..ea2499bbb4 100644
--- a/build.sbt
+++ b/build.sbt
@@ -189,7 +189,7 @@ generateDotnetDoc := {
   runCmd(Seq(
     "echo",
     s"""PROJECT_NAME = \"$packageName\"
-PROJECT_NUMBER = \"${dotnetVersion.value}\"
+PROJECT_NUMBER = \"${dotnetedVersion(version.value)}\"
 USE_MDFILE_AS_MAINPAGE = \"README.md\"
 RECURSIVE = YES""".stripMargin,
     "|", "tee", "-a", "Doxyfile"
diff --git a/project/build.scala b/project/build.scala
index 635a643f57..5009931af0 100644
--- a/project/build.scala
+++ b/project/build.scala
@@ -30,6 +30,16 @@ object BuildUtils {
     }
   }
 
+  def dotnetedVersion(version: String): String = {
+    version match {
+      case s if s.contains("-") => {
+        val versionArray = s.split("-".toCharArray)
+        versionArray.head + "-rc" + versionArray.drop(1).dropRight(1).mkString("")
+      }
+      case s => s
+    }
+  }
+
   def runCmd(cmd: Seq[String],
              wd: File = new File("."),
              envVars: Map[String, String] = Map()): Unit = {

From 612e300309ee0e114a0a76c3d65bfd10b65c7207 Mon Sep 17 00:00:00 2001
From: Xinyue Ruan 
Date: Thu, 30 Jun 2022 19:23:11 +0800
Subject: [PATCH 09/21] fix publishDoc

---
 build.sbt | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/build.sbt b/build.sbt
index ea2499bbb4..755577f1a8 100644
--- a/build.sbt
+++ b/build.sbt
@@ -179,8 +179,10 @@ generatePythonDoc := {
 
 val generateDotnetDoc = TaskKey[Unit]("generateDotnetDoc", "Generate documentation for dotnet classes")
 generateDotnetDoc := {
-  runTaskForAllInCompile(packageDotnet).value
-  runTaskForAllInCompile(mergeDotnetCode).value
+  Def.sequential(
+    runTaskForAllInCompile(dotnetCodeGen),
+    runTaskForAllInCompile(mergeDotnetCode)
+  ).value
   val dotnetSrcDir = join(rootGenDir.value, "src", "dotnet")
   runCmd(Seq("doxygen", "-g"), dotnetSrcDir)
   FileUtils.copyFile(join(baseDirectory.value, "README.md"), join(dotnetSrcDir, "README.md"))

From b4c856d64622d7783870987b767819720e6d2903 Mon Sep 17 00:00:00 2001
From: Xinyue Ruan 
Date: Thu, 30 Jun 2022 19:32:12 +0800
Subject: [PATCH 10/21] fix pipeline

---
 pipeline.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/pipeline.yaml b/pipeline.yaml
index 10c6a787e6..8f8d211abf 100644
--- a/pipeline.yaml
+++ b/pipeline.yaml
@@ -60,6 +60,9 @@ jobs:
     - template: templates/update_cli.yml
     - template: templates/conda.yml
     - template: templates/kv.yml
+    - task: ShellScript@2
+      inputs:
+        scriptPath: tools/dotnet/dotnetSetup.sh
     - bash: |
         set -e
         sudo apt-get install doxygen

From 3579628004b6ff84d1946af902099fbbf3f0a0ac Mon Sep 17 00:00:00 2001
From: Xinyue Ruan 
Date: Thu, 30 Jun 2022 19:52:12 +0800
Subject: [PATCH 11/21] fix pipeline ;

---
 build.sbt     | 2 +-
 pipeline.yaml | 5 +----
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/build.sbt b/build.sbt
index 755577f1a8..cac909a98d 100644
--- a/build.sbt
+++ b/build.sbt
@@ -186,7 +186,7 @@ generateDotnetDoc := {
   val dotnetSrcDir = join(rootGenDir.value, "src", "dotnet")
   runCmd(Seq("doxygen", "-g"), dotnetSrcDir)
   FileUtils.copyFile(join(baseDirectory.value, "README.md"), join(dotnetSrcDir, "README.md"))
-  runCmd(Seq("sed", "-i", "\'s/img width=\"800\"/img width=\"300\"/g\'", "README.md"), dotnetSrcDir)
+  runCmd(Seq("sed", "-i", s"""\'s/img width=\"800\"/img width=\"300\"/g\'""", "README.md"), dotnetSrcDir)
   val packageName = name.value.split("-").map(_.capitalize).mkString(" ")
   runCmd(Seq(
     "echo",
diff --git a/pipeline.yaml b/pipeline.yaml
index 8f8d211abf..5a9a4082a3 100644
--- a/pipeline.yaml
+++ b/pipeline.yaml
@@ -60,14 +60,11 @@ jobs:
     - template: templates/update_cli.yml
     - template: templates/conda.yml
     - template: templates/kv.yml
-    - task: ShellScript@2
-      inputs:
-        scriptPath: tools/dotnet/dotnetSetup.sh
     - bash: |
         set -e
         sudo apt-get install doxygen
         source activate synapseml
-        sbt packagePython packageDotnet
+        sbt packagePython
         sbt publishBlob publishDocs publishR publishPython
         sbt publishSigned
         sbt genBuildInfo

From 50ca64db9b552516e32bad9491de131e82b7b392 Mon Sep 17 00:00:00 2001
From: Xinyue Ruan 
Date: Fri, 1 Jul 2022 09:02:51 +0800
Subject: [PATCH 12/21] fix filenotfound exception

---
 .../microsoft/azure/synapse/ml/codegen/DotnetCodegen.scala  | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/DotnetCodegen.scala b/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/DotnetCodegen.scala
index fc10c61b84..f687bc73b4 100644
--- a/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/DotnetCodegen.scala
+++ b/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/DotnetCodegen.scala
@@ -30,8 +30,12 @@ object DotnetCodegen {
       conf.dotnetSrcDir.mkdir()
     }
     val curProject = conf.name.split("-").drop(1).map(s => s.capitalize).mkString("")
+    val projectDir = join(conf.dotnetSrcDir, "synapse", "ml")
+    if (!projectDir.exists()){
+      projectDir.mkdirs()
+    }
     // TODO: update SynapseML.DotnetBase version whenever we upload a new one
-    writeFile(new File(join(conf.dotnetSrcDir, "synapse", "ml"), s"${curProject}ProjectSetup.csproj"),
+    writeFile(new File(projectDir, s"${curProject}ProjectSetup.csproj"),
       s"""
          |
          |  

From 4626d7b9a0da1de1c42e57aace3d861eb5a79bee Mon Sep 17 00:00:00 2001
From: Xinyue Ruan 
Date: Fri, 1 Jul 2022 10:07:43 +0800
Subject: [PATCH 13/21] fix sed

---
 build.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sbt b/build.sbt
index cac909a98d..b72cf47424 100644
--- a/build.sbt
+++ b/build.sbt
@@ -186,7 +186,7 @@ generateDotnetDoc := {
   val dotnetSrcDir = join(rootGenDir.value, "src", "dotnet")
   runCmd(Seq("doxygen", "-g"), dotnetSrcDir)
   FileUtils.copyFile(join(baseDirectory.value, "README.md"), join(dotnetSrcDir, "README.md"))
-  runCmd(Seq("sed", "-i", s"""\'s/img width=\"800\"/img width=\"300\"/g\'""", "README.md"), dotnetSrcDir)
+  runCmd(Seq("sed", "-i", s"""s/img width=\"800\"/img width=\"300\"/g""", "README.md"), dotnetSrcDir)
   val packageName = name.value.split("-").map(_.capitalize).mkString(" ")
   runCmd(Seq(
     "echo",

From 685e0615d682ec67eac92634905fec8981c306a1 Mon Sep 17 00:00:00 2001
From: Xinyue Ruan 
Date: Fri, 1 Jul 2022 10:40:48 +0800
Subject: [PATCH 14/21] fix Doxyfile

---
 build.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sbt b/build.sbt
index b72cf47424..aed06e1c6f 100644
--- a/build.sbt
+++ b/build.sbt
@@ -194,7 +194,7 @@ generateDotnetDoc := {
 PROJECT_NUMBER = \"${dotnetedVersion(version.value)}\"
 USE_MDFILE_AS_MAINPAGE = \"README.md\"
 RECURSIVE = YES""".stripMargin,
-    "|", "tee", "-a", "Doxyfile"
+    ">>", "Doxyfile"
   ), dotnetSrcDir)
   runCmd(Seq("doxygen"), dotnetSrcDir)
 }

From e899f85c0881805381e55aa73926d61c9239daab Mon Sep 17 00:00:00 2001
From: Xinyue Ruan 
Date: Fri, 1 Jul 2022 10:43:20 +0800
Subject: [PATCH 15/21] fix Doxyfile

---
 build.sbt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build.sbt b/build.sbt
index aed06e1c6f..1e0a5d256e 100644
--- a/build.sbt
+++ b/build.sbt
@@ -190,10 +190,10 @@ generateDotnetDoc := {
   val packageName = name.value.split("-").map(_.capitalize).mkString(" ")
   runCmd(Seq(
     "echo",
-    s"""PROJECT_NAME = \"$packageName\"
+    s"""\'PROJECT_NAME = \"$packageName\"
 PROJECT_NUMBER = \"${dotnetedVersion(version.value)}\"
 USE_MDFILE_AS_MAINPAGE = \"README.md\"
-RECURSIVE = YES""".stripMargin,
+RECURSIVE = YES\'""".stripMargin,
     ">>", "Doxyfile"
   ), dotnetSrcDir)
   runCmd(Seq("doxygen"), dotnetSrcDir)

From eafe84fb9d085d08b528ba3391699f932ee06540 Mon Sep 17 00:00:00 2001
From: Xinyue Ruan 
Date: Fri, 1 Jul 2022 11:16:22 +0800
Subject: [PATCH 16/21] fix Doxyfile

---
 build.sbt | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/build.sbt b/build.sbt
index 1e0a5d256e..d523cb6993 100644
--- a/build.sbt
+++ b/build.sbt
@@ -189,13 +189,10 @@ generateDotnetDoc := {
   runCmd(Seq("sed", "-i", s"""s/img width=\"800\"/img width=\"300\"/g""", "README.md"), dotnetSrcDir)
   val packageName = name.value.split("-").map(_.capitalize).mkString(" ")
   runCmd(Seq(
-    "echo",
-    s"""\'PROJECT_NAME = \"$packageName\"
+    s"""echo \'PROJECT_NAME = \"$packageName\"
 PROJECT_NUMBER = \"${dotnetedVersion(version.value)}\"
 USE_MDFILE_AS_MAINPAGE = \"README.md\"
-RECURSIVE = YES\'""".stripMargin,
-    ">>", "Doxyfile"
-  ), dotnetSrcDir)
+RECURSIVE = YES\' >> Doxyfile""".stripMargin), dotnetSrcDir)
   runCmd(Seq("doxygen"), dotnetSrcDir)
 }
 

From 03f0d8eb7e0f44b0f25a71766098cda561b107ce Mon Sep 17 00:00:00 2001
From: Xinyue Ruan 
Date: Fri, 1 Jul 2022 11:24:00 +0800
Subject: [PATCH 17/21] fix Doxyfile

---
 build.sbt | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/build.sbt b/build.sbt
index d523cb6993..74393cabf1 100644
--- a/build.sbt
+++ b/build.sbt
@@ -188,11 +188,10 @@ generateDotnetDoc := {
   FileUtils.copyFile(join(baseDirectory.value, "README.md"), join(dotnetSrcDir, "README.md"))
   runCmd(Seq("sed", "-i", s"""s/img width=\"800\"/img width=\"300\"/g""", "README.md"), dotnetSrcDir)
   val packageName = name.value.split("-").map(_.capitalize).mkString(" ")
-  runCmd(Seq(
-    s"""echo \'PROJECT_NAME = \"$packageName\"
-PROJECT_NUMBER = \"${dotnetedVersion(version.value)}\"
-USE_MDFILE_AS_MAINPAGE = \"README.md\"
-RECURSIVE = YES\' >> Doxyfile""".stripMargin), dotnetSrcDir)
+  runCmd(Seq("echo", "\'PROJECT_NAME = \"$packageName\"\'", ">>", "Doxyfile"), dotnetSrcDir)
+  runCmd(Seq("echo", "\'PROJECT_NUMBER = \"${dotnetedVersion(version.value)}\"\'", ">>", "Doxyfile"), dotnetSrcDir)
+  runCmd(Seq("echo", "\'USE_MDFILE_AS_MAINPAGE = \"README.md\"\'", ">>", "Doxyfile"), dotnetSrcDir)
+  runCmd(Seq("echo", "\'RECURSIVE = YES\'", ">>", "Doxyfile"), dotnetSrcDir)
   runCmd(Seq("doxygen"), dotnetSrcDir)
 }
 

From 56c0f26759a8c9a36fe95fd8a802a23329067661 Mon Sep 17 00:00:00 2001
From: Xinyue Ruan 
Date: Fri, 1 Jul 2022 11:26:33 +0800
Subject: [PATCH 18/21] fix Doxyfile

---
 build.sbt | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/build.sbt b/build.sbt
index 74393cabf1..c20e4f88a6 100644
--- a/build.sbt
+++ b/build.sbt
@@ -188,10 +188,10 @@ generateDotnetDoc := {
   FileUtils.copyFile(join(baseDirectory.value, "README.md"), join(dotnetSrcDir, "README.md"))
   runCmd(Seq("sed", "-i", s"""s/img width=\"800\"/img width=\"300\"/g""", "README.md"), dotnetSrcDir)
   val packageName = name.value.split("-").map(_.capitalize).mkString(" ")
-  runCmd(Seq("echo", "\'PROJECT_NAME = \"$packageName\"\'", ">>", "Doxyfile"), dotnetSrcDir)
-  runCmd(Seq("echo", "\'PROJECT_NUMBER = \"${dotnetedVersion(version.value)}\"\'", ">>", "Doxyfile"), dotnetSrcDir)
-  runCmd(Seq("echo", "\'USE_MDFILE_AS_MAINPAGE = \"README.md\"\'", ">>", "Doxyfile"), dotnetSrcDir)
-  runCmd(Seq("echo", "\'RECURSIVE = YES\'", ">>", "Doxyfile"), dotnetSrcDir)
+  runCmd(Seq("echo \'PROJECT_NAME = \"$packageName\"\' >> Doxyfile"), dotnetSrcDir)
+  runCmd(Seq("echo \'PROJECT_NUMBER = \"${dotnetedVersion(version.value)}\"\' >> Doxyfile"), dotnetSrcDir)
+  runCmd(Seq("echo \'USE_MDFILE_AS_MAINPAGE = \"README.md\"\' >> Doxyfile"), dotnetSrcDir)
+  runCmd(Seq("echo \'RECURSIVE = YES\' >> Doxyfile"), dotnetSrcDir)
   runCmd(Seq("doxygen"), dotnetSrcDir)
 }
 

From dc35252fa3beb2017d9f5dfa1a2c72d9f1edf106 Mon Sep 17 00:00:00 2001
From: Xinyue Ruan 
Date: Fri, 1 Jul 2022 11:29:37 +0800
Subject: [PATCH 19/21] fix Doxyfile

---
 build.sbt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/build.sbt b/build.sbt
index c20e4f88a6..a03ac0a7e1 100644
--- a/build.sbt
+++ b/build.sbt
@@ -188,9 +188,9 @@ generateDotnetDoc := {
   FileUtils.copyFile(join(baseDirectory.value, "README.md"), join(dotnetSrcDir, "README.md"))
   runCmd(Seq("sed", "-i", s"""s/img width=\"800\"/img width=\"300\"/g""", "README.md"), dotnetSrcDir)
   val packageName = name.value.split("-").map(_.capitalize).mkString(" ")
-  runCmd(Seq("echo \'PROJECT_NAME = \"$packageName\"\' >> Doxyfile"), dotnetSrcDir)
-  runCmd(Seq("echo \'PROJECT_NUMBER = \"${dotnetedVersion(version.value)}\"\' >> Doxyfile"), dotnetSrcDir)
-  runCmd(Seq("echo \'USE_MDFILE_AS_MAINPAGE = \"README.md\"\' >> Doxyfile"), dotnetSrcDir)
+  runCmd(Seq(s"""echo \'PROJECT_NAME = \"$packageName\"\' >> Doxyfile"""), dotnetSrcDir)
+  runCmd(Seq(s"""echo \'PROJECT_NUMBER = \"${dotnetedVersion(version.value)}\"\' >> Doxyfile"""), dotnetSrcDir)
+  runCmd(Seq(s"""echo \'USE_MDFILE_AS_MAINPAGE = \"README.md\"\' >> Doxyfile"""), dotnetSrcDir)
   runCmd(Seq("echo \'RECURSIVE = YES\' >> Doxyfile"), dotnetSrcDir)
   runCmd(Seq("doxygen"), dotnetSrcDir)
 }

From 2398c3bccca74d8c249f6b183cb2ce4da9087a43 Mon Sep 17 00:00:00 2001
From: Xinyue Ruan 
Date: Fri, 1 Jul 2022 12:36:17 +0800
Subject: [PATCH 20/21] fix generateDotnetDoc

---
 build.sbt | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/build.sbt b/build.sbt
index a03ac0a7e1..ec9e321088 100644
--- a/build.sbt
+++ b/build.sbt
@@ -188,10 +188,16 @@ generateDotnetDoc := {
   FileUtils.copyFile(join(baseDirectory.value, "README.md"), join(dotnetSrcDir, "README.md"))
   runCmd(Seq("sed", "-i", s"""s/img width=\"800\"/img width=\"300\"/g""", "README.md"), dotnetSrcDir)
   val packageName = name.value.split("-").map(_.capitalize).mkString(" ")
-  runCmd(Seq(s"""echo \'PROJECT_NAME = \"$packageName\"\' >> Doxyfile"""), dotnetSrcDir)
-  runCmd(Seq(s"""echo \'PROJECT_NUMBER = \"${dotnetedVersion(version.value)}\"\' >> Doxyfile"""), dotnetSrcDir)
-  runCmd(Seq(s"""echo \'USE_MDFILE_AS_MAINPAGE = \"README.md\"\' >> Doxyfile"""), dotnetSrcDir)
-  runCmd(Seq("echo \'RECURSIVE = YES\' >> Doxyfile"), dotnetSrcDir)
+  val fileContent =
+    s"""PROJECT_NAME = "$packageName"
+       |PROJECT_NUMBER = "${dotnetedVersion(version.value)}"
+       |USE_MDFILE_AS_MAINPAGE = "README.md"
+       |RECURSIVE = YES
+       |""".stripMargin
+  val doxygenHelperFile = join(dotnetSrcDir, "DoxygenHelper.txt")
+  if (doxygenHelperFile.exists()) FileUtils.forceDelete(doxygenHelperFile)
+  FileUtils.writeStringToFile(doxygenHelperFile, fileContent, "utf-8")
+  runCmd(Seq("bash", "-c","cat DoxygenHelper.txt >> Doxyfile", ""), dotnetSrcDir)
   runCmd(Seq("doxygen"), dotnetSrcDir)
 }
 

From 0e3681d18fa6a2159acd5be2b953fa225eab9dee Mon Sep 17 00:00:00 2001
From: Xinyue Ruan 
Date: Fri, 1 Jul 2022 14:43:06 +0800
Subject: [PATCH 21/21] fix missing graph in dotnet documentation website

---
 pipeline.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pipeline.yaml b/pipeline.yaml
index 5a9a4082a3..375bd28653 100644
--- a/pipeline.yaml
+++ b/pipeline.yaml
@@ -63,6 +63,7 @@ jobs:
     - bash: |
         set -e
         sudo apt-get install doxygen
+        sudo apt-get install graphviz -y
         source activate synapseml
         sbt packagePython
         sbt publishBlob publishDocs publishR publishPython