microsoft · mhamilton723 · Jul 1, 2022 · Jun 21, 2022 · Jun 21, 2022 · Jun 22, 2022
@@ -99,6 +99,12 @@ genBuildInfo := {
   FileUtils.writeStringToFile(infoFile, buildInfo, "utf-8")
 }
 
+val rootGenDir = SettingKey[File]("rootGenDir")
+rootGenDir := {
+  val targetDir = (root / Compile / packageBin / artifactPath).value.getParentFile
+  join(targetDir, "generated")
+}
+
 // scalastyle:off line.size.limit
 val genSleetConfig = TaskKey[Unit]("genSleetConfig",
   "generate sleet.json file for sleet configuration so we can push nuget package to the blob")
@@ -148,21 +154,9 @@ publishDotnetTestBase := {
   val dotnetHelperFile = join(dotnetTestBaseDir, "SynapseMLVersion.cs")
   if (dotnetHelperFile.exists()) FileUtils.forceDelete(dotnetHelperFile)
   FileUtils.writeStringToFile(dotnetHelperFile, fileContent, "utf-8")
-  runCmd(
-    Seq("dotnet", "pack", "--output", join(dotnetTestBaseDir, "target").getAbsolutePath),
-    dotnetTestBaseDir
-  )
+  packDotnetAssemblyCmd(join(dotnetTestBaseDir, "target").getAbsolutePath, dotnetTestBaseDir)
   val packagePath = join(dotnetTestBaseDir, "target", s"SynapseML.DotnetE2ETest.0.9.1.nupkg").getAbsolutePath
-  runCmd(
-    Seq("sleet", "push", packagePath, "--config", join(rootGenDir.value, "sleet.json").getAbsolutePath,
-      "--source", "SynapseMLNuget", "--force")
-  )
-}
-
-val rootGenDir = SettingKey[File]("rootGenDir")
-rootGenDir := {
-  val targetDir = (root / Compile / packageBin / artifactPath).value.getParentFile
-  join(targetDir, "generated")
+  publishDotnetAssemblyCmd(packagePath, rootGenDir.value)
 }
 
 def runTaskForAllInCompile(task: TaskKey[Unit]): Def.Initialize[Task[Seq[Unit]]] = {
@@ -183,6 +177,30 @@ generatePythonDoc := {
   runCmd(activateCondaEnv ++ Seq("sphinx-build", "-b", "html", "doc", "../../../doc/pyspark"), dir)
 }
 
+val generateDotnetDoc = TaskKey[Unit]("generateDotnetDoc", "Generate documentation for dotnet classes")
+generateDotnetDoc := {
+  Def.sequential(
+    runTaskForAllInCompile(dotnetCodeGen),
+    runTaskForAllInCompile(mergeDotnetCode)
+  ).value
+  val dotnetSrcDir = join(rootGenDir.value, "src", "dotnet")
+  runCmd(Seq("doxygen", "-g"), dotnetSrcDir)
+  FileUtils.copyFile(join(baseDirectory.value, "README.md"), join(dotnetSrcDir, "README.md"))
+  runCmd(Seq("sed", "-i", s"""s/img width=\"800\"/img width=\"300\"/g""", "README.md"), dotnetSrcDir)
+  val packageName = name.value.split("-").map(_.capitalize).mkString(" ")
+  val fileContent =
+    s"""PROJECT_NAME = "$packageName"
+       |PROJECT_NUMBER = "${dotnetedVersion(version.value)}"
+       |USE_MDFILE_AS_MAINPAGE = "README.md"
+       |RECURSIVE = YES
+       |""".stripMargin
+  val doxygenHelperFile = join(dotnetSrcDir, "DoxygenHelper.txt")
+  if (doxygenHelperFile.exists()) FileUtils.forceDelete(doxygenHelperFile)
+  FileUtils.writeStringToFile(doxygenHelperFile, fileContent, "utf-8")
+  runCmd(Seq("bash", "-c","cat DoxygenHelper.txt >> Doxyfile", ""), dotnetSrcDir)
+  runCmd(Seq("doxygen"), dotnetSrcDir)
+}
+
 val packageSynapseML = TaskKey[Unit]("packageSynapseML", "package all projects into SynapseML")
 packageSynapseML := {
   def writeSetupFileToTarget(dir: File): Unit = {
@@ -250,10 +268,11 @@ publishPypi := {
   )
 }
 
-val publishDocs = TaskKey[Unit]("publishDocs", "publish docs for scala and python")
+val publishDocs = TaskKey[Unit]("publishDocs", "publish docs for scala, python and dotnet")
 publishDocs := {
   generatePythonDoc.value
   (root / Compile / unidoc).value
+  generateDotnetDoc.value
   val html =
     """
       |<html><body><pre style="font-size: 150%;">
@@ -268,6 +287,9 @@ publishDocs := {
   if (scalaDir.exists()) FileUtils.forceDelete(scalaDir)
   FileUtils.copyDirectory(join(targetDir, "unidoc"), scalaDir)
   FileUtils.writeStringToFile(join(unifiedDocDir.toString, "index.html"), html, "utf-8")
+  val dotnetDir = join(unifiedDocDir.toString, "dotnet")
+  if (dotnetDir.exists()) FileUtils.forceDelete(dotnetDir)
+  FileUtils.copyDirectory(join(codegenDir, "src", "dotnet", "html"), dotnetDir)
   uploadToBlob(unifiedDocDir.toString, version.value, "docs")
 }
 

@@ -239,7 +239,13 @@ trait HasSetLinkedService extends Wrappable with HasURL with HasSubscriptionKey
   }
 
   override def dotnetAdditionalMethods: String = super.dotnetAdditionalMethods + {
-    s"""
+    s"""/// <summary>
+       |/// Sets value for linkedService
+       |/// </summary>
+       |/// <param name=\"value\">
+       |/// linkedService name
+       |/// </param>
+       |/// <returns> New $dotnetClassName object </returns>
        |public $dotnetClassName SetLinkedService(string value) =>
        |    $dotnetClassWrapperName(Reference.Invoke(\"setLinkedService\", value));
        |""".stripMargin
@@ -280,7 +286,13 @@ trait HasSetLocation extends Wrappable with HasURL with HasUrlPath {
   }
 
   override def dotnetAdditionalMethods: String = super.dotnetAdditionalMethods + {
-    s"""
+    s"""/// <summary>
+       |/// Sets value for location
+       |/// </summary>
+       |/// <param name=\"value\">
+       |/// Location of the cognitive service
+       |/// </param>
+       |/// <returns> New $dotnetClassName object </returns>
        |public $dotnetClassName SetLocation(string value) =>
        |    $dotnetClassWrapperName(Reference.Invoke(\"setLocation\", value));
        |""".stripMargin

@@ -37,7 +37,7 @@ trait HasPages extends HasServiceParams {
   val pages = new ServiceParam[String](this, "pages", "The page selection only leveraged for" +
     " multi-page PDF and TIFF documents. Accepted input include single pages (e.g.'1, 2' -> pages 1 and 2 will be " +
     "processed), finite (e.g. '2-5' -> pages 2 to 5 will be processed) and open-ended ranges (e.g. '5-' -> all the" +
-    " pages from page 5 will be processed & e.g. '-10' -> pages 1 to 10 will be processed). All of these can be mixed" +
+    " pages from page 5 will be processed; e.g. '-10' -> pages 1 to 10 will be processed). All of these can be mixed" +
     " together and ranges are allowed to overlap (eg. '-5, 1, 3, 5-10' - pages 1 to 10 will be processed). The" +
     " service will accept the request if it can process at least one page of the document (e.g. using '5-100' on a " +
     "5 page document is a valid input where page 5 will be processed). If no page range is provided, the entire" +

@@ -256,10 +256,11 @@ class Translate(override val uid: String) extends TextTranslatorBase(uid)
   override protected def getInternalTransformer(schema: StructType): PipelineModel =
     customGetInternalTransformer(schema, Seq("text", "toLanguage"))
 
-  val toLanguage = new ServiceParam[Seq[String]](this, "toLanguage", "Specifies the language of the output" +
-    " text. The target language must be one of the supported languages included in the translation scope." +
-    " For example, use to=de to translate to German. It's possible to translate to multiple languages simultaneously" +
-    " by repeating the parameter in the query string. For example, use to=de&to=it to translate to German and Italian.",
+  val toLanguage = new ServiceParam[Seq[String]](this, "toLanguage",
+    "Specifies the language of the output text. The target language must be one of the supported languages" +
+      " included in the translation scope. For example, use to=de to translate to German. It's possible to translate" +
+      " to multiple languages simultaneously by repeating the parameter in the query string. For example, use " +
+      "to=de and to=it to translate to German and Italian.",
     isRequired = true, isURLParam = true,
     toValueString = { seq => seq.mkString(",") })
 

@@ -0,0 +1,8 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+namespace System.Runtime.CompilerServices
+{
+    internal static class IsExternalInit {}
+}
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net5.0</TargetFramework>
+    <TargetFramework>netstandard2.1</TargetFramework>
     <LangVersion>9.0</LangVersion>
     <AssemblyName>SynapseML.DotnetBase</AssemblyName>
     <IsPackable>true</IsPackable>

@@ -16,7 +16,6 @@
 using Microsoft.Spark.Utils;
 using SynapseML.Dotnet.Utils;
 using Synapse.ML.LightGBM.Param;
-using Microsoft.Spark.ML.Feature;
 
 namespace Microsoft.Spark.ML.Feature
 {

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net5.0</TargetFramework>
+    <TargetFramework>netstandard2.1</TargetFramework>
     <LangVersion>9.0</LangVersion>
     <AssemblyName>SynapseML.DotnetE2ETest</AssemblyName>
     <IsPackable>true</IsPackable>

diff --git a/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/DotnetCodegen.scala b/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/DotnetCodegen.scala
@@ -30,16 +30,20 @@ object DotnetCodegen {
       conf.dotnetSrcDir.mkdir()
     }
     val curProject = conf.name.split("-").drop(1).map(s => s.capitalize).mkString("")
+    val projectDir = join(conf.dotnetSrcDir, "synapse", "ml")
+    if (!projectDir.exists()){
+      projectDir.mkdirs()
+    }
     // TODO: update SynapseML.DotnetBase version whenever we upload a new one
-    writeFile(new File(join(conf.dotnetSrcDir, "synapse", "ml"), s"${curProject}ProjectSetup.csproj"),
+    writeFile(new File(projectDir, s"${curProject}ProjectSetup.csproj"),
       s"""<Project Sdk="Microsoft.NET.Sdk">
          |
          |  <PropertyGroup>
-         |    <TargetFramework>net5.0</TargetFramework>
+         |    <TargetFramework>netstandard2.1</TargetFramework>
          |    <LangVersion>9.0</LangVersion>
          |    <AssemblyName>SynapseML.$curProject</AssemblyName>
          |    <IsPackable>true</IsPackable>
-         |
+         |    <GenerateDocumentationFile>true</GenerateDocumentationFile>
          |    <Description>.NET for SynapseML.$curProject</Description>
          |    <Version>${conf.dotnetVersion}</Version>
          |  </PropertyGroup>

diff --git a/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/DotnetWrappable.scala b/core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/DotnetWrappable.scala
@@ -4,7 +4,7 @@
 package com.microsoft.azure.synapse.ml.codegen
 
 import com.microsoft.azure.synapse.ml.core.env.FileUtilities
-import com.microsoft.azure.synapse.ml.param.WrappableParam
+import com.microsoft.azure.synapse.ml.param.{ServiceParam, WrappableParam}
 import org.apache.commons.lang.StringUtils.capitalize
 import org.apache.spark.ml._
 import org.apache.spark.ml.evaluation.Evaluator
@@ -120,14 +120,21 @@ trait DotnetWrappable extends BaseWrappable {
     val capName = p.name.capitalize
     val docString =
       s"""|/// <summary>
-          |/// Sets ${p.name} value for <see cref=\"${p.name}\"/>
+          |/// Sets value for ${p.name}
           |/// </summary>
-          |/// <param name=\"${p.name}\">
+          |/// <param name=\"value\">
           |/// ${p.doc}
           |/// </param>
           |/// <returns> New $dotnetClassName object </returns>""".stripMargin
     p match {
       // TODO: Fix UDF & UDPyF confusion; ParamSpaceParam, BallTreeParam, ConditionalBallTreeParam type
+      case sp: ServiceParam[_] =>
+        s"""|$docString
+            |${sp.dotnetSetter(dotnetClassName, capName, dotnetClassWrapperName)}
+            |
+            |${docString.replaceFirst(sp.name, s"${sp.name} column")}
+            |${sp.dotnetSetterForSrvParamCol(dotnetClassName, capName, dotnetClassWrapperName)}
+            |""".stripMargin
       case wp: WrappableParam[_] =>
         s"""|$docString
             |${wp.dotnetSetter(dotnetClassName, capName, dotnetClassWrapperName)}
@@ -148,7 +155,7 @@ trait DotnetWrappable extends BaseWrappable {
     val capName = p.name.capitalize
     val docString =
       s"""|/// <summary>
-          |/// Gets ${p.name} value for <see cref=\"${p.name}\"/>
+          |/// Gets ${p.name} value
           |/// </summary>
           |/// <returns>
           |/// ${p.name}: ${p.doc}

@@ -132,14 +132,17 @@ class ServiceParam[T: TypeTag](parent: Params,
 
   override private[ml] def dotnetSetter(dotnetClassName: String,
                                         capName: String,
-                                        dotnetClassWrapperName: String): String = {
+                                        dotnetClassWrapperName: String): String =
     s"""|public $dotnetClassName Set$capName($dotnetType value) =>
         |    $dotnetClassWrapperName(Reference.Invoke(\"set$capName\", (object)value));
-        |
-        |public $dotnetClassName Set${capName}Col(string value) =>
+        |""".stripMargin
+
+  private[ml] def dotnetSetterForSrvParamCol(dotnetClassName: String,
+                                             capName: String,
+                                             dotnetClassWrapperName: String): String =
+    s"""|public $dotnetClassName Set${capName}Col(string value) =>
         |    $dotnetClassWrapperName(Reference.Invoke(\"set${capName}Col\", value));
         |""".stripMargin
-  }
 
   override private[ml] def dotnetGetter(capName: String): String = {
     dotnetType match {

@@ -136,7 +136,7 @@ trait RankingTrainValidationSplitParams extends Wrappable with HasSeed {
     * @group param
     */
   val trainRatio: DoubleParam = new DoubleParam(this, "trainRatio",
-    "ratio between training set and validation set (>= 0 && <= 1)", ParamValidators.inRange(0, 1))
+    "ratio between training set and validation set (>= 0 and <= 1)", ParamValidators.inRange(0, 1))
 
   /** @group getParam */
   def getTrainRatio: Double = $(trainRatio)

@@ -73,7 +73,7 @@ object DotnetTestGen {
       s"""<Project Sdk="Microsoft.NET.Sdk">
          |
          |  <PropertyGroup>
-         |    <TargetFramework>net5.0</TargetFramework>
+         |    <TargetFramework>netcoreapp3.1</TargetFramework>
          |    <LangVersion>9.0</LangVersion>
          |    <AssemblyName>SynapseML.$curProject.Test</AssemblyName>
          |  </PropertyGroup>

@@ -62,6 +62,8 @@ jobs:
     - template: templates/kv.yml
     - bash: |
         set -e
+        sudo apt-get install doxygen
+        sudo apt-get install graphviz -y
         source activate synapseml
         sbt packagePython
         sbt publishBlob publishDocs publishR publishPython

@@ -77,8 +77,9 @@ object CodegenPlugin extends AutoPlugin {
     val publishDotnet = TaskKey[Unit]("publishDotnet", "publish dotnet nuget package")
     val testDotnet = TaskKey[Unit]("testDotnet", "test dotnet nuget package")
 
-    val mergePyCodeDir = SettingKey[File]("mergePyCodeDir")
+    val mergeCodeDir = SettingKey[File]("mergeCodeDir")
     val mergePyCode = TaskKey[Unit]("mergePyCode", "copy python code to a destination")
+    val mergeDotnetCode = TaskKey[Unit]("mergeDotnetCode", "copy dotnet code to a destination")
   }
 
   import autoImport._
@@ -218,7 +219,7 @@ object CodegenPlugin extends AutoPlugin {
         val versionArray = version.value.split("-".toCharArray)
         versionArray.head + "-rc" + versionArray.drop(1).dropRight(1).mkString("")
       } else {
-       version.value
+        version.value
       }
     },
     packageR := {
@@ -268,7 +269,12 @@ object CodegenPlugin extends AutoPlugin {
     },
     mergePyCode := {
       val srcDir = join(codegenDir.value, "src", "python", genPackageNamespace.value)
-      val destDir = join(mergePyCodeDir.value, "src", "python", genPackageNamespace.value)
+      val destDir = join(mergeCodeDir.value, "src", "python", genPackageNamespace.value)
+      FileUtils.copyDirectory(srcDir, destDir)
+    },
+    mergeDotnetCode := {
+      val srcDir = join(codegenDir.value, "src", "dotnet", genPackageNamespace.value)
+      val destDir = join(mergeCodeDir.value, "src", "dotnet", genPackageNamespace.value)
       FileUtils.copyDirectory(srcDir, destDir)
     },
     testPython := {
@@ -298,24 +304,19 @@ object CodegenPlugin extends AutoPlugin {
       val sourceDotnetDir = join(dotnetSrcDir.getAbsolutePath, genPackageNamespace.value)
       FileUtils.copyDirectory(sourceDotnetDir, destDotnetDir)
       val packageDir = join(codegenDir.value, "package", "dotnet").absolutePath
-      runCmd(
-        Seq("dotnet", "pack", "--output", packageDir),
-        join(dotnetSrcDir, "synapse", "ml"))
+      packDotnetAssemblyCmd(packageDir, join(dotnetSrcDir, "synapse", "ml"))
     },
     publishDotnet := {
       packageDotnet.value
       val dotnetPackageName = name.value.split("-").drop(1).map(s => s.capitalize).mkString("")
       val packagePath = join(codegenDir.value, "package", "dotnet",
         s"SynapseML.$dotnetPackageName.${dotnetVersion.value}.nupkg").absolutePath
-      val sleetConfigFile = join(mergePyCodeDir.value, "sleet.json").getAbsolutePath
-      runCmd(
-        Seq("sleet", "push", packagePath, "--config", sleetConfigFile, "--source", "SynapseMLNuget", "--force")
-      )
+      publishDotnetAssemblyCmd(packagePath, mergeCodeDir.value)
     },
     targetDir := {
       (Compile / packageBin / artifactPath).value.getParentFile
     },
-    mergePyCodeDir := {
+    mergeCodeDir := {
       join(baseDirectory.value.getParent, "target", "scala-2.12", "generated")
     },
     codegenDir := {

@@ -30,6 +30,16 @@ object BuildUtils {
     }
   }
 
+  def dotnetedVersion(version: String): String = {
+    version match {
+      case s if s.contains("-") => {
+        val versionArray = s.split("-".toCharArray)
+        versionArray.head + "-rc" + versionArray.drop(1).dropRight(1).mkString("")
+      }
+      case s => s
+    }
+  }
+
   def runCmd(cmd: Seq[String],
              wd: File = new File("."),
              envVars: Map[String, String] = Map()): Unit = {
@@ -63,6 +73,17 @@ object BuildUtils {
       workDir)
   }
 
+  def packDotnetAssemblyCmd(outputDir: String,
+                            workDir: File): Unit =
+    runCmd(Seq("dotnet", "pack", "--output", outputDir), workDir)
+
+  def publishDotnetAssemblyCmd(packagePath: String,
+                               sleetConfigDir: File): Unit =
+    runCmd(
+      Seq("sleet", "push", packagePath, "--config", join(sleetConfigDir, "sleet.json").getAbsolutePath,
+        "--source", "SynapseMLNuget", "--force")
+    )
+
   def uploadToBlob(source: String,
                    dest: String,
                    container: String,