feat: add publish python packages to pypi task

core/src/main/scala/com/microsoft/azure/synapse/ml/codegen/CodeGen.scala

@@ -158,8 +158,8 @@ object CodeGen {
          |    classifiers=[
          |        "Development Status :: 4 - Beta",
          |        "Intended Audience :: Developers",
-         |        "Intended Audience :: Data Scientists",
-         |        "Topic :: Software Development :: Datascience Tools",
+         |        "Intended Audience :: Science/Research",
+         |        "Topic :: Software Development :: Libraries",
          |        "License :: OSI Approved :: MIT License",
          |        "Programming Language :: Python :: 2",
          |        "Programming Language :: Python :: 3",

environment.yaml

@@ -22,3 +22,4 @@ dependencies:
     - ipython
     - pytest-codeblocks
     - azure-storage-blob
+    - twine

pipeline.yaml

@@ -259,6 +259,27 @@ jobs:
         tagSource: 'auto'
         releaseNotesFile: 'CHANGELOG.md'
         isDraft: true
+    - bash: echo "##vso[task.prependpath]$CONDA/bin"
+      condition: startsWith(variables['tag'], 'v')
+      displayName: Add conda to PATH
+    - bash: conda info
+      condition: startsWith(variables['tag'], 'v')
+    - bash: conda env create -f environment.yaml
+      condition: startsWith(variables['tag'], 'v')
+      displayName: Create Anaconda environment
+    - task: AzureKeyVault@1
+      condition: startsWith(variables['tag'], 'v')
+      inputs:
+        azureSubscription: 'MMLSpark Build'
+        keyVaultName: mmlspark-keys
+    - bash: |
+        source activate synapseml
+        sbt packagePython
+        sbt publishPypi
+      condition: startsWith(variables['tag'], 'v')
+      env:
+        PYPI-API-TOKEN: $(pypi-api-token)
+      displayName: 'publish python package to pypi'
 
 - job: PythonTests
   cancelTimeoutInMinutes: 0

project/CodegenPlugin.scala

@@ -62,6 +62,7 @@ object CodegenPlugin extends AutoPlugin {
     val installPipPackage = TaskKey[Unit]("installPipPackage", "install python sdk")
     val publishPython = TaskKey[Unit]("publishPython", "publish python wheel")
     val testPython = TaskKey[Unit]("testPython", "test python sdk")
+    val publishPypi = TaskKey[Unit]("publishPypi", "publish python wheel to pypi")
 
     val mergePyCodeDir = SettingKey[File]("mergePyCodeDir")
     val mergePyCode = TaskKey[Unit]("mergePyCode", "copy python code to a destination")
@@ -209,6 +210,17 @@ object CodegenPlugin extends AutoPlugin {
         join(codegenDir.value, "package", "python", fn).toString,
         version.value + "/" + fn, "pip")
     },
+    publishPypi := {
+      packagePython.value
+      publishLocal.value
+      val fn = s"${name.value.replace("-", "_")}-${pythonizedVersion.value}-py2.py3-none-any.whl"
+      runCmd(
+        activateCondaEnv.value ++
+          Seq("twine", "upload", "--skip-existing",
+            join(codegenDir.value, "package", "python", fn).toString,
+            "--username", "__token__", "--password", "${PYPI-API-TOKEN}")
+      )
+    },
     mergePyCode := {
       val srcDir = join(codegenDir.value, "src", "python", genPyPackageNamespace.value)
       val destDir = join(mergePyCodeDir.value, "src", "python", genPyPackageNamespace.value)

project/Secrets.scala

@@ -59,5 +59,6 @@ object Secrets {
     sys.env.getOrElse("PGP-PRIVATE", getSecret("pgp-private")).getBytes("UTF-8")))
   lazy val pgpPassword: String = sys.env.getOrElse("PGP-PW", getSecret("pgp-pw"))
   lazy val storageKey: String = sys.env.getOrElse("STORAGE_KEY", getSecret("storage-key"))
+  lazy val pypiAPIToken: String = sys.env.getOrElse("PYPI-API-TOKEN", getSecret("pypi-api-token"))
 
 }