[Modules] Follow-Up-To: Added Azure Key Vault key management service settings to Security profile

modules/container-service/managed-cluster/README.md

@@ -124,6 +124,11 @@ module managedCluster 'br:bicep/modules/container-service.managed-cluster:1.0.0'
       }
     ]
     autoUpgradeProfileUpgradeChannel: 'stable'
+    customerManagedKey: {
+      keyName: '<keyName>'
+      keyVaultNetworkAccess: 'Public'
+      keyVaultResourceId: '<keyVaultResourceId>'
+    }
     diagnosticSettings: [
       {
         eventHubAuthorizationRuleResourceId: '<eventHubAuthorizationRuleResourceId>'
@@ -339,6 +344,13 @@ module managedCluster 'br:bicep/modules/container-service.managed-cluster:1.0.0'
     "autoUpgradeProfileUpgradeChannel": {
       "value": "stable"
     },
+    "customerManagedKey": {
+      "value": {
+        "keyName": "<keyName>",
+        "keyVaultNetworkAccess": "Public",
+        "keyVaultResourceId": "<keyVaultResourceId>"
+      }
+    },
     "diagnosticSettings": {
       "value": [
         {
@@ -1167,6 +1179,7 @@ module managedCluster 'br:bicep/modules/container-service.managed-cluster:1.0.0'
 | [`autoUpgradeProfileUpgradeChannel`](#parameter-autoupgradeprofileupgradechannel) | string | Auto-upgrade channel on the AKS cluster. |
 | [`azurePolicyEnabled`](#parameter-azurepolicyenabled) | bool | Specifies whether the azurepolicy add-on is enabled or not. For security reasons, this setting should be enabled. |
 | [`azurePolicyVersion`](#parameter-azurepolicyversion) | string | Specifies the azure policy version to use. |
+| [`customerManagedKey`](#parameter-customermanagedkey) | object | The customer managed key definition. |
 | [`diagnosticSettings`](#parameter-diagnosticsettings) | array | The diagnostic settings of the service. |
 | [`disableLocalAccounts`](#parameter-disablelocalaccounts) | bool | If set to true, getting static credentials will be disabled for this cluster. This must only be used on Managed Clusters that are AAD enabled. |
 | [`disableRunCommand`](#parameter-disableruncommand) | bool | Whether to disable run command for the cluster or not. |
@@ -1497,6 +1510,49 @@ Specifies the azure policy version to use.
 - Type: string
 - Default: `'v2'`
 
+### Parameter: `customerManagedKey`
+
+The customer managed key definition.
+- Required: No
+- Type: object
+
+
+| Name | Required | Type | Description |
+| :-- | :-- | :--| :-- |
+| [`keyName`](#parameter-customermanagedkeykeyname) | Yes | string | Required. The name of the customer managed key to use for encryption. |
+| [`keyVaultNetworkAccess`](#parameter-customermanagedkeykeyvaultnetworkaccess) | Yes | string | Required. Network access of key vault. The possible values are Public and Private. Public means the key vault allows public access from all networks. Private means the key vault disables public access and enables private link. The default value is Public. |
+| [`keyVaultResourceId`](#parameter-customermanagedkeykeyvaultresourceid) | Yes | string | Required. The resource ID of a key vault to reference a customer managed key for encryption from. |
+| [`keyVersion`](#parameter-customermanagedkeykeyversion) | No | string | Optional. The version of the customer managed key to reference for encryption. If not provided, using 'latest'. |
+
+### Parameter: `customerManagedKey.keyName`
+
+Required. The name of the customer managed key to use for encryption.
+
+- Required: Yes
+- Type: string
+
+### Parameter: `customerManagedKey.keyVaultNetworkAccess`
+
+Required. Network access of key vault. The possible values are Public and Private. Public means the key vault allows public access from all networks. Private means the key vault disables public access and enables private link. The default value is Public.
+
+- Required: Yes
+- Type: string
+- Allowed: `[Private, Public]`
+
+### Parameter: `customerManagedKey.keyVaultResourceId`
+
+Required. The resource ID of a key vault to reference a customer managed key for encryption from.
+
+- Required: Yes
+- Type: string
+
+### Parameter: `customerManagedKey.keyVersion`
+
+Optional. The version of the customer managed key to reference for encryption. If not provided, using 'latest'.
+
+- Required: No
+- Type: string
+
 ### Parameter: `diagnosticSettings`
 
 The diagnostic settings of the service.

modules/container-service/managed-cluster/agent-pool/main.json

@@ -5,8 +5,8 @@
   "metadata": {
     "_generator": {
       "name": "bicep",
-      "version": "0.22.6.54827",
-      "templateHash": "15823498371287518640"
+      "version": "0.23.1.45101",
+      "templateHash": "13811832596066396545"
     },
     "name": "Azure Kubernetes Service (AKS) Managed Cluster Agent Pools",
     "description": "This module deploys an Azure Kubernetes Service (AKS) Managed Cluster Agent Pool.",

modules/container-service/managed-cluster/main.bicep

@@ -348,6 +348,18 @@ param httpProxyConfig object = {}
 @description('Optional. Identities associated with the cluster.')
 param identityProfile object = {}
 
+@description('Optional. The customer managed key definition.')
+param customerManagedKey customerManagedKeyType
+
+resource cMKKeyVault 'Microsoft.KeyVault/vaults@2023-02-01' existing = if (!empty(customerManagedKey.?keyVaultResourceId)) {
+  name: last(split((customerManagedKey.?keyVaultResourceId ?? 'dummyVault'), '/'))
+  scope: resourceGroup(split((customerManagedKey.?keyVaultResourceId ?? '//'), '/')[2], split((customerManagedKey.?keyVaultResourceId ?? '////'), '/')[4])
+
+  resource cMKKey 'keys@2023-02-01' existing = if (!empty(customerManagedKey.?keyVaultResourceId) && !empty(customerManagedKey.?keyName)) {
+    name: customerManagedKey.?keyName ?? 'dummyKey'
+  }
+}
+
 var formattedUserAssignedIdentities = reduce(map((managedIdentities.?userAssignedResourceIds ?? []), (id) => { '${id}': {} }), {}, (cur, next) => union(cur, next)) // Converts the flat array to an object like { '${id1}': {}, '${id2}': {} }
 
 var identity = !empty(managedIdentities) ? {
@@ -539,6 +551,12 @@ resource managedCluster 'Microsoft.ContainerService/managedClusters@2023-07-02-p
       userAssignedIdentityExceptions: podIdentityProfileUserAssignedIdentityExceptions
     }
     securityProfile: {
+      azureKeyVaultKms: !empty(customerManagedKey) ? {
+        enabled: true
+        keyId: !empty(customerManagedKey.?keyVersion ?? '') ? '${cMKKeyVault::cMKKey.properties.keyUri}/${customerManagedKey!.keyVersion}' : cMKKeyVault::cMKKey.properties.keyUriWithVersion
+        keyVaultNetworkAccess: customerManagedKey!.keyVaultNetworkAccess
+        keyVaultResourceId: customerManagedKey!.keyVaultNetworkAccess == 'Private' ? cMKKeyVault.id : null
+      } : null
       defender: enableAzureDefender ? {
         securityMonitoring: {
           enabled: enableAzureDefender
@@ -806,3 +824,17 @@ type diagnosticSettingType = {
   @description('Optional. The full ARM resource ID of the Marketplace resource to which you would like to send Diagnostic Logs.')
   marketplacePartnerResourceId: string?
 }[]?
+
+type customerManagedKeyType = {
+  @description('Required. The resource ID of a key vault to reference a customer managed key for encryption from.')
+  keyVaultResourceId: string
+
+  @description('Required. The name of the customer managed key to use for encryption.')
+  keyName: string
+
+  @description('Optional. The version of the customer managed key to reference for encryption. If not provided, using \'latest\'.')
+  keyVersion: string?
+
+  @description('Required. Network access of key vault. The possible values are Public and Private. Public means the key vault allows public access from all networks. Private means the key vault disables public access and enables private link. The default value is Public.')
+  keyVaultNetworkAccess: ('Private' | 'Public')
+}?

modules/container-service/managed-cluster/main.json

@@ -6,7 +6,7 @@
     "_generator": {
       "name": "bicep",
       "version": "0.23.1.45101",
-      "templateHash": "10758692765653328788"
+      "templateHash": "4013697482173328246"
     },
     "name": "Azure Kubernetes Service (AKS) Managed Clusters",
     "description": "This module deploys an Azure Kubernetes Service (AKS) Managed Cluster.",
@@ -232,6 +232,41 @@
         }
       },
       "nullable": true
+    },
+    "customerManagedKeyType": {
+      "type": "object",
+      "properties": {
+        "keyVaultResourceId": {
+          "type": "string",
+          "metadata": {
+            "description": "Required. The resource ID of a key vault to reference a customer managed key for encryption from."
+          }
+        },
+        "keyName": {
+          "type": "string",
+          "metadata": {
+            "description": "Required. The name of the customer managed key to use for encryption."
+          }
+        },
+        "keyVersion": {
+          "type": "string",
+          "nullable": true,
+          "metadata": {
+            "description": "Optional. The version of the customer managed key to reference for encryption. If not provided, using 'latest'."
+          }
+        },
+        "keyVaultNetworkAccess": {
+          "type": "string",
+          "allowedValues": [
+            "Private",
+            "Public"
+          ],
+          "metadata": {
+            "description": "Required. Network access of key vault. The possible values are Public and Private. Public means the key vault allows public access from all networks. Private means the key vault disables public access and enables private link. The default value is Public."
+          }
+        }
+      },
+      "nullable": true
     }
   },
   "parameters": {
@@ -938,6 +973,12 @@
       "metadata": {
         "description": "Optional. Identities associated with the cluster."
       }
+    },
+    "customerManagedKey": {
+      "$ref": "#/definitions/customerManagedKeyType",
+      "metadata": {
+        "description": "Optional. The customer managed key definition."
+      }
     }
   },
   "variables": {
@@ -983,6 +1024,27 @@
     }
   },
   "resources": {
+    "cMKKeyVault::cMKKey": {
+      "condition": "[and(not(empty(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'))), and(not(empty(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'))), not(empty(tryGet(parameters('customerManagedKey'), 'keyName')))))]",
+      "existing": true,
+      "type": "Microsoft.KeyVault/vaults/keys",
+      "apiVersion": "2023-02-01",
+      "subscriptionId": "[split(coalesce(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'), '//'), '/')[2]]",
+      "resourceGroup": "[split(coalesce(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'), '////'), '/')[4]]",
+      "name": "[format('{0}/{1}', last(split(coalesce(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'), 'dummyVault'), '/')), coalesce(tryGet(parameters('customerManagedKey'), 'keyName'), 'dummyKey'))]",
+      "dependsOn": [
+        "cMKKeyVault"
+      ]
+    },
+    "cMKKeyVault": {
+      "condition": "[not(empty(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId')))]",
+      "existing": true,
+      "type": "Microsoft.KeyVault/vaults",
+      "apiVersion": "2023-02-01",
+      "subscriptionId": "[split(coalesce(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'), '//'), '/')[2]]",
+      "resourceGroup": "[split(coalesce(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'), '////'), '/')[4]]",
+      "name": "[last(split(coalesce(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'), 'dummyVault'), '/'))]"
+    },
     "defaultTelemetry": {
       "condition": "[parameters('enableDefaultTelemetry')]",
       "type": "Microsoft.Resources/deployments",
@@ -1116,6 +1178,7 @@
           "userAssignedIdentityExceptions": "[parameters('podIdentityProfileUserAssignedIdentityExceptions')]"
         },
         "securityProfile": {
+          "azureKeyVaultKms": "[if(not(empty(parameters('customerManagedKey'))), createObject('enabled', true(), 'keyId', if(not(empty(coalesce(tryGet(parameters('customerManagedKey'), 'keyVersion'), ''))), format('{0}/{1}', reference('cMKKeyVault::cMKKey').keyUri, parameters('customerManagedKey').keyVersion), reference('cMKKeyVault::cMKKey').keyUriWithVersion), 'keyVaultNetworkAccess', parameters('customerManagedKey').keyVaultNetworkAccess, 'keyVaultResourceId', if(equals(parameters('customerManagedKey').keyVaultNetworkAccess, 'Private'), extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', split(coalesce(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'), '//'), '/')[2], split(coalesce(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'), '////'), '/')[4]), 'Microsoft.KeyVault/vaults', last(split(coalesce(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'), 'dummyVault'), '/'))), null())), null())]",
           "defender": "[if(parameters('enableAzureDefender'), createObject('securityMonitoring', createObject('enabled', parameters('enableAzureDefender')), 'logAnalyticsWorkspaceResourceId', if(not(empty(parameters('monitoringWorkspaceId'))), parameters('monitoringWorkspaceId'), null())), null())]",
           "workloadIdentity": "[if(parameters('enableWorkloadIdentity'), createObject('enabled', parameters('enableWorkloadIdentity')), null())]"
         },
@@ -1134,7 +1197,10 @@
           }
         },
         "supportPlan": "[parameters('supportPlan')]"
-      }
+      },
+      "dependsOn": [
+        "cMKKeyVault"
+      ]
     },
     "managedCluster_lock": {
       "condition": "[and(not(empty(coalesce(parameters('lock'), createObject()))), not(equals(tryGet(parameters('lock'), 'kind'), 'None')))]",

modules/container-service/managed-cluster/tests/e2e/azure/dependencies.bicep

@@ -79,6 +79,13 @@ resource keyVault 'Microsoft.KeyVault/vaults@2022-11-01' = {
       kty: 'RSA'
     }
   }
+
+  resource kmskey 'keys@2022-07-01' = {
+    name: 'kmsEncryptionKey'
+    properties: {
+      kty: 'RSA'
+    }
+  }
 }
 
 resource diskEncryptionSet 'Microsoft.Compute/diskEncryptionSets@2022-07-02' = {
@@ -98,6 +105,16 @@ resource diskEncryptionSet 'Microsoft.Compute/diskEncryptionSets@2022-07-02' = {
   }
 }
 
+resource keyPermissionsKeyVaultCryptoUser 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
+  name: guid('msi-${keyVault.id}-${location}-${managedIdentity.id}-KeyVault-Crypto-User-RoleAssignment')
+  scope: keyVault
+  properties: {
+    principalId: managedIdentity.properties.principalId
+    roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '12338af0-0e69-4776-bea7-57ae8d297424') // KeyVault-Crypto-User
+    principalType: 'ServicePrincipal'
+  }
+}
+
 resource keyPermissions 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
   name: guid('msi-${keyVault.id}-${location}-${managedIdentity.id}-KeyVault-Key-Read-RoleAssignment')
   scope: keyVault
@@ -113,13 +130,6 @@ resource proximityPlacementGroup 'Microsoft.Compute/proximityPlacementGroups@202
   location: location
 }
 
-@description('The resource ID of the created Virtual Network Subnet.')
-output subnetResourceIds array = [
-  virtualNetwork.properties.subnets[0].id
-  virtualNetwork.properties.subnets[1].id
-  virtualNetwork.properties.subnets[2].id
-]
-
 resource dnsZone 'Microsoft.Network/dnsZones@2018-05-01' = {
   name: dnsZoneName
   location: 'global'
@@ -160,3 +170,18 @@ output dnsZoneResourceId string = dnsZone.id
 
 @description('The resource ID of the created Log Analytics Workspace.')
 output logAnalyticsWorkspaceResourceId string = logAnalyticsWorkspace.id
+
+@description('The resource ID of the created Key Vault.')
+output keyVaultResourceId string = keyVault.id
+
+@description('The name of the Key Vault Encryption Key.')
+output keyVaultEncryptionKeyName string = keyVault::key.name
+
+@description('The resource ID of the created Virtual Network System Agent Pool Subnet.')
+output systemPoolSubnetResourceId string = virtualNetwork.properties.subnets[0].id
+
+@description('The resource ID of the created Virtual Network Agent Pool 1 Subnet.')
+output agentPool1SubnetResourceId string = virtualNetwork.properties.subnets[1].id
+
+@description('The resource ID of the created Virtual Network Agent Pool 2 Subnet.')
+output agentPool2SubnetResourceId string = virtualNetwork.properties.subnets[2].id

modules/container-service/managed-cluster/tests/e2e/azure/main.test.bicep

@@ -92,7 +92,7 @@ module testDeployment '../../../main.bicep' = {
         storageProfile: 'ManagedDisks'
         type: 'VirtualMachineScaleSets'
         vmSize: 'Standard_DS2_v2'
-        vnetSubnetID: nestedDependencies.outputs.subnetResourceIds[0]
+        vnetSubnetID: nestedDependencies.outputs.systemPoolSubnetResourceId
       }
     ]
     agentPools: [
@@ -119,7 +119,7 @@ module testDeployment '../../../main.bicep' = {
         storageProfile: 'ManagedDisks'
         type: 'VirtualMachineScaleSets'
         vmSize: 'Standard_DS2_v2'
-        vnetSubnetID: nestedDependencies.outputs.subnetResourceIds[1]
+        vnetSubnetID: nestedDependencies.outputs.agentPool1SubnetResourceId
         proximityPlacementGroupResourceId: nestedDependencies.outputs.proximityPlacementGroupResourceId
       }
       {
@@ -145,7 +145,7 @@ module testDeployment '../../../main.bicep' = {
         storageProfile: 'ManagedDisks'
         type: 'VirtualMachineScaleSets'
         vmSize: 'Standard_DS2_v2'
-        vnetSubnetID: nestedDependencies.outputs.subnetResourceIds[2]
+        vnetSubnetID: nestedDependencies.outputs.agentPool2SubnetResourceId
       }
     ]
     autoUpgradeProfileUpgradeChannel: 'stable'
@@ -189,6 +189,11 @@ module testDeployment '../../../main.bicep' = {
     enableAzureDefender: true
     enableKeyvaultSecretsProvider: true
     enablePodSecurityPolicy: false
+    customerManagedKey: {
+      keyName: nestedDependencies.outputs.keyVaultEncryptionKeyName
+      keyVaultNetworkAccess: 'Public'
+      keyVaultResourceId: nestedDependencies.outputs.keyVaultResourceId
+    }
     lock: {
       kind: 'CanNotDelete'
       name: 'myCustomLockName'

modules/container-service/managed-cluster/tests/e2e/priv/dependencies.bicep

@@ -31,14 +31,12 @@ resource virtualNetwork 'Microsoft.Network/virtualNetworks@2023-04-01' = {
         addressPrefix
       ]
     }
-    subnets: [
-      {
-        name: 'defaultSubnet'
+    subnets: map(range(0, 2), i => {
+        name: 'subnet-${i}'
         properties: {
-          addressPrefix: cidrSubnet(addressPrefix, 16, 0)
+          addressPrefix: cidrSubnet(addressPrefix, 24, i)
         }
-      }
-    ]
+      })
   }
 }
 
@@ -85,3 +83,9 @@ output privateDnsZoneResourceId string = privateDnsZone.id
 
 @description('The resource ID of the VirtualNetwork created.')
 output vNetResourceId string = virtualNetwork.id
+
+@description('The resource ID of the created Virtual Network System Agent Pool Subnet.')
+output systemPoolSubnetResourceId string = virtualNetwork.properties.subnets[0].id
+
+@description('The resource ID of the created Virtual Network Agent Pool 1 Subnet.')
+output agentPoolSubnetResourceId string = virtualNetwork.properties.subnets[1].id