Azure VM template fails to deploy after CasC update, randomly, and after controller reboot until each template is manually verified.

[AD-Code]

Jenkins and plugins versions report

Environment

Jenkins: 2.426.3
OS: Linux - 4.18.0-513.5.1.el8_9.x86_64
Java: 17.0.9 - Eclipse Adoptium (OpenJDK 64-Bit Server VM)
---
Office-365-Connector:4.21.0
adoptopenjdk:1.5
ansicolor:1.0.4
ant:497.v94e7d9fffa_b_9
antisamy-markup-formatter:162.v0e6ec0fcfcf6
apache-httpcomponents-client-4-api:4.5.14-208.v438351942757
apache-httpcomponents-client-5-api:5.3.1-1.0
artifactory:4.0.5
asm-api:9.6-3.v2e1fa_b_338cd7
authentication-tokens:1.53.v1c90fd9191a_b_
authorize-project:1.7.1
azure-container-agents:253.vd2f5cd5c5040
azure-credentials:312.v0f3973cd1e59
azure-sdk:157.v855da_0b_eb_dc2
azure-vm-agents:901.ved986df424b_3
blueocean:1.27.11
blueocean-bitbucket-pipeline:1.27.11
blueocean-commons:1.27.11
blueocean-config:1.27.11
blueocean-core-js:1.27.11
blueocean-dashboard:1.27.11
blueocean-display-url:2.4.2
blueocean-events:1.27.11
blueocean-git-pipeline:1.27.11
blueocean-github-pipeline:1.27.11
blueocean-i18n:1.27.11
blueocean-jwt:1.27.11
blueocean-personalization:1.27.11
blueocean-pipeline-api-impl:1.27.11
blueocean-pipeline-editor:1.27.11
blueocean-pipeline-scm-api:1.27.11
blueocean-rest:1.27.11
blueocean-rest-impl:1.27.11
blueocean-web:1.27.11
bootstrap5-api:5.3.3-1
bouncycastle-api:2.30.1.77-225.v26ea_c9455fd9
branch-api:2.1152.v6f101e97dd77
build-name-setter:2.4.2
build-timeout:1.32
build-timestamp:1.0.3
caffeine-api:3.1.8-133.v17b_1ff2e0599
checks-api:2.0.2
cloud-stats:336.v788e4055508b_
cloudbees-bitbucket-branch-source:880.vcf4056c5a_71f
cloudbees-folder:6.858.v898218f3609d
command-launcher:107.v773860566e2e
commons-httpclient3-api:3.1-3
commons-lang3-api:3.13.0-62.v7d18e55f51e2
commons-text-api:1.11.0-95.v22a_d30ee5d36
conditional-buildstep:1.4.3
config-file-provider:968.ve1ca_eb_913f8c
configuration-as-code:1775.v810dc950b_514
copyartifact:722.v0662a_9b_e22a_c
credentials:1337.v60b_d7b_c7b_c9f
credentials-binding:657.v2b_19db_7d6e6d
dashboard-view:2.508.va_74654f026d1
data-tables-api:2.0.2-1
delivery-pipeline-plugin:1.4.2
display-url-api:2.200.vb_9327d658781
docker-build-step:2.11
docker-commons:439.va_3cb_0a_6a_fb_29
docker-java-api:3.3.4-86.v39b_a_5ede342c
docker-plugin:1.5
docker-workflow:572.v950f58993843
durable-task:550.v0930093c4b_a_6
echarts-api:5.5.0-1
email-ext:2.105
embeddable-build-status:467.v4a_954796e45d
external-monitor-job:215.v2e88e894db_f8
external-workspace-manager:1.3.1
favorite:2.208.v91d65b_7792a_c
font-awesome-api:6.5.1-3
fortify:23.1.40
generic-webhook-trigger:2.1.0
git:5.2.1
git-client:4.7.0
git-server:114.v068a_c7cc2574
github:1.38.0
github-api:1.318-461.v7a_c09c9fa_d63
github-branch-source:1781.va_153cda_09d1b_
github-pullrequest:0.7.0
gradle:2.10
greenballs:1.15.1
gson-api:2.10.1-15.v0d99f670e0a_7
handy-uri-templates-2-api:2.1.8-30.v7e777411b_148
hp-application-automation-tools-plugin:24.1.0
htmlpublisher:1.33
instance-identity:185.v303dc7c645f9
ionicons-api:56.v1b_1c8c49374e
ivy:2.5
jackson2-api:2.17.0-379.v02de8ec9f64c
jakarta-activation-api:2.1.3-1
jakarta-mail-api:2.1.3-1
javadoc:243.vb_b_503b_b_45537
javax-activation-api:1.2.0-6
javax-mail-api:1.6.2-9
jaxb:2.3.9-1
jdk-tool:73.vddf737284550
jenkins-design-language:1.27.11
jersey2-api:2.41-133.va_03323b_a_1396
jira:3.13
jjwt-api:0.11.5-77.v646c772fddb_0
job-dsl:1.87
job-restrictions:0.8
jobConfigHistory:1229.v3039470161a_d
joda-time-api:2.12.7-29.v5a_b_e3a_82269a_
jquery:1.12.4-1
jquery3-api:3.7.1-2
jsch:0.2.16-86.v42e010d9484b_
json-api:20240303-41.v94e11e6de726
json-path-api:2.9.0-58.v62e3e85b_a_655
junit:1259.v65ffcef24a_88
kubernetes:4203.v1dd44f5b_1cf9
kubernetes-client-api:6.10.0-240.v57880ce8b_0b_2
kubernetes-credentials:0.11
ldap:719.vcb_d039b_77d0d
lockable-resources:1245.vb_05f8a_4e28db_
mail-watcher-plugin:1.19
mailer:470.vc91f60c5d8e2
matrix-auth:3.2.2
matrix-project:822.824.v14451b_c0fd42
maven-plugin:3.23
mercurial:1260.vdfb_723cdcc81
metrics:4.2.21-449.v6960d7c54c69
mina-sshd-api-common:2.12.0-90.v9f7fb_9fa_3d3b_
mina-sshd-api-core:2.12.0-90.v9f7fb_9fa_3d3b_
msbuild:1.30
mstest:1.0.5
mstestrunner:1.5.0
next-build-number:1.8
nexus-jenkins-plugin:3.14.431.v37ca_dc788b_b_1
nodejs:1.6.1
nodelabelparameter:1.12.0
okhttp-api:4.11.0-172.vda_da_1feeb_c6e
pam-auth:1.10
parameterized-trigger:787.v665fcf2a_830b_
pipeline-build-step:540.vb_e8849e1a_b_d8
pipeline-github-lib:42.v0739460cda_c4
pipeline-graph-analysis:216.vfd8b_ece330ca_
pipeline-groovy-lib:704.vc58b_8890a_384
pipeline-input-step:477.v339683a_8d55e
pipeline-milestone-step:111.v449306f708b_7
pipeline-model-api:2.2184.v0b_358b_953e69
pipeline-model-definition:2.2184.v0b_358b_953e69
pipeline-model-extensions:2.2184.v0b_358b_953e69
pipeline-rest-api:2.34
pipeline-stage-step:305.ve96d0205c1c6
pipeline-stage-tags-metadata:2.2184.v0b_358b_953e69
pipeline-stage-view:2.34
pipeline-utility-steps:2.16.2
plain-credentials:179.vc5cb_98f6db_38
plugin-util-api:4.1.0
powershell:2.1
prism-api:1.29.0-13
prometheus:2.5.1
pubsub-light:1.18
rebuild:330.v645b_7df10e2a_
resource-disposer:0.23
role-strategy:713.vb_3837801b_8cc
run-condition:1.7
scm-api:689.v237b_6d3a_ef7f
scmskip:50.vfb_3a_f04242a_a_
script-security:1326.vdb_c154de8669
shared-workspace:1.0.2
shiningpanda:0.24
short-workspace-path:0.3
snakeyaml-api:2.2-111.vc6598e30cc65
sonar:2.17.2
sse-gateway:1.26
ssh-agent:346.vda_a_c4f2c8e50
ssh-credentials:326.v7fcb_a_ef6194b_
ssh-slaves:2.948.vb_8050d697fec
sshd:3.322.v159e91f6a_550
structs:337.v1b_04ea_4df7c8
templating-engine:2.5.3
throttle-concurrents:2.14
timestamper:1.26
token-macro:400.v35420b_922dcb_
trilead-api:2.142.v748523a_76693
validating-string-parameter:183.v3748e79b_9737
variant:60.v7290fc0eb_b_cd
versionnumber:1.11
vstestrunner:1.0.8
windows-azure-storage:419.v4046cd70d2e3
workflow-aggregator:596.v8c21c963d92d
workflow-api:1291.v51fd2a_625da_7
workflow-basic-steps:1049.v257a_e6b_30fb_d
workflow-cps:3883.vb_3ff2a_e3eea_f
workflow-durable-task-step:1331.vc8c2fed35334
workflow-job:1385.vb_58b_86ea_fff1
workflow-multibranch:773.vc4fe1378f1d5
workflow-scm-step:427.v4ca_6512e7df1
workflow-step-api:657.v03b_e8115821b_
workflow-support:881.v7663695646cf
ws-cleanup:0.45

What Operating System are you using (both controller, and any agents involved in the problem)?

All controllers are running in kubernetes on linux os. Agent templates are both windows and linux.

Reproduction steps

Several things trigger this situation.

1. CasC files are updated and reloaded or simply reloaded with no changes.
2. Controller pod is restarted
3. It just randomly occurs with no known trigger

Once those conditions occur to get it working again the 'verify template' button must be manually clicked. Once that occurs things start working as normal.

Expected Results

CasC is updated or reloaded, controller is rebooted, or time passes and I would hope that the verify template function works without manual intervention.

Actual Results

The templates shouldn't randomly stop working and I shouldn't have to monitor each controller and manually click a button to get the templates verified. The logs show this:
Nov 06, 2024 11:42:04 AM WARNING com.microsoft.azure.vmagent.AzureVMMaintainPoolTask provisionNodes
Template failed to verify, cannot be provisioned

To me it reads like the job to verify the template behind the scenes runs but fails. I would expect that the scheduled task to verify the template is the same task that gets executed when the 'verify template' button is pressed.

Anything else?

No response

Are you interested in contributing a fix?

No response

[johnl2323]

Please also provide the Azure Cloud JcasC that you are using. Thanks.

[AD-Code]

jenkins:
  clouds:
  - azureVM:
      azureCredentialsId: "<creds id>"
      configurationStatus: "pass"
      deploymentTimeout: 1800
      existingResourceGroupName: "<azure RG that houses all resources>"
      maxVirtualMachinesLimit: 5
      name: "<cloud name>"
      resourceGroupReferenceType: "existing"
      vmTemplates:
      - agentWorkspace: "/var/jenkins_home"
        builtInImage: "Ubuntu 22.04 LTS"
        credentialsId: "azureCreds"
        diskType: "managed"
        doNotUseMachineIfInitFails: true
        enableUAMI: true
        executeInitScriptAsRoot: true
        existingStorageAccountName: "<azure storage account name>"
        imageReference:
          galleryImageDefinition: "<gallery image def>"
          galleryImageVersion: "latest"
          galleryName: "<gallery library>"
          galleryResourceGroup: "<gallery rg>"
          gallerySubscriptionId: "<gallery sub>"
        imageTopLevelType: "advanced"
        initScript: |-
          #!/bin/bash
          echo 'sleep 60'
          sleep 60
          continued...
        javaPath: "java"
        labels: "linux-docker"
        location: "Central US"
        maxVirtualMachinesLimit: 2
        maximumDeploymentSize: 1
        noOfParallelJobs: 3
        osDiskSize: 200
        osDiskStorageAccountType: "Premium_LRS"
        osType: "Linux"
        retentionStrategy:
          azureVMCloudPool:
            poolSize: 2
            retentionInHours: 4
        storageAccountNameReferenceType: "existing"
        storageAccountType: "Standard_LRS"
        subnetName: "<vnet subnet>"
        tags:
        - name: "CustomTag"
          value: "CustomValue"
        templateDesc: "Linux agents"
        templateName: "LinuxVM"
        uamiID: "<azure id of user managed identity>"
        usageMode: NORMAL
        usePrivateIP: true
        virtualMachineSize: "Standard_D8s_v5"
        virtualNetworkName: "<vnet name>"
        virtualNetworkResourceGroupName: "<vnet rg>"

[timja]

Likely duplicate of #567

Please upgrade to https://github.com/jenkinsci/azure-vm-agents-plugin/releases/tag/971.v88fcc9f5167c