add whisper to autoscaler

ansible/nomad-client.yml

@@ -30,6 +30,7 @@
           runtimeArgs: []
     docker_daemon_options_extras: "{{ docker_daemon_nvidia_options if nvidia_docker_flag else {} }}"
     shard_role: nomad-pool
+    nomad_autoscaler_instance_type: "{{ 'whisper' if nomad_pool_type=='whisper' else 'nomad' }}"
     nomad_node_role: client
     nomad_gpu_count: 0
     nomad_node_class: "{{ nomad_pool_type }}"
@@ -145,8 +146,10 @@
     - { role: "logrotate", tags: "logrotate"}
     - { role: "journald", tags: "journald"}
     - { role: "rsyslog", rsyslog_install_flag: false, tags: "rsyslog", rsyslog_tcp_output_enabled: true}
-    - { role: "autoscaler-sidecar", tags: "autoscaler-sidecar",
-        autoscaler_instance_type: "nomad", cloud_provider: "oracle", autoscaler_install_flag: false, autoscaler_configure_flag: true, when: nomad_enable_jitsi_autoscaler}
+    - { role: "whisper", tags: "whisper", when: nomad_pool_type == "whisper" }
+    - { role: "autoscaler-sidecar", 
+        tags: "autoscaler-sidecar",
+        autoscaler_instance_type: "{{ nomad_autoscaler_instance_type }}", cloud_provider: "oracle", autoscaler_install_flag: false, autoscaler_configure_flag: true, when: nomad_enable_jitsi_autoscaler}
     - { role: "consul-agent", tags: "consul", consul_install_flag: false}
     - { role: "nvidia-docker", tags: "nvidia-docker", when: nvidia_docker_flag }
     - { role: "docker-daemon-config", tags: "docker-daemon-config" }

ansible/roles/autoscaler-sidecar/defaults/main.yml

@@ -16,6 +16,7 @@ autoscaler_force_shutdown_command_by_type:
   'jigasi': 'service jigasi stop'
   'sip-jibri': 'service jibri stop'
   'nomad': 'nomad node drain -self -enable -force -detach -yes'
+  'whisper': 'nomad node drain -self -enable -force -detach -yes'
 autoscaler_git_repo: https://github.com/jitsi/jitsi-autoscaler-sidecar.git
 autoscaler_graceful_shutdown_script: "{{
   autoscaler_graceful_shutdown_scripts[autoscaler_instance_type]
@@ -26,14 +27,16 @@ autoscaler_graceful_shutdown_scripts:
   'jigasi': /usr/local/bin/graceful-shutdown-wrapper-jigasi.sh
   'sip-jibri': /opt/jitsi/jibri/wait_graceful_shutdown.sh
   'nomad': /usr/local/bin/nomad_graceful_shutdown.sh
+  'whisper': /usr/local/bin/whisper-graceful-shutdown.sh
 autoscaler_graceful_shutdown_wrapper: /usr/local/bin/graceful_shutdown_wrapper.sh
 autoscaler_group: "default"
 autoscaler_groupname: jsidecar
 autoscaler_health_script: "/usr/local/bin/autoscaler-sidecar-health.sh"
 autoscaler_health_url: "http://localhost:{{ autoscaler_express_port }}/health"
 autoscaler_install_flag: true
 autoscaler_instance_id: "{{
-  ansible_ec2_instance_id if ansible_ec2_instance_id is defined
+  cloud_instance_id if cloud_instance_id is defined
+  else ansible_ec2_instance_id if ansible_ec2_instance_id is defined
   else oracle_instance_id if oracle_instance_id is defined else 'default' }}"
 autoscaler_instance_type: 'JVB'
 autoscaler_key_id: "{{ asap_key['id'] if asap_key['id'] is defined else 'default' }}"
@@ -67,6 +70,7 @@ autoscaler_sidecar_instance_version_command_by_type:
   'jigasi': "dpkg -s jigasi | grep Version | awk '{print $2}' | sed 's/..$//'"
   'sip-jibri': "dpkg -s jibri | grep Version | awk '{print $2}' | sed 's/..$//'"
   'nomad': "nomad --version | head -1 | cut -d' ' -f2 | tr -d 'v'"
+  'whisper': "nomad --version | head -1 | cut -d' ' -f2 | tr -d 'v'"
 autoscaler_sidecar_private_ip: "{{ ansible_default_ipv4.address }}"
 autoscaler_sidecar_public_ip: "{{ ansible_ec2_public_ipv4 if ansible_ec2_public_ipv4 is defined else false }}"
 autoscaler_sidecar_service_name: jitsi-autoscaler-sidecar
@@ -80,6 +84,7 @@ autoscaler_stats_retrieve_urls:
   'jibri': http://localhost:2222/jibri/api/v1.0/health
   'jigasi': http://localhost:8788/about/stats
   'sip-jibri': http://localhost:2222/jibri/api/v1.0/health
+  'whisper': http://{{ ansible_default_ipv4.address }}:8003/state
 autoscaler_status_url: "https://{{ autoscaler_server_host }}/sidecar/status"
 autoscaler_shutdown_url: "https://{{ autoscaler_server_host }}/sidecar/shutdown"
 autoscaler_terminate_script: '/usr/local/bin/terminate_instance.sh'

ansible/roles/jigasi/defaults/main.yml

@@ -170,7 +170,7 @@ jigasi_transcriber_remote_select_aud: "jitsi"
 jigasi_transcriber_oci_flag: false
 jigasi_transcriber_whisper_private_key: "replaceme"
 jigasi_transcriber_whisper_private_key_name: "replaceme"
-jigasi_transcriber_whisper_websocket_url: "wss://{{ hcv_environment }}-{{ oracle_region }}-whisper.jitsi.net:443/streaming-whisper/ws/"
+jigasi_transcriber_whisper_websocket_url: "wss://{{ hcv_environment }}-{{ oracle_region }}-whisper.jitsi.net:443/whisper/ws/"
 jigasi_transcription_path: /var/lib/jigasi/transcripts
 jigasi_trusted_domains: false
 jigasi_udp_buffer_max_backlog: 2000

ansible/roles/whisper/tasks/main.yml

@@ -0,0 +1,6 @@
+- name: Copy graceful shutdown script
+  ansible.builtin.template:
+    src: whisper-graceful-shutdown.sh.j2
+    dest: /usr/local/bin/whisper-graceful-shutdown.sh
+    mode: 0755
+    owner: root

ansible/roles/whisper/templates/whisper-graceful-shutdown.sh.j2

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# Sets the server to drain mode and waits for all the connections to be closed before exiting
+# with code 0 (clean shutdown). If the connections are not closed within the specified time,
+# the script will exit with code 1 (dirty) and the autoscaler will shut down the instance anyway.
+
+# It takes two parameters: WAIT_SECONDS and STATE_URL
+# WAIT_SECONDS: Seconds to wait for the server to enter drain mode and close all connections.
+#               Defaults to 100.
+# STATE_URL:    The URL of the whisper state endpoint. Defaults to http://localhost:8003/state.
+
+# Example usage: ./shutdown_script.sh 100 http://<local ip>:8003/state
+
+
+WAIT_SECONDS=100
+STATE_URL="http://{{ ansible_default_ipv4.address }}:8003/state"
+
+[[ -z $1 ]] && echo "Parameter WAIT_SECONDS not set, using default $WAIT_SECONDS" || WAIT_SECONDS=$1
+[[ -z $2 ]] && echo "Parameter STATE_URL not set, using default $STATE_URL" || STATE_URL=$2
+
+echo "Waiting for $WAIT_SECONDS seconds for the server to enter drain mode and close all connections."
+
+START_TIME=$(date +%s)
+
+while true; do
+  RESPONSE=$(curl -s $STATE_URL)
+  WHISPER_STATE=$(echo $RESPONSE | jq -r '.state')
+  ACTIVE_CONNECTIONS=$(echo $RESPONSE | jq -r '.connections')
+  if [[ $WHISPER_STATE != "drain" ]]; then
+    echo "Setting server in drain mode."
+    curl -s -XPOST $STATE_URL -d '{"state": "drain"}' | jq -r '.request_status'
+  else
+    echo "Server is in drain mode. Waiting for $ACTIVE_CONNECTIONS connections to be closed before shutting down"
+    if [[ $ACTIVE_CONNECTIONS -eq 0 ]]; then
+      echo "All connections are closed. Shutting down the instance."
+      exit 0
+    fi
+  fi
+  CURRENT_TIME=$(date +%s)
+  ELAPSED_TIME=$((CURRENT_TIME - START_TIME))
+  if [[ $ELAPSED_TIME -gt $WAIT_SECONDS ]]; then
+    break
+  fi
+  sleep 2
+done
+
+echo "Server did not close all connections within $WAIT_SECONDS seconds. Shutting down anyway."
+exit 1