From 86c0812fe5d5bc9db3fb374a84893c05f135bee6 Mon Sep 17 00:00:00 2001 From: RazvanP Date: Wed, 11 Dec 2024 12:49:54 +0200 Subject: [PATCH 1/5] add whisper to autoscaler --- ansible/build-gpu-oracle.yml | 3 ++ .../autoscaler-sidecar/defaults/main.yml | 3 ++ .../streaming-whisper-graceful-shutdown.sh | 48 +++++++++++++++++++ .../roles/streaming-whisper/tasks/main.yml | 13 +++++ 4 files changed, 67 insertions(+) create mode 100644 ansible/roles/streaming-whisper/files/streaming-whisper-graceful-shutdown.sh create mode 100644 ansible/roles/streaming-whisper/tasks/main.yml diff --git a/ansible/build-gpu-oracle.yml b/ansible/build-gpu-oracle.yml index 7b28d47a..91dfaaf9 100644 --- a/ansible/build-gpu-oracle.yml +++ b/ansible/build-gpu-oracle.yml @@ -39,4 +39,7 @@ # docker pull most recent running version of skynet and whisper - { role: "gpu-docker-pull", tags: "gpu-docker", gpu_docker_type: "skynet" } - { role: "gpu-docker-pull", tags: "gpu-docker", gpu_docker_type: "whisper" } + - { role: "streaming-whisper", tags: "gpu-docker", gpu_docker_type: "whisper" } + - { role: "autoscaler-sidecar", tags: "autoscaler-sidecar", + autoscaler_instance_type: "streaming-whisper", autoscaler_install_flag: true, autoscaler_configure_flag: false, jitsi_autoscaler_sidecar: true} - { role: "clean-system", tags: "clean-system, build" } diff --git a/ansible/roles/autoscaler-sidecar/defaults/main.yml b/ansible/roles/autoscaler-sidecar/defaults/main.yml index 5cea487d..37a896cc 100644 --- a/ansible/roles/autoscaler-sidecar/defaults/main.yml +++ b/ansible/roles/autoscaler-sidecar/defaults/main.yml @@ -16,6 +16,7 @@ autoscaler_force_shutdown_command_by_type: 'jigasi': 'service jigasi stop' 'sip-jibri': 'service jibri stop' 'nomad': 'nomad node drain -self -enable -force -detach -yes' + 'streaming-whisper': 'nomad node drain -self -enable -force -detach -yes' autoscaler_git_repo: https://github.com/jitsi/jitsi-autoscaler-sidecar.git autoscaler_graceful_shutdown_script: "{{ autoscaler_graceful_shutdown_scripts[autoscaler_instance_type] @@ -26,6 +27,7 @@ autoscaler_graceful_shutdown_scripts: 'jigasi': /usr/local/bin/graceful-shutdown-wrapper-jigasi.sh 'sip-jibri': /opt/jitsi/jibri/wait_graceful_shutdown.sh 'nomad': /usr/local/bin/nomad_graceful_shutdown.sh + 'streaming-whisper': /usr/local/bin/streaming-whisper-graceful-shutdown.sh autoscaler_graceful_shutdown_wrapper: /usr/local/bin/graceful_shutdown_wrapper.sh autoscaler_group: "default" autoscaler_groupname: jsidecar @@ -80,6 +82,7 @@ autoscaler_stats_retrieve_urls: 'jibri': http://localhost:2222/jibri/api/v1.0/health 'jigasi': http://localhost:8788/about/stats 'sip-jibri': http://localhost:2222/jibri/api/v1.0/health + 'streaming-whisper': http://localhost:8003/state autoscaler_status_url: "https://{{ autoscaler_server_host }}/sidecar/status" autoscaler_shutdown_url: "https://{{ autoscaler_server_host }}/sidecar/shutdown" autoscaler_terminate_script: '/usr/local/bin/terminate_instance.sh' diff --git a/ansible/roles/streaming-whisper/files/streaming-whisper-graceful-shutdown.sh b/ansible/roles/streaming-whisper/files/streaming-whisper-graceful-shutdown.sh new file mode 100644 index 00000000..c10d625e --- /dev/null +++ b/ansible/roles/streaming-whisper/files/streaming-whisper-graceful-shutdown.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +# Sets the server to drain mode and waits for all the connections to be closed before exiting +# with code 0 (clean shutdown). If the connections are not closed within the specified time, +# the script will exit with code 1 (dirty) and the autoscaler will shut down the instance anyway. + +# It takes two parameters: WAIT_SECONDS and STATE_URL +# WAIT_SECONDS: Seconds to wait for the server to enter drain mode and close all connections. +# Defaults to 100. +# STATE_URL: The URL of the whisper state endpoint. Defaults to http://localhost:8003/state. + +# Example usage: ./shutdown_script.sh 100 http://localhost:8003/state + + +WAIT_SECONDS=100 +STATE_URL="http://localhost:8003/state" + +[[ -z $1 ]] && echo "Parameter WAIT_SECONDS not set, using default $WAIT_SECONDS" || WAIT_SECONDS=$1 +[[ -z $2 ]] && echo "Parameter STATE_URL not set, using default $STATE_URL" || STATE_URL=$2 + +echo "Waiting for $WAIT_SECONDS seconds for the server to enter drain mode and close all connections." + +START_TIME=$(date +%s) + +while true; do + RESPONSE=$(curl -s $STATE_URL) + WHISPER_STATE=$(echo $RESPONSE | jq -r '.state') + ACTIVE_CONNECTIONS=$(echo $RESPONSE | jq -r '.connections') + if [[ $WHISPER_STATE != "drain" ]]; then + echo "Setting server in drain mode." + curl -s -XPOST $STATE_URL -d '{"state": "drain"}' | jq -r '.request_status' + else + echo "Server is in drain mode. Waiting for $ACTIVE_CONNECTIONS connections to be closed before shutting down" + if [[ $ACTIVE_CONNECTIONS -eq 0 ]]; then + echo "All connections are closed. Shutting down the instance." + exit 0 + fi + fi + CURRENT_TIME=$(date +%s) + ELAPSED_TIME=$((CURRENT_TIME - START_TIME)) + if [[ $ELAPSED_TIME -gt $WAIT_SECONDS ]]; then + break + fi + sleep 2 +done + +echo "Server did not close all connections within $WAIT_SECONDS seconds. Shutting down anyway." +exit 1 diff --git a/ansible/roles/streaming-whisper/tasks/main.yml b/ansible/roles/streaming-whisper/tasks/main.yml new file mode 100644 index 00000000..6df8fe4e --- /dev/null +++ b/ansible/roles/streaming-whisper/tasks/main.yml @@ -0,0 +1,13 @@ +- name: Install dependencies + ansible.builtin.package: + name: + - jq + - curl + state: latest + +- name: Copy graceful shutdown script + ansible.builtin.copy: + src: streaming-whisper-graceful-shutdown.sh + dest: /usr/local/bin/streaming-whisper-graceful-shutdown.sh + mode: 0755 + owner: root From 475f5628b095b977940375cd412e20f0dbd57ebb Mon Sep 17 00:00:00 2001 From: RazvanP Date: Wed, 11 Dec 2024 16:16:39 +0200 Subject: [PATCH 2/5] Renamed the role to whisper --- ansible/build-gpu-oracle.yml | 4 +--- ansible/roles/autoscaler-sidecar/defaults/main.yml | 6 +++--- ansible/roles/jigasi/defaults/main.yml | 2 +- ansible/roles/streaming-whisper/tasks/main.yml | 13 ------------- .../files/whisper-graceful-shutdown.sh} | 0 ansible/roles/whisper/tasks/main.yml | 6 ++++++ 6 files changed, 11 insertions(+), 20 deletions(-) delete mode 100644 ansible/roles/streaming-whisper/tasks/main.yml rename ansible/roles/{streaming-whisper/files/streaming-whisper-graceful-shutdown.sh => whisper/files/whisper-graceful-shutdown.sh} (100%) create mode 100644 ansible/roles/whisper/tasks/main.yml diff --git a/ansible/build-gpu-oracle.yml b/ansible/build-gpu-oracle.yml index 91dfaaf9..51b3594e 100644 --- a/ansible/build-gpu-oracle.yml +++ b/ansible/build-gpu-oracle.yml @@ -39,7 +39,5 @@ # docker pull most recent running version of skynet and whisper - { role: "gpu-docker-pull", tags: "gpu-docker", gpu_docker_type: "skynet" } - { role: "gpu-docker-pull", tags: "gpu-docker", gpu_docker_type: "whisper" } - - { role: "streaming-whisper", tags: "gpu-docker", gpu_docker_type: "whisper" } - - { role: "autoscaler-sidecar", tags: "autoscaler-sidecar", - autoscaler_instance_type: "streaming-whisper", autoscaler_install_flag: true, autoscaler_configure_flag: false, jitsi_autoscaler_sidecar: true} + - { role: "whisper", tags: "gpu-docker" } - { role: "clean-system", tags: "clean-system, build" } diff --git a/ansible/roles/autoscaler-sidecar/defaults/main.yml b/ansible/roles/autoscaler-sidecar/defaults/main.yml index 37a896cc..c4ae4982 100644 --- a/ansible/roles/autoscaler-sidecar/defaults/main.yml +++ b/ansible/roles/autoscaler-sidecar/defaults/main.yml @@ -16,7 +16,7 @@ autoscaler_force_shutdown_command_by_type: 'jigasi': 'service jigasi stop' 'sip-jibri': 'service jibri stop' 'nomad': 'nomad node drain -self -enable -force -detach -yes' - 'streaming-whisper': 'nomad node drain -self -enable -force -detach -yes' + 'whisper': 'nomad node drain -self -enable -force -detach -yes' autoscaler_git_repo: https://github.com/jitsi/jitsi-autoscaler-sidecar.git autoscaler_graceful_shutdown_script: "{{ autoscaler_graceful_shutdown_scripts[autoscaler_instance_type] @@ -27,7 +27,7 @@ autoscaler_graceful_shutdown_scripts: 'jigasi': /usr/local/bin/graceful-shutdown-wrapper-jigasi.sh 'sip-jibri': /opt/jitsi/jibri/wait_graceful_shutdown.sh 'nomad': /usr/local/bin/nomad_graceful_shutdown.sh - 'streaming-whisper': /usr/local/bin/streaming-whisper-graceful-shutdown.sh + 'whisper': /usr/local/bin/whisper-graceful-shutdown.sh autoscaler_graceful_shutdown_wrapper: /usr/local/bin/graceful_shutdown_wrapper.sh autoscaler_group: "default" autoscaler_groupname: jsidecar @@ -82,7 +82,7 @@ autoscaler_stats_retrieve_urls: 'jibri': http://localhost:2222/jibri/api/v1.0/health 'jigasi': http://localhost:8788/about/stats 'sip-jibri': http://localhost:2222/jibri/api/v1.0/health - 'streaming-whisper': http://localhost:8003/state + 'whisper': http://localhost:8003/state autoscaler_status_url: "https://{{ autoscaler_server_host }}/sidecar/status" autoscaler_shutdown_url: "https://{{ autoscaler_server_host }}/sidecar/shutdown" autoscaler_terminate_script: '/usr/local/bin/terminate_instance.sh' diff --git a/ansible/roles/jigasi/defaults/main.yml b/ansible/roles/jigasi/defaults/main.yml index 3209a067..025e81f0 100644 --- a/ansible/roles/jigasi/defaults/main.yml +++ b/ansible/roles/jigasi/defaults/main.yml @@ -170,7 +170,7 @@ jigasi_transcriber_remote_select_aud: "jitsi" jigasi_transcriber_oci_flag: false jigasi_transcriber_whisper_private_key: "replaceme" jigasi_transcriber_whisper_private_key_name: "replaceme" -jigasi_transcriber_whisper_websocket_url: "wss://{{ hcv_environment }}-{{ oracle_region }}-whisper.jitsi.net:443/streaming-whisper/ws/" +jigasi_transcriber_whisper_websocket_url: "wss://{{ hcv_environment }}-{{ oracle_region }}-whisper.jitsi.net:443/whisper/ws/" jigasi_transcription_path: /var/lib/jigasi/transcripts jigasi_trusted_domains: false jigasi_udp_buffer_max_backlog: 2000 diff --git a/ansible/roles/streaming-whisper/tasks/main.yml b/ansible/roles/streaming-whisper/tasks/main.yml deleted file mode 100644 index 6df8fe4e..00000000 --- a/ansible/roles/streaming-whisper/tasks/main.yml +++ /dev/null @@ -1,13 +0,0 @@ -- name: Install dependencies - ansible.builtin.package: - name: - - jq - - curl - state: latest - -- name: Copy graceful shutdown script - ansible.builtin.copy: - src: streaming-whisper-graceful-shutdown.sh - dest: /usr/local/bin/streaming-whisper-graceful-shutdown.sh - mode: 0755 - owner: root diff --git a/ansible/roles/streaming-whisper/files/streaming-whisper-graceful-shutdown.sh b/ansible/roles/whisper/files/whisper-graceful-shutdown.sh similarity index 100% rename from ansible/roles/streaming-whisper/files/streaming-whisper-graceful-shutdown.sh rename to ansible/roles/whisper/files/whisper-graceful-shutdown.sh diff --git a/ansible/roles/whisper/tasks/main.yml b/ansible/roles/whisper/tasks/main.yml new file mode 100644 index 00000000..db40dcbb --- /dev/null +++ b/ansible/roles/whisper/tasks/main.yml @@ -0,0 +1,6 @@ +- name: Copy graceful shutdown script + ansible.builtin.copy: + src: whisper-graceful-shutdown.sh + dest: /usr/local/bin/whisper-graceful-shutdown.sh + mode: 0755 + owner: root From b3146a9708ad5ab544f9f2253a985bd61a4d77c1 Mon Sep 17 00:00:00 2001 From: Aaron van Meerten Date: Wed, 11 Dec 2024 09:24:45 -0600 Subject: [PATCH 3/5] fix state url host --- ansible/roles/whisper/tasks/main.yml | 4 ++-- .../whisper-graceful-shutdown.sh.j2} | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename ansible/roles/whisper/{files/whisper-graceful-shutdown.sh => templates/whisper-graceful-shutdown.sh.j2} (92%) diff --git a/ansible/roles/whisper/tasks/main.yml b/ansible/roles/whisper/tasks/main.yml index db40dcbb..d6ccab9d 100644 --- a/ansible/roles/whisper/tasks/main.yml +++ b/ansible/roles/whisper/tasks/main.yml @@ -1,6 +1,6 @@ - name: Copy graceful shutdown script - ansible.builtin.copy: - src: whisper-graceful-shutdown.sh + ansible.builtin.template: + src: whisper-graceful-shutdown.sh.j2 dest: /usr/local/bin/whisper-graceful-shutdown.sh mode: 0755 owner: root diff --git a/ansible/roles/whisper/files/whisper-graceful-shutdown.sh b/ansible/roles/whisper/templates/whisper-graceful-shutdown.sh.j2 similarity index 92% rename from ansible/roles/whisper/files/whisper-graceful-shutdown.sh rename to ansible/roles/whisper/templates/whisper-graceful-shutdown.sh.j2 index c10d625e..d0ade4ca 100644 --- a/ansible/roles/whisper/files/whisper-graceful-shutdown.sh +++ b/ansible/roles/whisper/templates/whisper-graceful-shutdown.sh.j2 @@ -9,11 +9,11 @@ # Defaults to 100. # STATE_URL: The URL of the whisper state endpoint. Defaults to http://localhost:8003/state. -# Example usage: ./shutdown_script.sh 100 http://localhost:8003/state +# Example usage: ./shutdown_script.sh 100 http://:8003/state WAIT_SECONDS=100 -STATE_URL="http://localhost:8003/state" +STATE_URL="http://{{ ansible_default_ipv4.address }}:8003/state" [[ -z $1 ]] && echo "Parameter WAIT_SECONDS not set, using default $WAIT_SECONDS" || WAIT_SECONDS=$1 [[ -z $2 ]] && echo "Parameter STATE_URL not set, using default $STATE_URL" || STATE_URL=$2 From d6cac1e21a2a9e42a5a65e454838c0bb51a91532 Mon Sep 17 00:00:00 2001 From: Aaron van Meerten Date: Wed, 11 Dec 2024 09:37:02 -0600 Subject: [PATCH 4/5] updates just because --- ansible/build-gpu-oracle.yml | 1 - ansible/nomad-client.yml | 7 +++++-- ansible/roles/autoscaler-sidecar/defaults/main.yml | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/ansible/build-gpu-oracle.yml b/ansible/build-gpu-oracle.yml index 51b3594e..7b28d47a 100644 --- a/ansible/build-gpu-oracle.yml +++ b/ansible/build-gpu-oracle.yml @@ -39,5 +39,4 @@ # docker pull most recent running version of skynet and whisper - { role: "gpu-docker-pull", tags: "gpu-docker", gpu_docker_type: "skynet" } - { role: "gpu-docker-pull", tags: "gpu-docker", gpu_docker_type: "whisper" } - - { role: "whisper", tags: "gpu-docker" } - { role: "clean-system", tags: "clean-system, build" } diff --git a/ansible/nomad-client.yml b/ansible/nomad-client.yml index 67a4ee69..06d0e14d 100644 --- a/ansible/nomad-client.yml +++ b/ansible/nomad-client.yml @@ -30,6 +30,7 @@ runtimeArgs: [] docker_daemon_options_extras: "{{ docker_daemon_nvidia_options if nvidia_docker_flag else {} }}" shard_role: nomad-pool + nomad_autoscaler_instance_type: "{{ 'whisper' if nomad_pool_type=='whisper' else 'nomad' }}" nomad_node_role: client nomad_gpu_count: 0 nomad_node_class: "{{ nomad_pool_type }}" @@ -145,8 +146,10 @@ - { role: "logrotate", tags: "logrotate"} - { role: "journald", tags: "journald"} - { role: "rsyslog", rsyslog_install_flag: false, tags: "rsyslog", rsyslog_tcp_output_enabled: true} - - { role: "autoscaler-sidecar", tags: "autoscaler-sidecar", - autoscaler_instance_type: "nomad", cloud_provider: "oracle", autoscaler_install_flag: false, autoscaler_configure_flag: true, when: nomad_enable_jitsi_autoscaler} + - { role: "whisper", tags: "whisper", when: nomad_pool_type == "whisper" } + - { role: "autoscaler-sidecar", + tags: "autoscaler-sidecar", + autoscaler_instance_type: "{{ nomad_autoscaler_instance_type }}", cloud_provider: "oracle", autoscaler_install_flag: false, autoscaler_configure_flag: true, when: nomad_enable_jitsi_autoscaler} - { role: "consul-agent", tags: "consul", consul_install_flag: false} - { role: "nvidia-docker", tags: "nvidia-docker", when: nvidia_docker_flag } - { role: "docker-daemon-config", tags: "docker-daemon-config" } diff --git a/ansible/roles/autoscaler-sidecar/defaults/main.yml b/ansible/roles/autoscaler-sidecar/defaults/main.yml index c4ae4982..898a5ebe 100644 --- a/ansible/roles/autoscaler-sidecar/defaults/main.yml +++ b/ansible/roles/autoscaler-sidecar/defaults/main.yml @@ -69,6 +69,7 @@ autoscaler_sidecar_instance_version_command_by_type: 'jigasi': "dpkg -s jigasi | grep Version | awk '{print $2}' | sed 's/..$//'" 'sip-jibri': "dpkg -s jibri | grep Version | awk '{print $2}' | sed 's/..$//'" 'nomad': "nomad --version | head -1 | cut -d' ' -f2 | tr -d 'v'" + 'whisper': "nomad --version | head -1 | cut -d' ' -f2 | tr -d 'v'" autoscaler_sidecar_private_ip: "{{ ansible_default_ipv4.address }}" autoscaler_sidecar_public_ip: "{{ ansible_ec2_public_ipv4 if ansible_ec2_public_ipv4 is defined else false }}" autoscaler_sidecar_service_name: jitsi-autoscaler-sidecar @@ -82,7 +83,7 @@ autoscaler_stats_retrieve_urls: 'jibri': http://localhost:2222/jibri/api/v1.0/health 'jigasi': http://localhost:8788/about/stats 'sip-jibri': http://localhost:2222/jibri/api/v1.0/health - 'whisper': http://localhost:8003/state + 'whisper': http://{{ ansible_default_ipv4.address }}:8003/state autoscaler_status_url: "https://{{ autoscaler_server_host }}/sidecar/status" autoscaler_shutdown_url: "https://{{ autoscaler_server_host }}/sidecar/shutdown" autoscaler_terminate_script: '/usr/local/bin/terminate_instance.sh' From 5dfedc80ab62a88c9aa4b7355cca688b2efc0c05 Mon Sep 17 00:00:00 2001 From: Aaron van Meerten Date: Wed, 11 Dec 2024 09:44:47 -0600 Subject: [PATCH 5/5] support cloud_instance_id var in autoscaler-sidecar --- ansible/roles/autoscaler-sidecar/defaults/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ansible/roles/autoscaler-sidecar/defaults/main.yml b/ansible/roles/autoscaler-sidecar/defaults/main.yml index 898a5ebe..dfbf2471 100644 --- a/ansible/roles/autoscaler-sidecar/defaults/main.yml +++ b/ansible/roles/autoscaler-sidecar/defaults/main.yml @@ -35,7 +35,8 @@ autoscaler_health_script: "/usr/local/bin/autoscaler-sidecar-health.sh" autoscaler_health_url: "http://localhost:{{ autoscaler_express_port }}/health" autoscaler_install_flag: true autoscaler_instance_id: "{{ - ansible_ec2_instance_id if ansible_ec2_instance_id is defined + cloud_instance_id if cloud_instance_id is defined + else ansible_ec2_instance_id if ansible_ec2_instance_id is defined else oracle_instance_id if oracle_instance_id is defined else 'default' }}" autoscaler_instance_type: 'JVB' autoscaler_key_id: "{{ asap_key['id'] if asap_key['id'] is defined else 'default' }}"