From c1941df7e767e729f4b874631faf21921280b5ae Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 14 Jun 2024 11:28:14 -0300 Subject: [PATCH] chore: adjustments for k8s deployment (#7531) * chore: handle TERM in datatracker-start.sh * chore: delay celery start if migration needed * chore: skip-checks when migrating * chore: label beat/celery as deleteBeforeUpgrade Used by the infra-k8s deployment process to flag these as needing to be shut down before a new release rolls out. * chore: increase termination grace periods --- dev/build/celery-start.sh | 8 ++ dev/build/datatracker-start.sh | 19 +++- k8s/beat.yaml | 124 ++++++++++++------------- k8s/celery.yaml | 162 +++++++++++++++++---------------- k8s/datatracker.yaml | 2 +- 5 files changed, 171 insertions(+), 144 deletions(-) diff --git a/dev/build/celery-start.sh b/dev/build/celery-start.sh index c229defa2f..c8d4450da3 100644 --- a/dev/build/celery-start.sh +++ b/dev/build/celery-start.sh @@ -5,6 +5,14 @@ echo "Running Datatracker checks..." ./ietf/manage.py check +if ! ietf/manage.py migrate --skip-checks --check ; then + echo "Unapplied migrations found, waiting to start..." + sleep 5 + while ! ietf/manage.py migrate --skip-checks --check ; do + sleep 5 + done +fi + cleanup () { # Cleanly terminate the celery app by sending it a TERM, then waiting for it to exit. if [[ -n "${celery_pid}" ]]; then diff --git a/dev/build/datatracker-start.sh b/dev/build/datatracker-start.sh index 390b46af86..7fee9394f0 100644 --- a/dev/build/datatracker-start.sh +++ b/dev/build/datatracker-start.sh @@ -4,14 +4,29 @@ echo "Running Datatracker checks..." ./ietf/manage.py check echo "Running Datatracker migrations..." -./ietf/manage.py migrate --settings=settings_local +./ietf/manage.py migrate --skip-checks --settings=settings_local echo "Starting Datatracker..." +# trap TERM and shut down gunicorn +cleanup () { + if [[ -n "${gunicorn_pid}" ]]; then + echo "Terminating gunicorn..." + kill -TERM "${gunicorn_pid}" + wait "${gunicorn_pid}" + fi +} + +trap 'trap "" TERM; cleanup' TERM + +# start gunicorn in the background so we can trap the TERM signal gunicorn \ --workers "${DATATRACKER_GUNICORN_WORKERS:-9}" \ --max-requests "${DATATRACKER_GUNICORN_MAX_REQUESTS:-32768}" \ --timeout "${DATATRACKER_GUNICORN_TIMEOUT:-180}" \ --bind :8000 \ --log-level "${DATATRACKER_GUNICORN_LOG_LEVEL:-info}" \ - ietf.wsgi:application + ${DATATRACKER_GUNICORN_EXTRA_ARGS} \ + ietf.wsgi:application & +gunicorn_pid=$! +wait "${gunicorn_pid}" diff --git a/k8s/beat.yaml b/k8s/beat.yaml index 3400b21cbf..99317ab77a 100644 --- a/k8s/beat.yaml +++ b/k8s/beat.yaml @@ -1,61 +1,63 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: beat -spec: - replicas: 1 - revisionHistoryLimit: 2 - selector: - matchLabels: - app: beat - strategy: - type: Recreate - template: - metadata: - labels: - app: beat - spec: - securityContext: - runAsNonRoot: true - containers: - - name: beat - image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG" - imagePullPolicy: Always - ports: - - containerPort: 8000 - name: http - protocol: TCP - volumeMounts: - - name: dt-vol - mountPath: /a - - name: dt-tmp - mountPath: /tmp - - name: dt-cfg - mountPath: /workspace/ietf/settings_local.py - subPath: settings_local.py - env: - - name: "CONTAINER_ROLE" - value: "beat" - envFrom: - - configMapRef: - name: django-config - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsUser: 1000 - runAsGroup: 1000 - volumes: - # To be overriden with the actual shared volume - - name: dt-vol - - name: dt-tmp - emptyDir: - sizeLimit: "2Gi" - - name: dt-cfg - configMap: - name: files-cfgmap - dnsPolicy: ClusterFirst - restartPolicy: Always - terminationGracePeriodSeconds: 30 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: beat + labels: + deleteBeforeUpgrade: yes +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: beat + strategy: + type: Recreate + template: + metadata: + labels: + app: beat + spec: + securityContext: + runAsNonRoot: true + containers: + - name: beat + image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG" + imagePullPolicy: Always + ports: + - containerPort: 8000 + name: http + protocol: TCP + volumeMounts: + - name: dt-vol + mountPath: /a + - name: dt-tmp + mountPath: /tmp + - name: dt-cfg + mountPath: /workspace/ietf/settings_local.py + subPath: settings_local.py + env: + - name: "CONTAINER_ROLE" + value: "beat" + envFrom: + - configMapRef: + name: django-config + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 1000 + runAsGroup: 1000 + volumes: + # To be overriden with the actual shared volume + - name: dt-vol + - name: dt-tmp + emptyDir: + sizeLimit: "2Gi" + - name: dt-cfg + configMap: + name: files-cfgmap + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 600 diff --git a/k8s/celery.yaml b/k8s/celery.yaml index 407f21e66d..132e71761f 100644 --- a/k8s/celery.yaml +++ b/k8s/celery.yaml @@ -1,80 +1,82 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: celery -spec: - replicas: 1 - revisionHistoryLimit: 2 - selector: - matchLabels: - app: celery - strategy: - type: Recreate - template: - metadata: - labels: - app: celery - spec: - securityContext: - runAsNonRoot: true - containers: - # ----------------------------------------------------- - # ScoutAPM Container - # ----------------------------------------------------- - - name: scoutapm - image: "scoutapp/scoutapm:version-1.4.0" - imagePullPolicy: IfNotPresent - livenessProbe: - exec: - command: - - "sh" - - "-c" - - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'" - securityContext: - readOnlyRootFilesystem: true - runAsUser: 65534 # "nobody" user by default - runAsGroup: 65534 # "nogroup" group by default - # ----------------------------------------------------- - # Celery Container - # ----------------------------------------------------- - - name: celery - image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG" - imagePullPolicy: Always - ports: - - containerPort: 8000 - name: http - protocol: TCP - volumeMounts: - - name: dt-vol - mountPath: /a - - name: dt-tmp - mountPath: /tmp - - name: dt-cfg - mountPath: /workspace/ietf/settings_local.py - subPath: settings_local.py - env: - - name: "CONTAINER_ROLE" - value: "celery" - envFrom: - - configMapRef: - name: django-config - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsUser: 1000 - runAsGroup: 1000 - volumes: - # To be overriden with the actual shared volume - - name: dt-vol - - name: dt-tmp - emptyDir: - sizeLimit: "2Gi" - - name: dt-cfg - configMap: - name: files-cfgmap - dnsPolicy: ClusterFirst - restartPolicy: Always - terminationGracePeriodSeconds: 30 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: celery + labels: + deleteBeforeUpgrade: yes +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: celery + strategy: + type: Recreate + template: + metadata: + labels: + app: celery + spec: + securityContext: + runAsNonRoot: true + containers: + # ----------------------------------------------------- + # ScoutAPM Container + # ----------------------------------------------------- + - name: scoutapm + image: "scoutapp/scoutapm:version-1.4.0" + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - "sh" + - "-c" + - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'" + securityContext: + readOnlyRootFilesystem: true + runAsUser: 65534 # "nobody" user by default + runAsGroup: 65534 # "nogroup" group by default + # ----------------------------------------------------- + # Celery Container + # ----------------------------------------------------- + - name: celery + image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG" + imagePullPolicy: Always + ports: + - containerPort: 8000 + name: http + protocol: TCP + volumeMounts: + - name: dt-vol + mountPath: /a + - name: dt-tmp + mountPath: /tmp + - name: dt-cfg + mountPath: /workspace/ietf/settings_local.py + subPath: settings_local.py + env: + - name: "CONTAINER_ROLE" + value: "celery" + envFrom: + - configMapRef: + name: django-config + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 1000 + runAsGroup: 1000 + volumes: + # To be overriden with the actual shared volume + - name: dt-vol + - name: dt-tmp + emptyDir: + sizeLimit: "2Gi" + - name: dt-cfg + configMap: + name: files-cfgmap + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 600 diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml index 7ca92ba99e..9771623138 100644 --- a/k8s/datatracker.yaml +++ b/k8s/datatracker.yaml @@ -77,7 +77,7 @@ spec: name: files-cfgmap dnsPolicy: ClusterFirst restartPolicy: Always - terminationGracePeriodSeconds: 30 + terminationGracePeriodSeconds: 60 --- apiVersion: v1 kind: Service