Merge pull request #18 from navendu-pottekkat/update-action/navendu-p…

…ottekkat

Update workflow to schedule performance tests

.github/load-test.yaml

@@ -0,0 +1,30 @@
+# Test configuration file for running performance benchmarks
+# See: https://docs.meshery.io/guides/performance-management#running-performance-benchmarks-through-mesheryctl
+test:
+  smp_version: v0.0.1
+
+  # The name of the test
+  # Will be overridden by the workflow with a test specific name (mesh)-(load-generator-(test-configuration-file)
+  name: Load Test
+  labels: {}
+
+  # Test configuration to be defined here
+  # In scheduled benchmarks, the load generator, the endpoint URL and the service mesh are dynamically injected 
+  clients:
+    - internal: false
+      load_generator: fortio
+      protocol: 1
+      connections: 2
+      rps: 10
+      headers: {}
+      cookies: {}
+      body: ''
+      content_type: ''
+      endpoint_urls:
+        - 'https://smp-spec.io'
+  duration: 60s
+
+# Service mesh under test in Service Mesh Performance Spec format
+# See: https://github.com/service-mesh-performance/service-mesh-performance/blob/master/protos/service_mesh.proto
+mesh:
+  type: 3

.github/soak-test.yaml

@@ -0,0 +1,30 @@
+# Test configuration file for running performance benchmarks
+# See: https://docs.meshery.io/guides/performance-management#running-performance-benchmarks-through-mesheryctl
+test:
+  smp_version: v0.0.1
+
+  # The name of the test
+  # Will be overridden by the workflow with a test specific name (mesh)-(load-generator-(test-configuration-file)
+  name: Soak Test
+  labels: {}
+
+  # Test configuration to be defined here
+  # In scheduled benchmarks, the load generator, the endpoint URL and the service mesh are dynamically injected 
+  clients:
+    - internal: false
+      load_generator: fortio
+      protocol: 1
+      connections: 2
+      rps: 10
+      headers: {}
+      cookies: {}
+      body: ''
+      content_type: ''
+      endpoint_urls:
+        - 'https://smp-spec.io'
+  duration: 60s
+
+# Service mesh under test in Service Mesh Performance Spec format
+# See: https://github.com/service-mesh-performance/service-mesh-performance/blob/master/protos/service_mesh.proto
+mesh:
+  type: 3

.github/workflows/configurable-benchmark-test.yaml

@@ -0,0 +1,73 @@
+# This workflow runs performance benchmarks with Meshery based on the configuration provided
+# This workflow needs to be triggered manually by providing the test configuration
+
+name: Configurable Benchmark Test
+on:
+  # for triggering manually, provide a token
+  workflow_dispatch:
+    inputs:
+      provider_token:
+        description: "token for remote provider"
+        required: false
+      profile_name:
+        description: "performance profile to use"
+        required: false
+      profile_filename:
+        description: "test configuration file"
+        required: false
+      service_mesh:
+        type: choice
+        description: "service mesh being tested"
+        options:
+          - app-mesh
+          - consul
+          - istio
+          - kuma
+          - linkerd
+          - osm
+      load_generator:
+        type: choice
+        description: "load generator to run tests with"
+        options:
+          - fortio
+          - wrk2
+          - nighthawk
+
+jobs:
+  manual-test:
+    name: Configurable Benchmark Test
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'workflow_dispatch' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        service-mesh: ['istio', 'linkerd', 'osm']
+        load-generator: ['fortio', 'wrk2']
+    steps:
+      - name: Setup Kubernetes
+        uses: manusa/[email protected]
+        with:
+          minikube version: 'v1.23.2'
+          kubernetes version: 'v1.22.2'
+          driver: docker
+
+      - name: Checkout Code
+        uses: actions/checkout@v2
+
+      - name: Install Service Mesh and Deploy Application
+        run: |
+          chmod +x .github/workflows/scripts/${{ matrix.service-mesh }}_deploy.sh
+          .github/workflows/scripts/${{ matrix.service-mesh }}_deploy.sh
+        shell: bash
+
+      - name: Run Benchmark Tests
+        uses: layer5io/meshery-smp-action@master
+        with:
+          provider_token: ${{ github.event.inputs.provider_token }}
+          platform: docker
+          profile_name: ${{ github.event.inputs.profile_name }}
+          profile_filename: ${{ github.event.inputs.profile_filename }}
+          endpoint_url: ${{env.ENDPOINT_URL}}
+          service_mesh: ${{env.SERVICE_MESH}}
+          load_generator: ${{ matrix.load-generator }}
+          test_name: '${{ matrix.service-mesh }}-${{ matrix.load-generator }}-${{ github.event.inputs.profile_filename }}${{ github.event.inputs.profile_name }}'

.github/workflows/scheduled-benchmarks.yml

@@ -1,6 +1,9 @@
+# This workflow runs performance benchmarks with Meshery based on the configuration provided
+# This workflow is scheduled to run daily but can also be triggered manually
+
 name: Scheduled Benchmark Tests
 on:
-  # for triggering manually, provide a token
+  # for triggering manually, provide a token, a test configuration file name or a performance profile name
   workflow_dispatch:
     inputs:
       provider_token:
@@ -21,6 +24,11 @@ jobs:
     name: Manual Benchmark Test
     runs-on: ubuntu-latest
     if: ${{ github.event_name == 'workflow_dispatch' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        service-mesh: ['istio', 'linkerd', 'osm']
+        load-generator: ['fortio', 'wrk2']
     steps:
       - name: Setup Kubernetes
         uses: manusa/[email protected]
@@ -34,8 +42,8 @@ jobs:
 
       - name: Install Service Mesh and Deploy Application
         run: |
-          chmod +x .github/workflows/scripts/istio_deploy.sh
-          .github/workflows/scripts/istio_deploy.sh
+          chmod +x .github/workflows/scripts/${{ matrix.service-mesh }}_deploy.sh
+          .github/workflows/scripts/${{ matrix.service-mesh }}_deploy.sh
         shell: bash
 
       - name: Run Benchmark Tests
@@ -46,11 +54,20 @@ jobs:
           profile_name: ${{ github.event.inputs.profile_name }}
           profile_filename: ${{ github.event.inputs.profile_filename }}
           endpoint_url: ${{env.ENDPOINT_URL}}
+          service_mesh: ${{env.SERVICE_MESH}}
+          load_generator: ${{ matrix.load-generator }}
+          test_name: '${{ matrix.service-mesh }}-${{ matrix.load-generator }}-${{ github.event.inputs.profile_filename }}${{ github.event.inputs.profile_name }}'
 
   scheduled-test:
     name: Scheduled Benchmark Test
     runs-on: ubuntu-latest
     if: ${{ github.event_name == 'schedule' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        service-mesh: ['istio', 'linkerd', 'osm']
+        load-generator: ['fortio', 'wrk2']
+        test-configuration: ['load-test.yaml','soak-test.yaml']
     steps:
       - name: Setup Kubernetes
         uses: manusa/[email protected]
@@ -64,14 +81,17 @@ jobs:
 
       - name: Install Service Mesh and Deploy Application
         run: |
-          chmod +x .github/workflows/scripts/istio_deploy.sh
-          .github/workflows/scripts/istio_deploy.sh
+          chmod +x .github/workflows/scripts/${{ matrix.service-mesh }}_deploy.sh
+          .github/workflows/scripts/${{ matrix.service-mesh }}_deploy.sh
         shell: bash
 
       - name: Run Benchmark Tests
         uses: layer5io/meshery-smp-action@master
         with:
           provider_token: ${{ secrets.MESHERY_TOKEN }}
           platform: docker
-          profile_name: istio-benchmark
-          profile_filename: istio-test-config
+          profile_filename: ${{ matrix.test-configuration }}
+          endpoint_url: ${{env.ENDPOINT_URL}}
+          service_mesh: ${{env.SERVICE_MESH}}
+          load_generator: ${{ matrix.load-generator }}
+          test_name: '${{ matrix.service-mesh }}-${{ matrix.load-generator }}-{{ github.event.inputs.profile_name }}'

.github/workflows/scripts/istio_deploy.sh

@@ -1,17 +1,36 @@
 #!/usr/bin/env bash
 
+# This script is used to deploy Istio on Kubernetes
+#
+# Also deploys the bookinfo application on Istio and passes the gateway URL to Meshery
+
+# See: https://github.com/service-mesh-performance/service-mesh-performance/blob/master/protos/service_mesh.proto
+export MESH_NAME='Istio'
+export SERVICE_MESH=3
+
 curl -L https://istio.io/downloadIstio | ISTIO_VERSION=1.11.4 sh -
 cd istio-1.11.4
 export PATH=$PWD/bin:$PATH
 istioctl install --set profile=demo -y
 kubectl label namespace default istio-injection=enabled
+
 kubectl apply -f samples/bookinfo/platform/kube/bookinfo.yaml
 kubectl apply -f samples/bookinfo/networking/bookinfo-gateway.yaml
+
+# Wait for the application to be ready
 sleep 100
+
+# Get the gateway URL and export it and
+# Expose the service inside the cluster
 export INGRESS_PORT=$(kubectl -n istio-system get service istio-ingressgateway -o jsonpath='{.spec.ports[?(@.name=="http2")].nodePort}')
 export SECURE_INGRESS_PORT=$(kubectl -n istio-system get service istio-ingressgateway -o jsonpath='{.spec.ports[?(@.name=="https")].nodePort}')
 export INGRESS_HOST=$(minikube ip)
-export GATEWAY_URL=$INGRESS_HOST:$INGRESS_PORT
+export GATEWAY_URL=http://$INGRESS_HOST:$INGRESS_PORT
+
 minikube tunnel &
-echo "$GATEWAY_URL"
-echo "ENDPOINT_URL=$GATEWAY_URL" >> $GITHUB_ENV
+
+echo "Service Mesh: $MESH_NAME - $SERVICE_MESH"
+echo "Gateway URL: $GATEWAY_URL"
+
+echo "ENDPOINT_URL=$GATEWAY_URL" >> $GITHUB_ENV
+echo "SERVICE_MESH=$SERVICE_MESH" >> $GITHUB_ENV

.github/workflows/scripts/linkerd_deploy.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+# This script is used to deploy Linkerd on Kubernetes
+#
+# Also deploys EmojiVoto on Linkerd and exposes the service to Meshery
+
+# See: https://github.com/service-mesh-performance/service-mesh-performance/blob/master/protos/service_mesh.proto
+export MESH_NAME='Linkerd'
+export SERVICE_MESH=5
+
+curl -fsL https://run.linkerd.io/install | sh
+export PATH=$PATH:/home/runner/.linkerd2/bin
+linkerd version
+linkerd check --pre
+linkerd install | kubectl apply -f -
+linkerd check
+
+curl -fsL https://run.linkerd.io/emojivoto.yml | kubectl apply -f -
+kubectl -n emojivoto port-forward svc/web-svc 8080:80 &
+kubectl get -n emojivoto deploy -o yaml | linkerd inject - | kubectl apply -f -
+
+# Wait for the application to be ready
+sleep 100
+
+echo "Service Mesh: $MESH_NAME - $SERVICE_MESH"
+echo "Endpoint URL: http://localhost:8080"
+
+# Pass the endpoint to be used by Meshery
+echo "ENDPOINT_URL=http://localhost:8080" >> $GITHUB_ENV
+echo "SERVICE_MESH=$SERVICE_MESH" >> $GITHUB_ENV

.github/workflows/scripts/osm_deploy.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+# This script is used to deploy Open Service Mesh on Kubernetes
+#
+# Also deploys Bookstore application on OSM
+
+# See: https://github.com/service-mesh-performance/service-mesh-performance/blob/master/protos/service_mesh.proto
+export MESH_NAME='Open Service Mesh'
+export SERVICE_MESH='10'
+
+system=$(uname -s)
+release=v0.11.1
+curl -L https://github.com/openservicemesh/osm/releases/download/${release}/osm-${release}-${system}-amd64.tar.gz | tar -vxzf - ./${system}-amd64/osm version
+osm install \
+    --set=OpenServiceMesh.enablePermissiveTrafficPolicy=true \
+    --set=OpenServiceMesh.deployPrometheus=true \
+    --set=OpenServiceMesh.deployGrafana=true \
+    --set=OpenServiceMesh.deployJaeger=true
+
+kubectl create namespace bookstore
+osm namespace add bookstore
+kubectl apply -f https://raw.githubusercontent.com/openservicemesh/osm/release-v0.11/docs/example/manifests/apps/bookstore.yaml
+kubectl get deployments -n bookstore
+kubectl get pods -n bookstore
+
+# Wait for the application to be ready
+sleep 100
+
+# Expose the application outside the cluster
+backend="$1"
+thisScript="$(dirname "$0")/$(basename "$0")"
+
+if [ -z "$backend" ]; then
+    echo "Usage: $thisScript <backend-name>"
+    exit 1
+fi
+
+POD="$(kubectl get pods --selector app="$backend" -n "$BOOKSTORE_NAMESPACE" --no-headers | grep 'Running' | awk 'NR==1{print $1}')"
+kubectl port-forward "$POD" -n "$BOOKSTORE_NAMESPACE" 15000:15000
+
+echo "Service Mesh: $MESH_NAME - $SERVICE_MESH"
+echo "Endpoint URL: http://localhost:15000"
+
+# Pass the endpoint to be used by Meshery
+echo "ENDPOINT_URL=http://localhost:15000" >> $GITHUB_ENV
+echo "SERVICE_MESH=$SERVICE_MESH" >> $GITHUB_ENV

action.yml

@@ -30,12 +30,30 @@ inputs:
     description: "Name of the performance profile"
     required: false
 
-  # overrides the endpoint_url specified in the tes configuration file
-  # use this for dynamically injecting the application endpoint while running the workflow
+  # overrides the endpoint_url specified in the test configuration file
+  # use this for dynamically injecting the application endpoint while using the action
   endpoint_url:
     description: "Endpoint in which the application is deployed"
     required: false
 
+  # overrides the mesh type specified in the test configuration file
+  # use this for dynamically injecting the service mesh while using the action
+  service_mesh:
+    description: "Service Mesh which is being tested"
+    required: false
+
+  # overrides the name specified in the test configuration file
+  # use this for dynamically injecting the test name while using the action
+  test_name:
+    description: "Name of the test"
+    required: false
+
+  # overrides the load generator specified in the test configuration file
+  # use this for dynamically injecting the load generator to test with while using the action
+  load_generator:
+    description: "Load generator to run tests with"
+    required: false
+
 runs:
   using: "node12"
   main: "main.js"