From 2e876f253fc650aeba085016d979edd119df167a Mon Sep 17 00:00:00 2001 From: Navendu Pottekkat Date: Thu, 11 Nov 2021 18:48:20 +0530 Subject: [PATCH 1/4] make service mesh dynamically injectable Signed-off-by: Navendu Pottekkat --- .github/workflows/scripts/istio_deploy.sh | 4 +++- action.yml | 10 ++++++++-- main.sh | 4 ++++ mesheryctl.sh | 13 ++++++++++++- 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/.github/workflows/scripts/istio_deploy.sh b/.github/workflows/scripts/istio_deploy.sh index fb3be68..05ad704 100644 --- a/.github/workflows/scripts/istio_deploy.sh +++ b/.github/workflows/scripts/istio_deploy.sh @@ -13,5 +13,7 @@ export SECURE_INGRESS_PORT=$(kubectl -n istio-system get service istio-ingressga export INGRESS_HOST=$(minikube ip) export GATEWAY_URL=$INGRESS_HOST:$INGRESS_PORT minikube tunnel & +export SERVICE_MESH=3 echo "$GATEWAY_URL" -echo "ENDPOINT_URL=$GATEWAY_URL" >> $GITHUB_ENV \ No newline at end of file +echo "ENDPOINT_URL=$GATEWAY_URL" >> $GITHUB_ENV +echo "ENDPOINT_URL=$SERVICE_MESH" >> $GITHUB_ENV \ No newline at end of file diff --git a/action.yml b/action.yml index c8f9dc8..5ca557f 100644 --- a/action.yml +++ b/action.yml @@ -30,12 +30,18 @@ inputs: description: "Name of the performance profile" required: false - # overrides the endpoint_url specified in the tes configuration file - # use this for dynamically injecting the application endpoint while running the workflow + # overrides the endpoint_url specified in the test configuration file + # use this for dynamically injecting the application endpoint while using the action endpoint_url: description: "Endpoint in which the application is deployed" required: false + # overrides the mesh type specified in the test configuration file + # use this for dynamically injecting the service mesh while using the action + service_mesh: + description: "Service Mesh which is being tested" + required: false + runs: using: "node12" main: "main.js" diff --git a/main.sh b/main.sh index c3c0a05..6596619 100755 --- a/main.sh +++ b/main.sh @@ -33,6 +33,10 @@ main() { commandArgs+=(--endpoint-url ${INPUT_ENDPOINT_URL}) fi + if [[ -n "${INPUT_SERVICE_MESH:-}" ]]; then + commandArgs+=(--service-mesh ${INPUT_SERVICE_MESH}) + fi + bash "$SCRIPT_DIR/mesheryctl.sh" "${commandArgs[@]}" } diff --git a/mesheryctl.sh b/mesheryctl.sh index 7c80b7b..7edf3ad 100755 --- a/mesheryctl.sh +++ b/mesheryctl.sh @@ -17,6 +17,7 @@ main() { local perf_filename= local perf_profile_name= local endpoint_url= + local service_mesh= parse_command_line "$@" @@ -37,8 +38,9 @@ main() { echo "Configuration file: $perf_filename" echo "Endpoint URL: $endpoint_url" + echo "Service Mesh: $service_mesh" - mesheryctl perf apply --file $GITHUB_WORKSPACE/.github/$perf_filename -t ~/auth.json --url "$endpoint_url" + mesheryctl perf apply --file $GITHUB_WORKSPACE/.github/$perf_filename -t ~/auth.json --url "$endpoint_url" --mesh "$service_mesh" else @@ -94,6 +96,15 @@ parse_command_line() { exit 1 fi ;; + --service-mesh) + if [[ -n "${2:-}" ]]; then + service_mesh=$2 + shift + else + echo "ERROR: '--service-mesh' cannot be empty." >&2 + exit 1 + fi + ;; *) break ;; From a94902848f48af7a95347e5f355608063326d841 Mon Sep 17 00:00:00 2001 From: Navendu Pottekkat Date: Thu, 11 Nov 2021 22:33:18 +0530 Subject: [PATCH 2/4] add test name and load gen as configs Signed-off-by: Navendu Pottekkat --- .github/workflows/scripts/istio_deploy.sh | 2 +- action.yml | 12 ++++++++++++ main.sh | 8 ++++++++ mesheryctl.sh | 24 ++++++++++++++++++++++- 4 files changed, 44 insertions(+), 2 deletions(-) diff --git a/.github/workflows/scripts/istio_deploy.sh b/.github/workflows/scripts/istio_deploy.sh index 05ad704..294b159 100644 --- a/.github/workflows/scripts/istio_deploy.sh +++ b/.github/workflows/scripts/istio_deploy.sh @@ -16,4 +16,4 @@ minikube tunnel & export SERVICE_MESH=3 echo "$GATEWAY_URL" echo "ENDPOINT_URL=$GATEWAY_URL" >> $GITHUB_ENV -echo "ENDPOINT_URL=$SERVICE_MESH" >> $GITHUB_ENV \ No newline at end of file +echo "SERVICE_MESH=$SERVICE_MESH" >> $GITHUB_ENV \ No newline at end of file diff --git a/action.yml b/action.yml index 5ca557f..485cc25 100644 --- a/action.yml +++ b/action.yml @@ -42,6 +42,18 @@ inputs: description: "Service Mesh which is being tested" required: false + # overrides the name specified in the test configuration file + # use this for dynamically injecting the test name while using the action + test_name: + description: "Name of the test" + required: false + + # overrides the load generator specified in the test configuration file + # use this for dynamically injecting the load generator to test with while using the action + load_generator: + description: "Load generator to run tests with" + required: false + runs: using: "node12" main: "main.js" diff --git a/main.sh b/main.sh index 6596619..3caf9e6 100755 --- a/main.sh +++ b/main.sh @@ -37,6 +37,14 @@ main() { commandArgs+=(--service-mesh ${INPUT_SERVICE_MESH}) fi + if [[ -n "${INPUT_TEST_NAME:-}" ]]; then + commandArgs+=(--test-name ${INPUT_TEST_NAME}) + fi + + if [[ -n "${INPUT_LOAD_GENERATOR:-}" ]]; then + commandArgs+=(--load-generator ${INPUT_LOAD_GENERATOR}) + fi + bash "$SCRIPT_DIR/mesheryctl.sh" "${commandArgs[@]}" } diff --git a/mesheryctl.sh b/mesheryctl.sh index 7edf3ad..78d2b31 100755 --- a/mesheryctl.sh +++ b/mesheryctl.sh @@ -18,6 +18,8 @@ main() { local perf_profile_name= local endpoint_url= local service_mesh= + local test_name= + local load_generator= parse_command_line "$@" @@ -39,8 +41,10 @@ main() { echo "Configuration file: $perf_filename" echo "Endpoint URL: $endpoint_url" echo "Service Mesh: $service_mesh" + echo "Test Name: $test_name" + echo "Load Generator: $load_generator" - mesheryctl perf apply --file $GITHUB_WORKSPACE/.github/$perf_filename -t ~/auth.json --url "$endpoint_url" --mesh "$service_mesh" + mesheryctl perf apply --file $GITHUB_WORKSPACE/.github/$perf_filename -t ~/auth.json --url "$endpoint_url" --mesh "$service_mesh" --name "$test_name" --load-generator "$load_generator" else @@ -105,6 +109,24 @@ parse_command_line() { exit 1 fi ;; + --test-name) + if [[ -n "${2:-}" ]]; then + test_name=$2 + shift + else + echo "ERROR: '--test-name' cannot be empty." >&2 + exit 1 + fi + ;; + --load-generator) + if [[ -n "${2:-}" ]]; then + load_generator=$2 + shift + else + echo "ERROR: '--load-generator' cannot be empty." >&2 + exit 1 + fi + ;; *) break ;; From 566080bf64648c644c0a1d6f8cf85e6374b5780e Mon Sep 17 00:00:00 2001 From: Navendu Pottekkat Date: Fri, 12 Nov 2021 00:25:30 +0530 Subject: [PATCH 3/4] update test config files Signed-off-by: Navendu Pottekkat --- .github/load-test.yaml | 30 ++++++++++++++++++++++++++++++ .github/soak-test.yaml | 30 ++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 .github/load-test.yaml create mode 100644 .github/soak-test.yaml diff --git a/.github/load-test.yaml b/.github/load-test.yaml new file mode 100644 index 0000000..0b31e07 --- /dev/null +++ b/.github/load-test.yaml @@ -0,0 +1,30 @@ +# Test configuration file for running performance benchmarks +# See: https://docs.meshery.io/guides/performance-management#running-performance-benchmarks-through-mesheryctl +test: + smp_version: v0.0.1 + + # The name of the test + # Will be overridden by the workflow with a test specific name (mesh)-(load-generator-(test-configuration-file) + name: Load Test + labels: {} + + # Test configuration to be defined here + # In scheduled benchmarks, the load generator, the endpoint URL and the service mesh are dynamically injected + clients: + - internal: false + load_generator: fortio + protocol: 1 + connections: 2 + rps: 10 + headers: {} + cookies: {} + body: '' + content_type: '' + endpoint_urls: + - 'https://smp-spec.io' + duration: 60s + +# Service mesh under test in Service Mesh Performance Spec format +# See: https://github.com/service-mesh-performance/service-mesh-performance/blob/master/protos/service_mesh.proto +mesh: + type: 3 diff --git a/.github/soak-test.yaml b/.github/soak-test.yaml new file mode 100644 index 0000000..ff23040 --- /dev/null +++ b/.github/soak-test.yaml @@ -0,0 +1,30 @@ +# Test configuration file for running performance benchmarks +# See: https://docs.meshery.io/guides/performance-management#running-performance-benchmarks-through-mesheryctl +test: + smp_version: v0.0.1 + + # The name of the test + # Will be overridden by the workflow with a test specific name (mesh)-(load-generator-(test-configuration-file) + name: Soak Test + labels: {} + + # Test configuration to be defined here + # In scheduled benchmarks, the load generator, the endpoint URL and the service mesh are dynamically injected + clients: + - internal: false + load_generator: fortio + protocol: 1 + connections: 2 + rps: 10 + headers: {} + cookies: {} + body: '' + content_type: '' + endpoint_urls: + - 'https://smp-spec.io' + duration: 60s + +# Service mesh under test in Service Mesh Performance Spec format +# See: https://github.com/service-mesh-performance/service-mesh-performance/blob/master/protos/service_mesh.proto +mesh: + type: 3 From a2bd629a307fd6d065d490ae6766ed6ac46d68bb Mon Sep 17 00:00:00 2001 From: Navendu Pottekkat Date: Fri, 12 Nov 2021 00:25:50 +0530 Subject: [PATCH 4/4] update workflow and scripts Signed-off-by: Navendu Pottekkat --- .github/test-config.yaml | 19 ----- .../configurable-benchmark-test.yaml | 73 +++++++++++++++++++ .github/workflows/scheduled-benchmarks.yml | 34 +++++++-- .github/workflows/scripts/istio_deploy.sh | 25 ++++++- .github/workflows/scripts/linkerd_deploy.sh | 30 ++++++++ .github/workflows/scripts/osm_deploy.sh | 46 ++++++++++++ 6 files changed, 197 insertions(+), 30 deletions(-) delete mode 100644 .github/test-config.yaml create mode 100644 .github/workflows/configurable-benchmark-test.yaml create mode 100644 .github/workflows/scripts/linkerd_deploy.sh create mode 100644 .github/workflows/scripts/osm_deploy.sh diff --git a/.github/test-config.yaml b/.github/test-config.yaml deleted file mode 100644 index 76131eb..0000000 --- a/.github/test-config.yaml +++ /dev/null @@ -1,19 +0,0 @@ -test: - smp_version: v0.0.1 - name: istio-test - labels: {} - clients: - - internal: false - load_generator: fortio - protocol: 1 - connections: 2 - rps: 5 - headers: {} - cookies: {} - body: '' - content_type: '' - endpoint_urls: - - 'https://meshery.io' - duration: 6s -mesh: - type: 3 diff --git a/.github/workflows/configurable-benchmark-test.yaml b/.github/workflows/configurable-benchmark-test.yaml new file mode 100644 index 0000000..9b8cf39 --- /dev/null +++ b/.github/workflows/configurable-benchmark-test.yaml @@ -0,0 +1,73 @@ +# This workflow runs performance benchmarks with Meshery based on the configuration provided +# This workflow needs to be triggered manually by providing the test configuration + +name: Configurable Benchmark Test +on: + # for triggering manually, provide a token + workflow_dispatch: + inputs: + provider_token: + description: "token for remote provider" + required: false + profile_name: + description: "performance profile to use" + required: false + profile_filename: + description: "test configuration file" + required: false + service_mesh: + type: choice + description: "service mesh being tested" + options: + - app-mesh + - consul + - istio + - kuma + - linkerd + - osm + load_generator: + type: choice + description: "load generator to run tests with" + options: + - fortio + - wrk2 + - nighthawk + +jobs: + manual-test: + name: Configurable Benchmark Test + runs-on: ubuntu-latest + if: ${{ github.event_name == 'workflow_dispatch' }} + strategy: + fail-fast: false + matrix: + service-mesh: ['istio', 'linkerd', 'osm'] + load-generator: ['fortio', 'wrk2'] + steps: + - name: Setup Kubernetes + uses: manusa/actions-setup-minikube@v2.4.1 + with: + minikube version: 'v1.23.2' + kubernetes version: 'v1.22.2' + driver: docker + + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Install Service Mesh and Deploy Application + run: | + chmod +x .github/workflows/scripts/${{ matrix.service-mesh }}_deploy.sh + .github/workflows/scripts/${{ matrix.service-mesh }}_deploy.sh + shell: bash + + - name: Run Benchmark Tests + uses: layer5io/meshery-smp-action@master + with: + provider_token: ${{ github.event.inputs.provider_token }} + platform: docker + profile_name: ${{ github.event.inputs.profile_name }} + profile_filename: ${{ github.event.inputs.profile_filename }} + endpoint_url: ${{env.ENDPOINT_URL}} + service_mesh: ${{env.SERVICE_MESH}} + load_generator: ${{ matrix.load-generator }} + test_name: '${{ matrix.service-mesh }}-${{ matrix.load-generator }}-${{ github.event.inputs.profile_filename }}${{ github.event.inputs.profile_name }}' diff --git a/.github/workflows/scheduled-benchmarks.yml b/.github/workflows/scheduled-benchmarks.yml index b3f76a0..4aec96f 100644 --- a/.github/workflows/scheduled-benchmarks.yml +++ b/.github/workflows/scheduled-benchmarks.yml @@ -1,6 +1,9 @@ +# This workflow runs performance benchmarks with Meshery based on the configuration provided +# This workflow is scheduled to run daily but can also be triggered manually + name: Scheduled Benchmark Tests on: - # for triggering manually, provide a token + # for triggering manually, provide a token, a test configuration file name or a performance profile name workflow_dispatch: inputs: provider_token: @@ -21,6 +24,11 @@ jobs: name: Manual Benchmark Test runs-on: ubuntu-latest if: ${{ github.event_name == 'workflow_dispatch' }} + strategy: + fail-fast: false + matrix: + service-mesh: ['istio', 'linkerd', 'osm'] + load-generator: ['fortio', 'wrk2'] steps: - name: Setup Kubernetes uses: manusa/actions-setup-minikube@v2.4.1 @@ -34,8 +42,8 @@ jobs: - name: Install Service Mesh and Deploy Application run: | - chmod +x .github/workflows/scripts/istio_deploy.sh - .github/workflows/scripts/istio_deploy.sh + chmod +x .github/workflows/scripts/${{ matrix.service-mesh }}_deploy.sh + .github/workflows/scripts/${{ matrix.service-mesh }}_deploy.sh shell: bash - name: Run Benchmark Tests @@ -46,11 +54,20 @@ jobs: profile_name: ${{ github.event.inputs.profile_name }} profile_filename: ${{ github.event.inputs.profile_filename }} endpoint_url: ${{env.ENDPOINT_URL}} + service_mesh: ${{env.SERVICE_MESH}} + load_generator: ${{ matrix.load-generator }} + test_name: '${{ matrix.service-mesh }}-${{ matrix.load-generator }}-${{ github.event.inputs.profile_filename }}${{ github.event.inputs.profile_name }}' scheduled-test: name: Scheduled Benchmark Test runs-on: ubuntu-latest if: ${{ github.event_name == 'schedule' }} + strategy: + fail-fast: false + matrix: + service-mesh: ['istio', 'linkerd', 'osm'] + load-generator: ['fortio', 'wrk2'] + test-configuration: ['load-test.yaml','soak-test.yaml'] steps: - name: Setup Kubernetes uses: manusa/actions-setup-minikube@v2.4.1 @@ -64,8 +81,8 @@ jobs: - name: Install Service Mesh and Deploy Application run: | - chmod +x .github/workflows/scripts/istio_deploy.sh - .github/workflows/scripts/istio_deploy.sh + chmod +x .github/workflows/scripts/${{ matrix.service-mesh }}_deploy.sh + .github/workflows/scripts/${{ matrix.service-mesh }}_deploy.sh shell: bash - name: Run Benchmark Tests @@ -73,5 +90,8 @@ jobs: with: provider_token: ${{ secrets.MESHERY_TOKEN }} platform: docker - profile_name: istio-benchmark - profile_filename: istio-test-config + profile_filename: ${{ matrix.test-configuration }} + endpoint_url: ${{env.ENDPOINT_URL}} + service_mesh: ${{env.SERVICE_MESH}} + load_generator: ${{ matrix.load-generator }} + test_name: '${{ matrix.service-mesh }}-${{ matrix.load-generator }}-{{ github.event.inputs.profile_name }}' diff --git a/.github/workflows/scripts/istio_deploy.sh b/.github/workflows/scripts/istio_deploy.sh index 294b159..1d9477f 100644 --- a/.github/workflows/scripts/istio_deploy.sh +++ b/.github/workflows/scripts/istio_deploy.sh @@ -1,19 +1,36 @@ #!/usr/bin/env bash +# This script is used to deploy Istio on Kubernetes +# +# Also deploys the bookinfo application on Istio and passes the gateway URL to Meshery + +# See: https://github.com/service-mesh-performance/service-mesh-performance/blob/master/protos/service_mesh.proto +export MESH_NAME='Istio' +export SERVICE_MESH=3 + curl -L https://istio.io/downloadIstio | ISTIO_VERSION=1.11.4 sh - cd istio-1.11.4 export PATH=$PWD/bin:$PATH istioctl install --set profile=demo -y kubectl label namespace default istio-injection=enabled + kubectl apply -f samples/bookinfo/platform/kube/bookinfo.yaml kubectl apply -f samples/bookinfo/networking/bookinfo-gateway.yaml + +# Wait for the application to be ready sleep 100 + +# Get the gateway URL and export it and +# Expose the service inside the cluster export INGRESS_PORT=$(kubectl -n istio-system get service istio-ingressgateway -o jsonpath='{.spec.ports[?(@.name=="http2")].nodePort}') export SECURE_INGRESS_PORT=$(kubectl -n istio-system get service istio-ingressgateway -o jsonpath='{.spec.ports[?(@.name=="https")].nodePort}') export INGRESS_HOST=$(minikube ip) -export GATEWAY_URL=$INGRESS_HOST:$INGRESS_PORT +export GATEWAY_URL=http://$INGRESS_HOST:$INGRESS_PORT + minikube tunnel & -export SERVICE_MESH=3 -echo "$GATEWAY_URL" + +echo "Service Mesh: $MESH_NAME - $SERVICE_MESH" +echo "Gateway URL: $GATEWAY_URL" + echo "ENDPOINT_URL=$GATEWAY_URL" >> $GITHUB_ENV -echo "SERVICE_MESH=$SERVICE_MESH" >> $GITHUB_ENV \ No newline at end of file +echo "SERVICE_MESH=$SERVICE_MESH" >> $GITHUB_ENV diff --git a/.github/workflows/scripts/linkerd_deploy.sh b/.github/workflows/scripts/linkerd_deploy.sh new file mode 100644 index 0000000..b922183 --- /dev/null +++ b/.github/workflows/scripts/linkerd_deploy.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +# This script is used to deploy Linkerd on Kubernetes +# +# Also deploys EmojiVoto on Linkerd and exposes the service to Meshery + +# See: https://github.com/service-mesh-performance/service-mesh-performance/blob/master/protos/service_mesh.proto +export MESH_NAME='Linkerd' +export SERVICE_MESH=5 + +curl -fsL https://run.linkerd.io/install | sh +export PATH=$PATH:/home/runner/.linkerd2/bin +linkerd version +linkerd check --pre +linkerd install | kubectl apply -f - +linkerd check + +curl -fsL https://run.linkerd.io/emojivoto.yml | kubectl apply -f - +kubectl -n emojivoto port-forward svc/web-svc 8080:80 & +kubectl get -n emojivoto deploy -o yaml | linkerd inject - | kubectl apply -f - + +# Wait for the application to be ready +sleep 100 + +echo "Service Mesh: $MESH_NAME - $SERVICE_MESH" +echo "Endpoint URL: http://localhost:8080" + +# Pass the endpoint to be used by Meshery +echo "ENDPOINT_URL=http://localhost:8080" >> $GITHUB_ENV +echo "SERVICE_MESH=$SERVICE_MESH" >> $GITHUB_ENV diff --git a/.github/workflows/scripts/osm_deploy.sh b/.github/workflows/scripts/osm_deploy.sh new file mode 100644 index 0000000..13008fd --- /dev/null +++ b/.github/workflows/scripts/osm_deploy.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +# This script is used to deploy Open Service Mesh on Kubernetes +# +# Also deploys Bookstore application on OSM + +# See: https://github.com/service-mesh-performance/service-mesh-performance/blob/master/protos/service_mesh.proto +export MESH_NAME='Open Service Mesh' +export SERVICE_MESH='10' + +system=$(uname -s) +release=v0.11.1 +curl -L https://github.com/openservicemesh/osm/releases/download/${release}/osm-${release}-${system}-amd64.tar.gz | tar -vxzf - ./${system}-amd64/osm version +osm install \ + --set=OpenServiceMesh.enablePermissiveTrafficPolicy=true \ + --set=OpenServiceMesh.deployPrometheus=true \ + --set=OpenServiceMesh.deployGrafana=true \ + --set=OpenServiceMesh.deployJaeger=true + +kubectl create namespace bookstore +osm namespace add bookstore +kubectl apply -f https://raw.githubusercontent.com/openservicemesh/osm/release-v0.11/docs/example/manifests/apps/bookstore.yaml +kubectl get deployments -n bookstore +kubectl get pods -n bookstore + +# Wait for the application to be ready +sleep 100 + +# Expose the application outside the cluster +backend="$1" +thisScript="$(dirname "$0")/$(basename "$0")" + +if [ -z "$backend" ]; then + echo "Usage: $thisScript " + exit 1 +fi + +POD="$(kubectl get pods --selector app="$backend" -n "$BOOKSTORE_NAMESPACE" --no-headers | grep 'Running' | awk 'NR==1{print $1}')" +kubectl port-forward "$POD" -n "$BOOKSTORE_NAMESPACE" 15000:15000 + +echo "Service Mesh: $MESH_NAME - $SERVICE_MESH" +echo "Endpoint URL: http://localhost:15000" + +# Pass the endpoint to be used by Meshery +echo "ENDPOINT_URL=http://localhost:15000" >> $GITHUB_ENV +echo "SERVICE_MESH=$SERVICE_MESH" >> $GITHUB_ENV \ No newline at end of file