Wwi21/46 implementing rate limiting (#122)

* Basic logic for token checking

* Initial Firebase Setup

* Implemented token system

* Linting checks

* Linting checks

* firebase_admin dependency added

* added requirement

* firebase_admin added to specific requirements

* adds service account certificate env var from github secrests

* removes superflous dependency 'pnmlbpmntransformer'

* 💚

* 💚 Changes service account certificate env var handling

* Check Tokens Function added

* adds basic decodation of base64 service account certificate

* Refactor service account certificate handling

* fixes set_force_std_xml flag position

* now uses correct var for decoding base64

* added tempfile for secret

* remove unused dependency

* test prints added

* test

* test

* test

* test

* test

* test

* test

* test

* test

* test

* test

* test

* t

* test

* test

* test

* test

* test

* improved code quality

* final touches

---------

Co-authored-by: Jeldrik Merkelbach <[email protected]>
Co-authored-by: Niyada <[email protected]>

.gcloudignore

@@ -0,0 +1,17 @@
+# This file specifies files that are *not* uploaded to Google Cloud
+# using gcloud. It follows the same syntax as .gitignore, with the addition of
+# "#!include" directives (which insert the entries of the given .gitignore-style
+# file at that point).
+#
+# For more information, run:
+#   $ gcloud topic gcloudignore
+#
+.gcloudignore
+# If you would like to upload your .git directory, .gitignore file or files
+# from your .gitignore file, remove the corresponding line
+# below:
+.git
+.gitignore
+
+node_modules
+#!include:.gitignore

.github/workflows/deploy.yml

@@ -77,6 +77,11 @@ jobs:
             entry_point: "post_transform"
             source_dir: "src/transform"
             description: "Transformation endpoint."
+          - name: checkTokens
+            test_dir: "checkTokens"
+            entry_point: "check_tokens"
+            source_dir: "src/checkTokens"
+            description: "[CANARY] Check Tokens endpoint."
             set_force_std_xml: true
 
     permissions:

.github/workflows/integration.yml

@@ -33,6 +33,11 @@ jobs:
             source_dir: "src/transform"
             description: "[CANARY] Transformation endpoint."
             set_force_std_xml: true
+          - name: checkTokens
+            test_dir: "checkTokens"
+            entry_point: "check_tokens"
+            source_dir: "src/checkTokens"
+            description: "[CANARY] Check Tokens endpoint."
 
     # Add "id-token" with the intended permissions.
     permissions:
@@ -48,18 +53,20 @@ jobs:
           PIPELINE_TIMESTAMP=$(date +%Y%m%d%H%M%S)
           echo "PIPELINE_TIMESTAMP=$PIPELINE_TIMESTAMP" >> $GITHUB_ENV
           echo "CANARY_FUNCTION_NAME=CANARY_${{ matrix.function.name }}_${{ github.event.pull_request.number || 'manual' }}_$PIPELINE_TIMESTAMP" >> $GITHUB_ENV
+          echo "GCP_SERVICE_ACCOUNT_CERTIFICATE=${{ secrets.GCP_SERVICE_ACCOUNT_CERTIFICATE }}" >> $GITHUB_ENV
+
 
       # Set FORCE_STD_XML for transform function only if the flag exists and is true
       - name: Set FORCE_STD_XML if required
         if: matrix.function.set_force_std_xml == 'true'
         run: echo "FORCE_STD_XML=true" >> $GITHUB_ENV
 
-      - name: Set FUNCTION_ENV_VARS
+      - name: Set FORCE_STD_XML_ENV_KV_PAIR
         run: |
           if [ "${{ matrix.function.set_force_std_xml }}" == "true" ]; then
-            echo "FUNCTION_ENV_VARS=FORCE_STD_XML=true" >> $GITHUB_ENV
+            echo "FORCE_STD_XML_ENV_KV_PAIR=FORCE_STD_XML=true" >> $GITHUB_ENV
           else
-            echo "FUNCTION_ENV_VARS=" >> $GITHUB_ENV
+            echo "FORCE_STD_XML_ENV_KV_PAIR=" >> $GITHUB_ENV
           fi
 
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
@@ -110,7 +117,7 @@ jobs:
           runtime: "python312"
           region: "europe-west3"
           description: ${{ matrix.function.description }}
-          env_vars: ${{ env.FUNCTION_ENV_VARS }}
+          env_vars: ${{ env.FORCE_STD_XML_ENV_KV_PAIR }},GCP_SERVICE_ACCOUNT_CERTIFICATE=${{ secrets.GCP_SERVICE_ACCOUNT_CERTIFICATE }}
 
       # Authenticating at GCP to get id token for deployed function
       # see https://github.com/google-github-actions/deploy-cloud-functions

.gitignore

@@ -13,5 +13,8 @@ src/transform/test_log
 # VS Code
 .vscode/
 
+# Service Account Secrets
+secrets/
+
 # MacOS
 .DS_Store

requirements.txt

@@ -24,6 +24,7 @@ urllib3==2.2.1
 watchdog==4.0.1
 Werkzeug==3.0.3
 wheel==0.43.0
+firebase_admin==6.5.0
 # 
 lxml==5.2.2
 pydantic==2.7.4

src/app.py

@@ -3,6 +3,7 @@
 from flask import Flask, request
 from health.main import get_health
 from transform.main import post_transform
+from checkTokens.main import check_Tokens
 from flask_cors import CORS
 
 
@@ -14,12 +15,15 @@ def health_route():
     """Mapping route for health endpoint."""
     return get_health(request)
 
-
 @app.route('/transform', methods=['POST'])
 def transform_route():
     """Mapping route for transform endpoint."""
     return post_transform(request)
 
+@app.route('/checkTokens', methods=['GET'])
+def checkTokens_route():
+    """Mapping route for checkTokens endpoint."""
+    return check_Tokens(request)
 
 if __name__ == '__main__':
     app.run(host='0.0.0.0', port=8080)

src/checkTokens/main.py

@@ -0,0 +1,45 @@
+"""Implements the 'check_Tokens' HTTP Cloud Function.
+
+This module defines a Google Cloud Function for RateLimiting the transform Endpoint.
+"""
+import base64
+import firebase_admin
+import functions_framework
+import json
+from firebase_admin import credentials, firestore
+from flask import jsonify
+import os
+
+GCP_SERVICE_ACCOUNT_CERTIFICATE_BASE64 = os.getenv( "GCP_SERVICE_ACCOUNT_CERTIFICATE" )
+if( GCP_SERVICE_ACCOUNT_CERTIFICATE_BASE64 is None ):
+    print( "Env var GCP_SERVICE_ACCOUNT_CERTIFICATE not found!" )
+
+GCP_SERVICE_ACCOUNT_CERTIFICATE_DECODED_BYTES = \
+    base64.b64decode(GCP_SERVICE_ACCOUNT_CERTIFICATE_BASE64)
+
+GCP_SERVICE_ACCOUNT_CERTIFICATE_DECODED_STRING = \
+    GCP_SERVICE_ACCOUNT_CERTIFICATE_DECODED_BYTES.decode('utf-8')
+
+cred_dict = json.loads(GCP_SERVICE_ACCOUNT_CERTIFICATE_DECODED_STRING, strict=False)
+cred = credentials.Certificate(cred_dict)
+firebase_admin.initialize_app(cred)
+db = firestore.client()
+
+@functions_framework.http
+def check_tokens(request):
+    """Check if there are tokens available in the Firestore database."""
+    if db is None:
+        return jsonify({"error": "No database available"}), 500
+
+    doc_ref = db.collection("api-tokens").document("token-document")
+    doc = doc_ref.get()
+    if doc.exists:
+        tokens = doc.to_dict().get("tokens", 0)
+        if tokens <= 0:
+            return jsonify({"error": "No tokens available"}), 400
+        else:
+            doc_ref.update({"tokens": tokens-1})
+            return jsonify({"tokens": tokens-1}), 200
+    else:
+        return jsonify({"error": "No document available"}), 404
+

src/checkTokens/requirements.txt

@@ -0,0 +1,2 @@
+firebase_admin==6.5.0
+functions-framework==3.7.0

src/transform/main.py

@@ -1,7 +1,7 @@
 """API to transform a given model into a selected direction."""
+import requests
 
 import os
-
 import flask
 import functions_framework
 from flask import jsonify, make_response
@@ -15,6 +15,14 @@
 from transformer.transform_petrinet_to_bpmn.transform import pnml_to_bpmn
 from transformer.utility.utility import clean_xml_string
 
+CHECK_TOKEN_URL = 'https://europe-west3-woped-422510.cloudfunctions.net/checkTokens'
+
+is_force_std_xml_active = os.getenv("FORCE_STD_XML")
+if is_force_std_xml_active is None:
+    raise Exception("Env variable is_force_std_xml_active not set!")
+
+
+
 is_force_std_xml_active = os.getenv("FORCE_STD_XML")
 if is_force_std_xml_active is None:
     raise Exception("Env variable is_force_std_xml_active not set!")
@@ -30,7 +38,14 @@ def post_transform(request: flask.Request):
     Args:
         request: A request with a parameter "direction" as transformation direction
         and a form with the xml model "bpmn" or "pnml".
-    """
+    """    
+    try:
+        response = requests.get(CHECK_TOKEN_URL)
+        if response.status_code == 400:
+            raise Exception("Token check not successful")
+    except Exception:
+        return jsonify({"error": "Token check not successful"}), 400
+
     if request.method == 'OPTIONS':
         # Handle CORS preflight request
         response = make_response()

src/transform/requirements.txt

@@ -1,4 +1,7 @@
+# test index
+-i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple
+firebase_admin==6.5.0
 lxml==5.2.2
 pydantic==2.7.4
 pydantic_xml==2.11.0
-defusedxml==0.7.1
+defusedxml==0.7.1

test/e2e/checkTokens/__init__.py

@@ -0,0 +1,2 @@
+
+"""This is the __init__ module for the unit tests."""

test/e2e/checkTokens/test_check_tokens.py

@@ -0,0 +1,10 @@
+"""Unit tests for the transform endpoint of the application."""
+
+import unittest
+
+class TestUnitCheckTokens(unittest.TestCase):
+    """A unit test class for testing the CheckTokens Endpoint of the application."""
+
+    def test_boilerplate(self):
+        """Boilerplate empty test function. This will always pass."""
+        self.assertTrue(True)

test/integration/checkTokens/__init__.py

@@ -0,0 +1,2 @@
+
+"""This is the __init__ module for the unit tests."""

test/integration/checkTokens/test_check_tokens.py

@@ -0,0 +1,10 @@
+"""Integration tests for the transform endpoint of the application."""
+
+import unittest
+
+class TestUnitCheckTokens(unittest.TestCase):
+    """An integration test class for testing the CheckTokens endpoint of the applic.."""
+
+    def test_boilerplate(self):
+        """Boilerplate empty test function. This will always pass."""
+        self.assertTrue(True)

test/unit/checkTokens/__init__.py

@@ -0,0 +1,2 @@
+
+"""This is the __init__ module for the unit tests."""

test/unit/checkTokens/test_check_tokens.py

@@ -0,0 +1,10 @@
+"""Unit tests for the transform endpoint of the application."""
+
+import unittest
+
+class TestUnitCheckTokens(unittest.TestCase):
+    """A unit test class for testing the CheckTokens Endpoint of the application."""
+
+    def test_boilerplate(self):
+        """Boilerplate empty test function. This will always pass."""
+        self.assertTrue(True)