Merge branch 'main' of https://github.com/swirlai/swirl-search

.env.dist

@@ -11,3 +11,4 @@ SQL_PORT=5432
 MICROSOFT_CLIENT_ID=''
 MICROSOFT_CLIENT_SECRET=''
 MICROSOFT_REDIRECT_URI=''
+CSRF_TRUSTED_ORIGINS='http://localhost:8000'

.env.docker

@@ -13,3 +13,4 @@ MICROSOFT_CLIENT_SECRET=''
 MICROSOFT_REDIRECT_URI='http://localhost:8000/swirl/microsoft-callback'
 CELERY_BROKER_URL='redis://redis:6379/0'
 CELERY_RESULT_BACKEND='redis://redis:6379/0'
+CSRF_TRUSTED_ORIGINS='http://localhost:8000'

.github/workflows/integration-api-tests.yml → .github/workflows/docker-image-sa.yml

@@ -1,28 +1,34 @@
-name: IntegrationAPITests
+name: SA DockerBuild
 
 on:
   workflow_run:
-    workflows: [SmokeTests]
+    workflows: [Docker Build]
     types:
       - completed
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
+# For debugging
+# on:
+#   push:
+#     branches: ''
+
 jobs:
+
   build:
     if: (github.event_name == 'workflow_dispatch') || (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success')
     runs-on: ubuntu-latest
 
     steps:
-    - name: Download branch and run_id artifacts
+    - name: Download Branch and run_id Artifacts
       uses: dawidd6/action-download-artifact@v3
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
-        workflow: smoke-tests.yml
+        workflow: docker-image.yml
         name: branch-info-${{ github.event_name == 'workflow_run' && github.event.workflow_run.id || github.run_id }}
         path: ./artifacts
       continue-on-error: true # Allow the step to fail without stopping the workflow
-    - name: Determine branch for checkout
+    - name: Determine Branch for Checkout
       id: determine_branch
       run: |
         if [[ -f ./artifacts/branch.txt && -f ./artifacts/run_id.txt ]]; then
@@ -32,44 +38,34 @@ jobs:
           BRANCH_NAME=$(echo $GITHUB_REF | cut -d "/" -f 3)
           echo "branch=$BRANCH_NAME" >> $GITHUB_ENV
         fi
-    - name: Print branch to be checked out
+    - name: Print Branch to be Checked Out
       run: |
         echo "Branch to checkout: ${{ env.branch }}"
-    - name: Checkout the code
+    - name: Checkout the Code
       uses: actions/checkout@v4
       with:
         ref: ${{ env.branch }}
-    - name: Set up Python
-      uses: actions/setup-python@v5
+    - name: Login to Docker Hub
+      uses: docker/login-action@v3
       with:
-            python-version: '3.12'
-            cache: 'pip' # caching pip stuff
-    - name: Install Swirl
-      run: ./install.sh
-    - name: Update apt
-      run: sudo apt -o Acquire::Retries=3 update
-    - name: Upgrade Ubuntu to latest patches
-      run: sudo apt upgrade -y
-    - name: Install redis-server
-      run: sudo apt install -y redis-server
-    - name: Set up Swirl
-      run: python swirl.py setup
-    - name: Start Swirl
-      run: python swirl.py start
-    - name: Run integrated API tests
-      run: docker run --net=host -t swirlai/swirl-testing:latest-integrated-api sh -c "behave --tags=integrated_api"
-    - name: Ensure artifacts directory exists and write branch and run_id again
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+    - name: Docker Diagnostics
       run: |
-        mkdir -p ./artifacts
-        echo "${{ env.branch }}" > ./artifacts/branch.txt
-        echo "${{ env.original_run_id }}" > ./artifacts/run_id.txt
-    - name: Re-upload branch and run_id for subsequent workflows
-      uses: actions/upload-artifact@v4
-      with:
-        name: branch-info-${{ github.run_id }}
-        path: |
-          ./artifacts/branch.txt
-          ./artifacts/run_id.txt
+        docker system df
+    - name: Host System Diagnostics
+      run: |
+        df -h
+        du -sh *
+    - name: Docker Cleanup
+      run: |
+        docker system prune -af
+        docker volume prune -f
+        docker builder prune -f
+    - name: Builder Bootstrap
+      run: docker buildx create --name devBuilder --use --bootstrap
+    - name: Build the Docker Image
+      run: docker buildx build -f DevUtils/docker/Dockerfile.sa --platform linux/amd64,linux/arm64 --tag swirlai/swirl-search:latest-sa --push .
     - name: Upload log files
       if: always()
       uses: actions/upload-artifact@v4

.github/workflows/docker-image-spg-experimental.yml

@@ -1,4 +1,4 @@
-name: EXPERIMENT SpyglassDockerBuild
+name: Experimental Spyglass Docker Build
 
 on:
   # Allows you to run this workflow manually from the Actions tab
@@ -38,7 +38,7 @@ jobs:
       run: docker build --no-cache -t swirlai/spyglass:fork-x  -f Dockerfile.fork.spg .
     - name: Push the Docker Image
       run: docker push swirlai/spyglass:fork-x
-    - name: Upload log files
+    - name: Upload Log Files
       if: always()
       uses: actions/upload-artifact@v4
       with:

.github/workflows/docker-image-spg-preview.yml

@@ -1,4 +1,4 @@
-name: PREVIEW Spyglass Docker Build
+name: Preview Spyglass Docker Build
 
 on:
   # Allows you to run this workflow manually from the Actions tab
@@ -37,7 +37,7 @@ jobs:
       run: docker build --no-cache -t swirlai/spyglass:preview  -f Dockerfile.develop.spg .
     - name: Push the Docker Image
       run: docker push swirlai/spyglass:preview
-    - name: Upload log files
+    - name: Upload Log Files
       if: always()
       uses: actions/upload-artifact@v4
       with:

.github/workflows/docker-image-spg.yml

@@ -1,4 +1,4 @@
-name: LatestSpyglassDockerBuild
+name: Latest Spyglass Docker Build
 
 on:
   # Allow manual run of this workflow from the Actions tab
@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - name: Checkout the code
+    - name: Checkout the Code
       uses: actions/checkout@v4
     - name: Login to Docker Hub
       uses: docker/login-action@v3
@@ -37,7 +37,7 @@ jobs:
       run: docker build --no-cache -t swirlai/spyglass:latest -f Dockerfile.spg .
     - name: Push the Docker Image
       run: docker push swirlai/spyglass
-    - name: Upload log files
+    - name: Upload Log Files
       if: always()
       uses: actions/upload-artifact@v4
       with:

.github/workflows/docker-image.yml

@@ -1,4 +1,4 @@
-name: DockerBuild
+name: Docker Build
 
 on:
   # Allows manual run of this workflow from the Actions tab (on any branch)
@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - name: Checkout the code
+    - name: Checkout the Code
       uses: actions/checkout@v4
     - name: Login to Docker Hub
       uses: docker/login-action@v3
@@ -34,19 +34,35 @@ jobs:
         docker system prune -af
         docker volume prune -f
         docker builder prune -f
-    - name: Pull Latest UI Image
-      run: docker pull swirlai/spyglass:latest
     - name: Builder Bootstrap
       run: docker buildx create --name devBuilder --use --bootstrap
     - name: Build the Docker Image
       run: docker buildx build -t swirlai/swirl-search:latest --platform linux/amd64,linux/arm64 --push .
-    - name: Update repo description
+    - name: Update the Docker Repo Description
       uses: peter-evans/dockerhub-description@v4
       with:
         username: ${{ secrets.DOCKER_USERNAME_X }}
         password: ${{ secrets.DOCKER_PASSWORD_X }}
         repository: swirlai/swirl-search
-    - name: Upload log files
+    - name: Create Artifacts Directory
+      run: mkdir -p artifacts
+    - name: Set Branch Name
+      id: extract_branch
+      run: |
+        BRANCH_NAME=$(echo $GITHUB_REF | cut -d "/" -f 3)
+        echo "branch=$BRANCH_NAME" >> $GITHUB_ENV
+    - name: Write Branch and run_id to File
+      run: |
+        echo "${{ env.branch }}" > ./artifacts/branch.txt
+        echo "${{ github.run_id }}" > ./artifacts/run_id.txt
+    - name: Upload Branch and run_id Files as Artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: branch-info-${{ github.run_id }}
+        path: |
+          ./artifacts/branch.txt
+          ./artifacts/run_id.txt
+    - name: Upload Log Files
       if: always()
       uses: actions/upload-artifact@v4
       with:

.github/workflows/pages.yml

@@ -4,7 +4,7 @@
 # documentation.
 
 # Sample workflow for building and deploying a Jekyll site to GitHub Pages
-name: DeployDocsSite
+name: Deploy Documentation
 
 # Modified trigger to only start this workflow on docs dir changes
 on:
@@ -36,25 +36,25 @@ jobs:
       run:
         working-directory: docs # Added working-dir spec for docs dir
     steps:
-      - name: Checkout
+      - name: Checkout the Code
         uses: actions/checkout@v4
-      - name: Setup Ruby
+      - name: Set Up Ruby
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: '3.3' # Not needed with a .ruby-version file
           bundler-cache: true # Runs 'bundle install' and caches installed gems automatically
           cache-version: 0 # Increment this number if you need to re-download cached gems
           working-directory: 'docs' # Added working-dir param here after moving Ruby files to docs/ dir
 
-      - name: Setup Pages
+      - name: Set Up Pages
         id: pages
         uses: actions/configure-pages@v4
       - name: Build with Jekyll
         # Outputs to the './_site' directory by default
         run: bundle exec jekyll build --baseurl "${{ steps.pages.outputs.base_path }}"
         env:
           JEKYLL_ENV: production
-      - name: Upload artifact
+      - name: Upload Artifacts
         # Automatically uploads an artifact from the './_site' directory by default
         uses: actions/upload-pages-artifact@v3
         with:

.github/workflows/smoke-tests.yml → .github/workflows/qa-suite.yml

@@ -1,8 +1,8 @@
-name: SmokeTests
+name: QA Suite
 
 on:
   workflow_run:
-    workflows: [UnitTests]
+    workflows: [Unit Tests]
     types:
       - completed
   # Allows you to run this workflow manually from the Actions tab
@@ -15,15 +15,15 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - name: Download branch and run_id artifacts
+    - name: Download Branch and run_id Artifacts
       uses: dawidd6/action-download-artifact@v3
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
         workflow: unit-tests.yml
         name: branch-info-${{ github.event_name == 'workflow_run' && github.event.workflow_run.id || github.run_id }}
         path: ./artifacts
       continue-on-error: true # Allow the step to fail without stopping the workflow
-    - name: Determine branch for checkout
+    - name: Determine Branch for Checkout
       id: determine_branch
       run: |
         if [[ -f ./artifacts/branch.txt && -f ./artifacts/run_id.txt ]]; then
@@ -33,45 +33,56 @@ jobs:
           BRANCH_NAME=$(echo $GITHUB_REF | cut -d "/" -f 3)
           echo "branch=$BRANCH_NAME" >> $GITHUB_ENV
         fi
-    - name: Print branch to be checked out
+    - name: Print Branch to be Checked Out
       run: |
         echo "Branch to checkout: ${{ env.branch }}"
-    - name: Checkout the code
+    - name: Checkout the Code
       uses: actions/checkout@v4
       with:
         ref: ${{ env.branch }}
-    - name: Set up Python
+    - name: Set Up Python
       uses: actions/setup-python@v5
       with:
-            python-version: '3.12'
+            python-version: '3.12.2'
             cache: 'pip' # caching pip stuff
+    - name: Install Chromium
+      uses: browser-actions/setup-chrome@v1
+    - name: Chromium Install Details
+      run: |
+        chromium --version
+        which chromium
     - name: Install Swirl
       run: ./install.sh
     - name: Update apt
       run: sudo apt -o Acquire::Retries=3 update
-    - name: Upgrade Ubuntu to latest patches
+    - name: Upgrade Ubuntu to Latest Patches
       run: sudo apt upgrade -y
     - name: Install redis-server
       run: sudo apt install -y redis-server
-    - name: Set up Swirl
+    - name: Set Up Swirl
       run: python swirl.py setup
     - name: Start Swirl
       run: python swirl.py start
-    - name: Run smoke tests
-      run: docker run --net=host -t swirlai/swirl-testing:latest-smoke-test sh -c "behave **/docker_container/*.feature --tags=docker_api_smoke"
-    - name: Ensure artifacts directory exists and write branch and run_id again
+    - name: Login to Docker Hub
+      uses: docker/login-action@v3
+      with:
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+    - name: Run the QA Suite
+      run: docker run --net=host -t swirlai/swirl-search-qa:automated-tests sh -c "behave --tags=qa_suite"
+    - name: Ensure Artifacts Directory Exists and Write Branch and run_id Again
       run: |
         mkdir -p ./artifacts
         echo "${{ env.branch }}" > ./artifacts/branch.txt
         echo "${{ env.original_run_id }}" > ./artifacts/run_id.txt
-    - name: Re-upload branch and run_id for subsequent workflows
+    - name: Re-upload Branch and run_id for Subsequent Workflows
       uses: actions/upload-artifact@v4
       with:
         name: branch-info-${{ github.run_id }}
         path: |
           ./artifacts/branch.txt
           ./artifacts/run_id.txt
-    - name: Upload log files
+    - name: Upload Log Files
       if: always()
       uses: actions/upload-artifact@v4
       with: