Port 4908 investigate sonarqube ocean bug (#182)

integrations/gitlab/README.md

@@ -14,7 +14,7 @@ For more information about the installation visit the [Port Ocean helm chart](ht
 # The following script will install an Ocean integration at your K8s cluster using helm
 # integration.identifier: Change the identifier to describe your integration
 # integration.secrets.tokenMapping: Mapping of Gitlab tokens to Port Ocean tokens. example: {"THE_GROUP_TOKEN":["getport-labs/**", "GROUP/PROJECT PATTERN TO RUN FOR"]}
-# integration.config.appHost: The host of the Gitlab instance. If not specified, the default will be https://gitlab.com.
+# integration.config.appHost: The host of the Port Ocean app. Used for setting up the webhooks against the Gitlab.
 # ingress.annotations."nginx\.ingress\.kubernetes\.io/rewrite-target": Change the annotation value and key to match your ingress controller
 
 helm upgrade --install my-gitlab-integration port-labs/port-ocean \

integrations/sonarqube/.port/resources/port-app-config.yaml

@@ -1,3 +1,5 @@
+createMissingRelatedEntities: true
+deleteDependentEntities: true
 resources:
   - kind: projects
     selector:

integrations/sonarqube/CHANGELOG.md

@@ -7,6 +7,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 <!-- towncrier release notes start -->
 
+# Port_Ocean 0.1.5 (2023-10-22)
+
+### Features
+
+- Added a sanity check for sonarqube to check the sonarqube instance is accessible before starting the integration (PORT4908)
+
+### Improvements
+
+- Updated integration default port app config to have the `createMissingRelatedEntities` & `deleteDependentEntities` turned on by default (PORT-4908)
+- Change organizationId configuration to be optional for on prem installation (PORT-4908)
+- Added more verbose logging for the http request errors returning from the sonarqube (PORT-4908)
+- Updated integration default port app config to have the  &  turned on by default (PORT4908)
+
+### Bug Fixes
+
+- Changed the sonarqube api authentication to use basic auth for on prem installations (PORT-4908)
+
+
 # Sonarqube 0.1.4 (2023-10-15)
 
 ### Improvements

integrations/sonarqube/client.py

@@ -15,13 +15,17 @@ class Endpoints:
 
 class SonarQubeClient:
     def __init__(
-        self, base_url: str, api_key: str, organization_id: str, app_host: str
+        self,
+        base_url: str,
+        api_key: str,
+        organization_id: str | None,
+        app_host: str | None,
     ):
-        self.base_url = base_url or "https://sonarcloud.io"
+        self.base_url = base_url
         self.api_key = api_key
         self.organization_id = organization_id
         self.app_host = app_host
-        self.http_client = httpx.AsyncClient(headers=self.api_auth_header)
+        self.http_client = httpx.AsyncClient(**self.api_auth_params)
         self.metrics = [
             "code_smells",
             "coverage",
@@ -32,10 +36,19 @@ def __init__(
         ]
 
     @property
-    def api_auth_header(self) -> dict[str, Any]:
+    def api_auth_params(self) -> dict[str, Any]:
+        if self.organization_id:
+            return {
+                "headers": {
+                    "Authorization": f"Bearer {self.api_key}",
+                    "Content-Type": "application/json",
+                }
+            }
         return {
-            "Authorization": f"Bearer {self.api_key}",
-            "Content-Type": "application/json",
+            "auth": (self.api_key, ""),
+            "headers": {
+                "Content-Type": "application/json",
+            },
         }
 
     async def send_api_request(
@@ -51,7 +64,6 @@ async def send_api_request(
                 url=f"{self.base_url}/api/{endpoint}",
                 params=query_params,
                 json=json_data,
-                headers=self.api_auth_header,
             )
             response.raise_for_status()
             return response.json()
@@ -78,7 +90,6 @@ async def send_paginated_api_request(
                     url=f"{self.base_url}/api/{endpoint}",
                     params=query_params,
                     json=json_data,
-                    headers=self.api_auth_header,
                 )
                 response.raise_for_status()
                 response_json = response.json()
@@ -102,20 +113,30 @@ async def send_paginated_api_request(
                 f"HTTP error with status code: {e.response.status_code} and response text: {e.response.text}"
             )
             raise
+        except httpx.HTTPError as e:
+            logger.error(f"HTTP occurred while fetching paginated data: {e}")
+            raise
 
-    async def get_components(self) -> list[Any]:
+    async def get_components(self) -> list[dict[str, Any]]:
         """
         Retrieve all components from SonarQube organization.
 
         :return: A list of components associated with the specified organization.
         """
+        params = {}
+        if self.organization_id:
+            params["organization"] = self.organization_id
         logger.info(f"Fetching all components in organization: {self.organization_id}")
-        response = await self.send_paginated_api_request(
-            endpoint=Endpoints.PROJECTS,
-            data_key="components",
-            query_params={"organization": self.organization_id},
-        )
-        return response
+        try:
+            response = await self.send_paginated_api_request(
+                endpoint=Endpoints.PROJECTS,
+                data_key="components",
+                query_params=params,
+            )
+            return response
+        except Exception as e:
+            logger.error(f"Error occurred while fetching components: {e}")
+            raise
 
     async def get_single_component(self, project: dict[str, Any]) -> dict[str, Any]:
         """
@@ -312,6 +333,24 @@ async def get_analysis_for_task(
                 return analysis_object
         return {}  ## when no data is found
 
+    def sanity_check(self) -> None:
+        try:
+            response = httpx.get(f"{self.base_url}/api/system/status", timeout=5)
+            response.raise_for_status()
+            logger.info("Sonarqube sanity check passed")
+            logger.info(f"Sonarqube status: {response.json().get('status')}")
+            logger.info(f"Sonarqube version: {response.json().get('version')}")
+        except httpx.HTTPStatusError as e:
+            logger.error(
+                f"Sonarqube failed connectivity check to the sonarqube instance because of HTTP error: {e.response.status_code} and response text: {e.response.text}"
+            )
+            raise
+        except httpx.HTTPError as e:
+            logger.error(
+                f"Sonarqube failed connectivity check to the sonarqube instance because of HTTP error: {e}"
+            )
+            raise
+
     async def get_or_create_webhook_url(self) -> None:
         """
         Get or create webhook URL for projects
@@ -328,11 +367,14 @@ async def get_or_create_webhook_url(self) -> None:
         for project in projects:
             project_key = project["key"]
             logger.info(f"Fetching existing webhooks in project: {project_key}")
+            params = {}
+            if self.organization_id:
+                params["organization"] = self.organization_id
             webhooks_response = await self.send_api_request(
                 endpoint=f"{webhook_endpoint}/list",
                 query_params={
                     "project": project_key,
-                    "organization": self.organization_id,
+                    **params,
                 },
             )
 
@@ -342,11 +384,15 @@ async def get_or_create_webhook_url(self) -> None:
             if any(webhook["url"] == invoke_url for webhook in webhooks):
                 logger.info(f"Webhook already exists in project: {project_key}")
                 continue
+
+            params = {}
+            if self.organization_id:
+                params["organization"] = self.organization_id
             webhooks_to_create.append(
                 {
                     "name": "Port Ocean Webhook",
                     "project": project_key,
-                    "organization": self.organization_id,
+                    **params,
                 }
             )
 

integrations/sonarqube/config.yaml

@@ -14,4 +14,3 @@ integration:
   type: "sonarqube"
   config:
     sonarApiToken: "{{ from env SONAR_API_TOKEN }}"
-    sonarOrganizationID: "{{ from env SONAR_ORGANIZATION_ID }}"

integrations/sonarqube/main.py

@@ -1,8 +1,10 @@
 from typing import Any
+
 from loguru import logger
+
+from client import SonarQubeClient
 from port_ocean.context.ocean import ocean
 from port_ocean.core.ocean_types import ASYNC_GENERATOR_RESYNC_TYPE
-from client import SonarQubeClient
 
 
 class ObjectKind:
@@ -15,8 +17,8 @@ def init_sonar_client() -> SonarQubeClient:
     return SonarQubeClient(
         ocean.integration_config.get("sonar_url", "https://sonarcloud.io"),
         ocean.integration_config["sonar_api_token"],
-        ocean.integration_config.get("sonar_organization_id", ""),
-        ocean.integration_config.get("app_host", ""),
+        ocean.integration_config.get("sonar_organization_id"),
+        ocean.integration_config.get("app_host"),
     )
 
 
@@ -92,9 +94,10 @@ async def handle_sonarqube_webhook(webhook_data: dict[str, Any]) -> None:
 
 @ocean.on_start()
 async def on_start() -> None:
+    sonar_client = init_sonar_client()
+    sonar_client.sanity_check()
     if organization_key_missing_for_onpremise():
         logger.warning("Organization key is missing for an on-premise Sonarqube setup")
     ## We are making the real-time subscription of Sonar webhook events optional. That said, we only subscribe to webhook events when the user supplies the app_host config variable
     if ocean.integration_config.get("app_host"):
-        sonar_client = init_sonar_client()
         await sonar_client.get_or_create_webhook_url()

integrations/sonarqube/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sonarqube"
-version = "0.1.4"
+version = "0.1.5"
 description = "SonarQube projects and code quality analysis integration"
 authors = ["Port Team <[email protected]>"]