Merge pull request #1434 from swirlai/DS-2944

Ds 2944
swirlai · Sep 23, 2024 · 4a1fa2d · 4a1fa2d
2 parents 42beec0 + 77c11fd
commit 4a1fa2d
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 19 deletions.
diff --git a/swirl/models.py b/swirl/models.py
@@ -147,7 +147,7 @@ def getSearchPreQueryProcessorsDefault():
     return []
 
 def getSearchPostResultProcessorsDefault():
-    return ["DedupeByFieldPostResultProcessor","CosineRelevancyPostResultProcessor", "RedactPIIPostResultProcessor"]
+    return ["DedupeByFieldPostResultProcessor","CosineRelevancyPostResultProcessor"]
 
 class Search(models.Model):
     id = models.BigAutoField(primary_key=True)

diff --git a/swirl/processors/remove_pii.py b/swirl/processors/remove_pii.py
@@ -38,13 +38,13 @@ def remove_pii(text: str, query_string=None, redact=False) -> str:
     By default, Presidio redacts entities, replacing it with <entity-type>.
     The Presidio "redact" option removes the PII entirely.
     In SWIRL, remove means "remove the PII" and "redact" means "replace it with <entity-type>".
-    
+
     :param text: The input string (either query or result) to clean.
     :return: The text with PII removed.
     """
 
-    text = remove_tags(text)
-    pii_entities = analyzer.analyze(text=text, language='en')  
+    untagged_text = remove_tags(text)
+    pii_entities = analyzer.analyze(text=untagged_text, language='en')
 
     if not pii_entities:
         return text
@@ -55,11 +55,11 @@ def remove_pii(text: str, query_string=None, redact=False) -> str:
         operators = {"DEFAULT": OperatorConfig("replace")}
 
     anonymized_result = anonymizer.anonymize(
-        text=text, 
-        analyzer_results=pii_entities, 
+        text=untagged_text,
+        analyzer_results=pii_entities,
         operators=operators
     )
-    
+
     anonymized_text = anonymized_result.text
 
     if redact:
@@ -77,17 +77,17 @@ class RemovePIIQueryProcessor(QueryProcessor):
     """
     A SWIRL metasearch query processor that removes PII from search queries.
     """
-    
+
     type = 'RemovePIIQueryProcessor'
 
     def process(self) -> str:
         """
         :return: The processed query with PII removed.
         """
-        
+
         # Remove PII from the query
         cleaned_query = remove_pii(self.query_string)
-        
+
         return cleaned_query
 
 #############################################
@@ -97,19 +97,19 @@ class RedactPIIResultProcessor(ResultProcessor):
     A SWIRL result processor that removes PII from the search results.
     Meant to be run after CosineResultProcessor.
     """
-    
+
     type = "RemovePIIResultProcessor"
 
     def process(self) -> int:
         """
         :return: The number of modified results.
         """
         logger.debug(f"Processing {len(self.results)} results for PII removal.")
-        
+
         modified = 0
         for item in self.results:
             pii_modified = False
-            
+
             # Remove PII from 'title' and 'body' fields of each result
             if 'title' in item:
                 cleaned_title = redact_pii(item['title'], self.query_string)
@@ -131,14 +131,14 @@ def process(self) -> int:
                     if cleaned_payload != item['payload'][key]:
                         item['payload'][key] = cleaned_payload
                         pii_modified = True
-            
+
             if pii_modified:
                 modified += 1
 
         self.processed_results = self.results
         self.modified = modified
         logger.debug(f"PII removal complete. {self.modified} results modified.")
-        
+
         return self.modified
 
 #############################################
@@ -147,16 +147,16 @@ class RedactPIIPostResultProcessor(PostResultProcessor):
     """
     A SWIRL result processor that removes PII from all results.
     """
-    
+
     type = "RemovePIIPostResultProcessor"
 
     def process(self) -> int:
         """
         :return: The number of modified results.
         """
-        
+
         modified = 0
-        
+
         for result in self.results:
             for item in result.json_results:
                 pii_modified = False
@@ -182,5 +182,5 @@ def process(self) -> int:
                     modified += 1
             result.save()
 
-        self.results_updated = modified        
+        self.results_updated = modified
         return self.results_updated