From 486ee57bf8123116a326d1433bcb1b36ff88ee67 Mon Sep 17 00:00:00 2001
From: sigoden <sigoden@gmail.com>
Date: Fri, 10 Jan 2025 17:31:41 +0800
Subject: [PATCH] feat: discontinue `rag_min_score_*` config items (#1081)

---
 config.example.yaml | 12 +++++-------
 src/config/mod.rs   | 27 +--------------------------
 src/rag/mod.rs      | 16 +++-------------
 3 files changed, 9 insertions(+), 46 deletions(-)

diff --git a/config.example.yaml b/config.example.yaml
index aa26de7c..d629a5ee 100644
--- a/config.example.yaml
+++ b/config.example.yaml
@@ -35,13 +35,11 @@ summary_prompt: 'This is a summary of the chat history as a recap: '
 
 # ---- RAG ----
 # See [RAG-Guide](https://github.com/sigoden/aichat/wiki/RAG-Guide) for more details.
-rag_embedding_model: null                   # Specifies the embedding model to use
-rag_reranker_model: null                    # Specifies the rerank model to use
-rag_top_k: 5                                # Specifies the number of documents to retrieve
-rag_chunk_size: null                        # Specifies the chunk size
-rag_chunk_overlap: null                     # Specifies the chunk overlap
-rag_min_score_vector_search: 0              # Specifies the minimum relevance score for vector-based searching
-rag_min_score_keyword_search: 0             # Specifies the minimum relevance score for keyword-based searching
+rag_embedding_model: null        # Specifies the embedding model used for context retrieval
+rag_reranker_model: null         # Specifies the reranker model used for sorting retrieved documents
+rag_top_k: 5                     # Specifies the number of documents to retrieve for answering queries
+rag_chunk_size: null             # Defines the size of chunks for document processing in characters
+rag_chunk_overlap: null          # Defines the overlap between chunks
 # Defines the query structure using variables like __CONTEXT__ and __INPUT__ to tailor searches to specific needs
 rag_template: |
   Answer the query based on the context while respecting the rules. (user query, some textual context and rules, all inside xml tags)
diff --git a/src/config/mod.rs b/src/config/mod.rs
index ca0990a4..407d0414 100644
--- a/src/config/mod.rs
+++ b/src/config/mod.rs
@@ -123,8 +123,6 @@ pub struct Config {
     pub rag_top_k: usize,
     pub rag_chunk_size: Option<usize>,
     pub rag_chunk_overlap: Option<usize>,
-    pub rag_min_score_vector_search: f32,
-    pub rag_min_score_keyword_search: f32,
     pub rag_template: Option<String>,
 
     #[serde(default)]
@@ -197,8 +195,6 @@ impl Default for Config {
             rag_top_k: 5,
             rag_chunk_size: None,
             rag_chunk_overlap: None,
-            rag_min_score_vector_search: 0.0,
-            rag_min_score_keyword_search: 0.0,
             rag_template: None,
 
             document_loaders: Default::default(),
@@ -1413,22 +1409,8 @@ impl Config {
         abort_signal: AbortSignal,
     ) -> Result<String> {
         let (reranker_model, top_k) = rag.get_config();
-        let (min_score_vector_search, min_score_keyword_search) = {
-            let config = config.read();
-            (
-                config.rag_min_score_vector_search,
-                config.rag_min_score_keyword_search,
-            )
-        };
         let (embeddings, ids) = rag
-            .search(
-                text,
-                top_k,
-                min_score_vector_search,
-                min_score_keyword_search,
-                reranker_model.as_deref(),
-                abort_signal,
-            )
+            .search(text, top_k, reranker_model.as_deref(), abort_signal)
             .await?;
         let text = config.read().rag_template(&embeddings, text);
         rag.set_last_sources(&ids);
@@ -2222,13 +2204,6 @@ impl Config {
         if let Some(v) = read_env_value::<usize>(&get_env_name("rag_chunk_overlap")) {
             self.rag_chunk_overlap = v;
         }
-        if let Some(Some(v)) = read_env_value::<f32>(&get_env_name("rag_min_score_vector_search")) {
-            self.rag_min_score_vector_search = v;
-        }
-        if let Some(Some(v)) = read_env_value::<f32>(&get_env_name("rag_min_score_keyword_search"))
-        {
-            self.rag_min_score_keyword_search = v;
-        }
         if let Some(v) = read_env_value::<String>(&get_env_name("rag_template")) {
             self.rag_template = v;
         }
diff --git a/src/rag/mod.rs b/src/rag/mod.rs
index ceb83f69..8855f4fa 100644
--- a/src/rag/mod.rs
+++ b/src/rag/mod.rs
@@ -297,19 +297,11 @@ impl Rag {
         &self,
         text: &str,
         top_k: usize,
-        min_score_vector_search: f32,
-        min_score_keyword_search: f32,
         rerank_model: Option<&str>,
         abort_signal: AbortSignal,
     ) -> Result<(String, Vec<DocumentId>)> {
         let ret = abortable_run_with_spinner(
-            self.hybird_search(
-                text,
-                top_k,
-                min_score_vector_search,
-                min_score_keyword_search,
-                rerank_model,
-            ),
+            self.hybird_search(text, top_k, rerank_model),
             "Searching",
             abort_signal,
         )
@@ -497,13 +489,11 @@ impl Rag {
         &self,
         query: &str,
         top_k: usize,
-        min_score_vector_search: f32,
-        min_score_keyword_search: f32,
         rerank_model: Option<&str>,
     ) -> Result<Vec<(DocumentId, String)>> {
         let (vector_search_results, keyword_search_results) = tokio::join!(
-            self.vector_search(query, top_k, min_score_vector_search),
-            self.keyword_search(query, top_k, min_score_keyword_search)
+            self.vector_search(query, top_k, 0.0),
+            self.keyword_search(query, top_k, 0.0),
         );
 
         let vector_search_results = vector_search_results?;