Interactive RAG With MongoDB Atlas + Function Calling API

1
 # Chunk Ingest Strategy
2
 self.text_splitter = RecursiveCharacterTextSplitter(
3
        	# Set a really small chunk size, just to show.
4
        	chunk_size=4000, # THIS CHUNK SIZE IS FIXED - INGEST CHUNK SIZE DOES NOT CHANGE
5
        	chunk_overlap=200, # CHUNK OVERLAP IS FIXED
6
        	length_function=len,
7
        	add_start_index=True,
8
 )
9
 # load data from webpages using Playwright. One document will be created for each webpage
10
 # split the documents using a text splitter to create "chunks"
11
 loader = PlaywrightURLLoader(urls=urls, remove_selectors=["header", "footer"])  
12
 documents = loader.load_and_split(self.text_splitter)
13
 self.index.add_documents(
14
   		 documents
15
 )   

1
	{
2
    	"name": "<index-name>",
3
    	"type": "vectorSearch",
4
    	"fields":[
5
        	{
6
        	"type": "vector",
7
        	"path": <field-to-index>,
8
        	"numDimensions": <number-of-dimensions>,
9
        	"similarity": "euclidean | cosine | dotProduct"
10
        	},
11
        	...
12
    	]
13
	}

1
def recall(self, text, n_docs=2, min_rel_score=0.25, chunk_max_length=800,unique=True):
2
	 #$vectorSearch
3
	 print("recall=>"+str(text))
4
	 response = self.collection.aggregate([
5
	 {
6
   		 "$vectorSearch": {
7
       		 "index": "default",
8
       		 "queryVector": self.gpt4all_embd.embed_query(text), #GPT4AllEmbeddings()
9
       		 "path": "embedding",
10
       		 #"filter": {},
11
       		 "limit": 15, #Number (of type int only) of documents to return in the results. Value can't exceed the value of numCandidates.
12
       		 "numCandidates": 50 #Number of nearest neighbors to use during the search. You can't specify a number less than the number of documents to return (limit).
13
   		 }
14
	 },
15
	 {
16
   		 "$addFields":
17
   		 {
18
       		 "score": {
19
       		 "$meta": "vectorSearchScore"
20
   		 }
21
	 }
22
	 },
23
	 {
24
   		 "$match": {
25
       		 "score": {
26
       		 "$gte": min_rel_score
27
   		 }
28
	 }
29
	 },{"$project":{"score":1,"_id":0, "source":1, "text":1}}])
30
	 tmp_docs = []
31
	 str_response = []
32
	 for d in response:
33
   		 if len(tmp_docs) == n_docs:
34
       		 break
35
   		 if unique and d["source"] in tmp_docs:
36
       		 continue
37
   		 tmp_docs.append(d["source"])
38
   		 str_response.append({"URL":d["source"],"content":d["text"][:chunk_max_length],"score":d["score"]})
39
	 kb_output = f"Knowledgebase Results[{len(tmp_docs)}]:\n```{str(str_response)}```\n## \n```SOURCES: "+str(tmp_docs)+"```\n\n"
40
	 self.st.write(kb_output)
41
	 return str(kb_output)