RAG Evaluation: Detecting Hallucinations With Patronus AI and MongoDB

1
    import pymongo
2
    from pymongo.mongo_client import MongoClient
3
    from pymongo.operations import SearchIndexModel
4
5
    # Connect to your Atlas deployment
6
    uri = "<connectionString>"
7
    client = MongoClient(uri)
8
9
    # Access your database and collection
10
    database = client["<databaseName>"]
11
    collection = database["<collectionName>"]
12
13
    # Create your index model, then create the search index
14
    search_index_model = SearchIndexModel(
15
      definition={
16
        "fields": [
17
          {
18
            "type": "vector",
19
            "numDimensions": <numberofDimensions>,
20
            "path": "<fieldToIndex>",
21
            "similarity": "euclidean | cosine | dotProduct"
22
          },
23
          {
24
            "type": "filter",
25
            "path": "<fieldToIndex>"
26
          },
27
          ...
28
        ]
29
      },
30
      name="<index name>",
31
      type="vectorSearch",
32
    )
33
34
    result = collection.create_search_index(model=search_index_model)
35
    print(result)

1
    import pymongo
2
    import datetime
3
4
    # connect to your Atlas cluster
5
    client = pymongo.MongoClient("<connection-string>")
6
7
    # define pipeline
8
    pipeline = [
9
      {
10
        '$vectorSearch': {
11
          'index': "<index-name>", 
12
          'path':  "<field-to-search>",
13
          'queryVector': [<array-of-numbers>]
14
          'numCandidates': <number-of-candidates>,
15
          'limit': <number-of-results>
16
        }
17
      }
18
    ]
19
20
    # run pipeline
21
    result = client["db_name"][collection_name"].aggregate(pipeline)
22
23
    # print results
24
    for i in result:
25
        print(i)

1
    pip3 install -U huggingface_hub[cli]

1
huggingface-cli login

1
    huggingface-cli download PatronusAI/Patronus-Lynx-8B-Instruct --local-dir Patronus_8B

1
    # (Recommended) Create a new conda environment.
2
    conda create -n myenv python=3.10 -y
3
    # Activate the newly created environment.
4
    conda activate myenv

1
    # Install vLLM with CUDA 12.1.
2
    pip install vllm

1
    python -m vllm.entrypoints.openai.api_server --port 5123 --model PatronusAI/Patronus-Lynx-8B-Instruct --tokenizer meta-llama/Meta-Llama-3-8B

1
    curl http://localhost:5123/v1/chat/completions \
2
      -H "Content-Type: application/json" \
3
      -d '{
4
      "model": "PatronusAI/Patronus-Lynx-70B-Instruct",
5
      "messages": [
6
        {"role": "user", "content": "What is a hallucination?"}
7
      ]
8
    }'

1
    # Prompt template
2
    Given the following QUESTION, DOCUMENT and ANSWER you must analyze the provided answer and determine whether it is faithful to the contents of the DOCUMENT.
3
4
    The ANSWER must not offer new information beyond the context provided in the DOCUMENT.
5
6
    The ANSWER also must not contradict information provided in the DOCUMENT.
7
8
    Output your final score by strictly following this format: "PASS" if the answer is faithful to the DOCUMENT and "FAIL" if the answer is not faithful to the DOCUMENT.
9
10
    Show your reasoning.
11
12
    --
13
    QUESTION (THIS DOES NOT COUNT AS BACKGROUND INFORMATION):
14
    {{ user_input }}
15
16
    --
17
    DOCUMENT:
18
    {{ provided_context }}
19
20
    --
21
    ANSWER:
22
    {{ bot_response }}
23
24
    --
25
26
    Your output should be in JSON FORMAT with the keys "REASONING" and "SCORE".
27
28
    Ensure that the JSON is valid and properly formatted.
29
30
    {"REASONING": ["<your reasoning as bullet points>"], "SCORE": "<final score>"}

1
    import requests
2
    import json
3
4
    # Define the endpoint URL
5
    url = "http://localhost:5123/v1/chat/completions"
6
7
    # Define the prompt template
8
    prompt_template = """
9
    Given the following QUESTION, DOCUMENT and ANSWER you must analyze the provided answer and determine whether it is faithful to the contents of the DOCUMENT.
10
11
    The ANSWER must not offer new information beyond the context provided in the DOCUMENT.
12
13
    The ANSWER also must not contradict information provided in the DOCUMENT.
14
15
    Output your final score by strictly following this format: "PASS" if the answer is faithful to the DOCUMENT and "FAIL" if the answer is not faithful to the DOCUMENT.
16
17
    Show your reasoning.
18
19
    --
20
    QUESTION (THIS DOES NOT COUNT AS BACKGROUND INFORMATION):
21
    {user_input}
22
23
    --
24
    DOCUMENT:
25
    {provided_context}
26
27
    --
28
    ANSWER:
29
    {bot_response}
30
31
    --
32
33
    Your output should be in JSON FORMAT with the keys "REASONING" and "SCORE".
34
35
    Ensure that the JSON is valid and properly formatted.
36
37
    {"REASONING": ["<your reasoning as bullet points>"], "SCORE": "<final score>"}
38
    """
39
40
    # Define the input data
41
    data = {
42
        "model": "PatronusAI/Patronus-Lynx-70B-Instruct",
43
        "messages": [
44
            {"role": "system", "content": "You are an AI that evaluates the faithfulness of responses."},
45
            {"role": "user", "content": prompt_template.format(
46
                user_input="What is the capital of France?",
47
                provided_context="France is a country in Europe. The capital city of France is Paris.",
48
                bot_response="The capital of France is Lyon."
49
            )}
50
        ]
51
    }
52
53
    # Send the request
54
    response = requests.post(url, headers={"Content-Type": "application/json"}, data=json.dumps(data))
55
56
    # Process the server response
57
    response_json = response.json()
58
59
    # Extract reasoning and score
60
    reasoning = response_json.get("choices", [{}])[0].get("message", {}).get("content", {})
61
    score = response_json.get("choices", [{}])[0].get("message", {}).get("content", {}).get("SCORE", "")
62
63
    # Print the results
64
    print("Reasoning:", reasoning)
65
    print("Score:", score)