How to Index Fields for Vector Search

1
{
2
    "database": "sample_mflix",
3
    "collectionName": "embedded_movies",
4
    "type": "vectorSearch",
5
    "name": "vector_index",
6
    "fields": [ 
7
      {
8
        "type": "vector",
9
        "path": "plot_embedding",
10
        "numDimensions": 1536,
11
        "similarity": "dotProduct"
12
      }
13
    ]
14
}

1
{
2
    "database": "sample_mflix",
3
    "collectionName": "embedded_movies",
4
    "type": "vectorSearch",
5
    "name": "vector_index",
6
    "fields":[ 
7
      {
8
        "type": "vector",
9
        "path": "plot_embedding",
10
        "numDimensions": 1536,
11
        "similarity": "dotProduct"
12
      },
13
      {
14
        "type": "filter",
15
        "path": "genres"
16
      },
17
      {
18
        "type": "filter",
19
        "path": "year"
20
      }
21
    ]
22
}

1
{
2
  "fields": [{
3
    "type": "vector",
4
    "path": "plot_embedding",
5
    "numDimensions": 1536,
6
    "similarity": "dotProduct"
7
  }]
8
}

1
{
2
  "fields": [{
3
    "type": "vector",
4
    "path": "plot_embedding",
5
    "numDimensions": 1536,
6
    "similarity": "dotProduct"
7
  },
8
  {
9
    "type": "filter",
10
    "path": "genres"
11
  },
12
  {
13
    "type": "filter",
14
    "path": "year"
15
  }]
16
}

1
db.embedded_movies.createSearchIndex(
2
  "vector_index", 
3
  "vectorSearch", 
4
  {
5
    "fields": [
6
      {
7
        "type": "vector",
8
        "path": "plot_embedding",
9
        "numDimensions": 1536,
10
        "similarity": "dotProduct"
11
      }
12
    ]
13
  }
14
);

1
db.embedded_movies.createSearchIndex(
2
  "vector_index", 
3
  "vectorSearch", 
4
  {
5
    "fields": [
6
      {
7
        "type": "vector",
8
        "path": "plot_embedding",
9
        "numDimensions": 1536,
10
        "similarity": "dotProduct"
11
      },
12
      {
13
        "type": "filter",
14
        "path": "genres"
15
      },
16
      {
17
        "type": "filter",
18
        "path": "year"
19
      }
20
    ]
21
  }
22
);

1
package main
2
3
import (
4
	"context"
5
	"fmt"
6
	"log"
7
	"time"
8
9
	"go.mongodb.org/mongo-driver/bson"
10
	"go.mongodb.org/mongo-driver/mongo"
11
	"go.mongodb.org/mongo-driver/mongo/options"
12
)
13
14
func main() {
15
	ctx := context.Background()
16
17
	// Replace the placeholder with your Atlas connection string
18
	const uri = "<connectionString>"
19
20
	// Connect to your Atlas cluster
21
	clientOptions := options.Client().ApplyURI(uri)
22
	client, err := mongo.Connect(ctx, clientOptions)
23
	if err != nil {
24
		log.Fatalf("failed to connect to the server: %v", err)
25
	}
26
	defer func() { _ = client.Disconnect(ctx) }()
27
28
	// Set the namespace
29
	coll := client.Database("sample_mflix").Collection("embedded_movies")
30
31
	// Define the index details
32
	type vectorDefinitionField struct {
33
		Type          string `bson:"type"`
34
		Path          string `bson:"path"`
35
		NumDimensions int    `bson:"numDimensions"`
36
		Similarity    string `bson:"similarity"`
37
	}
38
39
	type vectorDefinition struct {
40
		Fields []vectorDefinitionField `bson:"fields"`
41
	}
42
43
	indexName := "vector_index"
44
	opts := options.SearchIndexes().SetName(indexName).SetType("vectorSearch")
45
46
	indexModel := mongo.SearchIndexModel{
47
		Definition: vectorDefinition{
48
			Fields: []vectorDefinitionField{{
49
				Type:          "vector",
50
				Path:          "plot_embedding",
51
				NumDimensions: 1536,
52
				Similarity:    "dotProduct"}},
53
		},
54
		Options: opts,
55
	}
56
57
	// Create the index
58
	searchIndexName, err := coll.SearchIndexes().CreateOne(ctx, indexModel)
59
	if err != nil {
60
		log.Fatalf("failed to create the search index: %v", err)
61
	}
62
	log.Println("New search index named " + searchIndexName + " is building.")
63
64
	// Await the creation of the index.
65
	log.Println("Polling to check if the index is ready. This may take up to a minute.")
66
	searchIndexes := coll.SearchIndexes()
67
	var doc bson.Raw
68
	for doc == nil {
69
		cursor, err := searchIndexes.List(ctx, options.SearchIndexes().SetName(searchIndexName))
70
		if err != nil {
71
			fmt.Errorf("failed to list search indexes: %w", err)
72
		}
73
74
		if !cursor.Next(ctx) {
75
			break
76
		}
77
78
		name := cursor.Current.Lookup("name").StringValue()
79
		queryable := cursor.Current.Lookup("queryable").Boolean()
80
		if name == searchIndexName && queryable {
81
			doc = cursor.Current
82
		} else {
83
			time.Sleep(5 * time.Second)
84
		}
85
	}
86
87
	log.Println(searchIndexName + " is ready for querying.")
88
}

1
package main
2
3
import (
4
	"context"
5
	"fmt"
6
	"log"
7
	"time"
8
9
	"go.mongodb.org/mongo-driver/bson"
10
	"go.mongodb.org/mongo-driver/mongo"
11
	"go.mongodb.org/mongo-driver/mongo/options"
12
)
13
14
func main() {
15
	ctx := context.Background()
16
17
	// Replace the placeholder with your Atlas connection string
18
	const uri = "<connectionString>"
19
20
	// Connect to your Atlas cluster
21
	clientOptions := options.Client().ApplyURI(uri)
22
	client, err := mongo.Connect(ctx, clientOptions)
23
	if err != nil {
24
		log.Fatalf("failed to connect to the server: %v", err)
25
	}
26
	defer func() { _ = client.Disconnect(ctx) }()
27
28
	// Set the namespace
29
	coll := client.Database("sample_mflix").Collection("embedded_movies")
30
	indexName := "vector_index"
31
	opts := options.SearchIndexes().SetName(indexName).SetType("vectorSearch")
32
33
	type vectorDefinitionField struct {
34
		Type          string `bson:"type"`
35
		Path          string `bson:"path"`
36
		NumDimensions int    `bson:"numDimensions"`
37
		Similarity    string `bson:"similarity"`
38
	}
39
40
	type filterField struct {
41
		Type string `bson:"type"`
42
		Path string `bson:"path"`
43
	}
44
45
	type indexDefinition struct {
46
		Fields []vectorDefinitionField `bson:"fields"`
47
	}
48
49
	vectorDefinition := vectorDefinitionField{
50
		Type:          "vector",
51
		Path:          "plot_embedding",
52
		NumDimensions: 1536,
53
		Similarity:    "dotProduct"}
54
	genreFilterDefinition := filterField{"filter", "genres"}
55
	yearFilterDefinition := filterField{"filter", "year"}
56
57
	indexModel := mongo.SearchIndexModel{
58
		Definition: bson.D{{"fields", [3]interface{}{
59
			vectorDefinition,
60
			genreFilterDefinition,
61
			yearFilterDefinition}}},
62
		Options: opts,
63
	}
64
65
	// Create the index
66
	searchIndexName, err := coll.SearchIndexes().CreateOne(ctx, indexModel)
67
	if err != nil {
68
		log.Fatalf("failed to create the search index: %v", err)
69
	}
70
	log.Println("New search index named " + searchIndexName + " is building.")
71
72
	// Await the creation of the index.
73
	log.Println("Polling to check if the index is ready. This may take up to a minute.")
74
	searchIndexes := coll.SearchIndexes()
75
	var doc bson.Raw
76
	for doc == nil {
77
		cursor, err := searchIndexes.List(ctx, options.SearchIndexes().SetName(searchIndexName))
78
		if err != nil {
79
			fmt.Errorf("failed to list search indexes: %w", err)
80
		}
81
82
		if !cursor.Next(ctx) {
83
			break
84
		}
85
86
		name := cursor.Current.Lookup("name").StringValue()
87
		queryable := cursor.Current.Lookup("queryable").Boolean()
88
		if name == searchIndexName && queryable {
89
			doc = cursor.Current
90
		} else {
91
			time.Sleep(5 * time.Second)
92
		}
93
	}
94
95
	log.Println(searchIndexName + " is ready for querying.")
96
}

1
const { MongoClient } = require("mongodb");
2
3
// connect to your Atlas deployment
4
const uri =  "<connectionString>";
5
6
const client = new MongoClient(uri);
7
8
async function run() {
9
   try {
10
     const database = client.db("sample_mflix");
11
     const collection = database.collection("embedded_movies");
12
    
13
     // define your Atlas Vector Search index
14
     const index = {
15
         name: "vector_index",
16
         type: "vectorSearch",
17
         definition: {
18
           "fields": [
19
             {
20
               "type": "vector",
21
               "numDimensions": 1536,
22
               "path": "plot_embedding",
23
               "similarity": "dotProduct"
24
             }
25
           ]
26
         }
27
     }
28
29
     // run the helper method
30
     const result = await collection.createSearchIndex(index);
31
     console.log(`New search index named ${result} is building.`);
32
33
     // wait for the index to be ready to query
34
     console.log("Polling to check if the index is ready. This may take up to a minute.")
35
     let isQueryable = false;
36
     while (!isQueryable) {
37
       const cursor = collection.listSearchIndexes();
38
       for await (const index of cursor) {
39
         if (index.name === result) {
40
           if (index.queryable) {
41
             console.log(`${result} is ready for querying.`);
42
             isQueryable = true;
43
           } else {
44
             await new Promise(resolve => setTimeout(resolve, 5000));
45
           }
46
         }
47
       }
48
     }
49
   } finally {
50
     await client.close();
51
   }
52
}
53
run().catch(console.dir);

1
const { MongoClient } = require("mongodb");
2
3
// connect to your Atlas deployment
4
const uri =  "<connectionString>";
5
6
const client = new MongoClient(uri);
7
8
async function run() {
9
   try {
10
     const database = client.db("sample_mflix");
11
     const collection = database.collection("embedded_movies");
12
    
13
     // define your Atlas Vector Search index
14
     const index = {
15
         name: "vector_index",
16
         type: "vectorSearch",
17
         definition: {
18
           "fields": [
19
             {
20
               "type": "vector",
21
               "numDimensions": 1536,
22
               "path": "plot_embedding",
23
               "similarity": "dotProduct"
24
             },
25
             {
26
               "type": "filter",
27
               "path": "genres"
28
             },
29
             {
30
               "type": "filter",
31
               "path": "year"
32
             }
33
           ]
34
         }
35
     }
36
37
     // run the helper method
38
     const result = await collection.createSearchIndex(index);
39
     console.log(`New search index named ${result} is building.`);
40
41
     // wait for the index to be ready to query
42
     console.log("Polling to check if the index is ready. This may take up to a minute.")
43
     let isQueryable = false;
44
     while (!isQueryable) {
45
       const cursor = collection.listSearchIndexes();
46
       for await (const index of cursor) {
47
         if (index.name === result) {
48
           if (index.queryable) {
49
             console.log(`${result} is ready for querying.`);
50
             isQueryable = true;
51
           } else {
52
             await new Promise(resolve => setTimeout(resolve, 5000));
53
           }
54
         }
55
       }
56
     }
57
   } finally {
58
     await client.close();
59
   }
60
}
61
run().catch(console.dir);

1
from pymongo.mongo_client import MongoClient
2
from pymongo.operations import SearchIndexModel
3
import time
4
5
# Connect to your Atlas deployment
6
uri = "<connectionString>"
7
client = MongoClient(uri)
8
9
# Access your database and collection
10
database = client["sample_mflix"]
11
collection = database["embedded_movies"]
12
13
# Create your index model, then create the search index
14
search_index_model = SearchIndexModel(
15
  definition={
16
    "fields": [
17
      {
18
        "type": "vector",
19
        "path": "plot_embedding",
20
        "numDimensions": 1536,
21
        "similarity": "dotProduct"
22
      }
23
    ]
24
  },
25
  name="vector_index",
26
  type="vectorSearch",
27
)
28
29
result = collection.create_search_index(model=search_index_model)
30
print("New search index named " + result + " is building.")
31
# Wait for initial sync to complete
32
print("Polling to check if the index is ready. This may take up to a minute.")
33
predicate=None
34
if predicate is None:
35
  predicate = lambda index: index.get("queryable") is True
36
37
while True:
38
  indices = list(collection.list_search_indexes(name))
39
  if len(indices) and predicate(indices[0]):
40
    break
41
  time.sleep(5)
42
print(result + " is ready for querying.")
43
44
client.close()

1
from pymongo.mongo_client import MongoClient
2
from pymongo.operations import SearchIndexModel
3
import time
4
5
# Connect to your Atlas deployment
6
uri = "<connectionString>"
7
client = MongoClient(uri)
8
9
# Access your database and collection
10
database = client["sample_mflix"]
11
collection = database["embedded_movies"]
12
13
# Create your index model, then create the search index
14
search_index_model = SearchIndexModel(
15
  definition={
16
    "fields": [
17
      {
18
        "type": "vector",
19
        "path": "plot_embedding",
20
        "numDimensions": 1536,
21
        "similarity": "dotProduct"
22
      },
23
      {
24
        "type": "filter",
25
        "path": "genres"
26
      },
27
      {
28
        "type": "filter",
29
        "path": "year"
30
      }
31
    ]
32
  },
33
  name="vector_index",
34
  type="vectorSearch",
35
)
36
37
result = collection.create_search_index(model=search_index_model)
38
print("New search index named " + result + " is building.")
39
# Wait for initial sync to complete
40
print("Polling to check if the index is ready. This may take up to a minute.")
41
predicate=None
42
if predicate is None:
43
  predicate = lambda index: index.get("queryable") is True
44
45
while True:
46
  indices = list(collection.list_search_indexes(result))
47
  if len(indices) and predicate(indices[0]):
48
    break
49
  time.sleep(5)
50
print(result + " is ready for querying.")
51
52
client.close()