Orchestrating MongoDB & BigQuery for ML Excellence with PyMongoArrow and BigQuery Pandas Libraries

1
!pip install --upgrade pip
2
!pip install pymongoarrow
3
!pip install pandas-gbq

1
#Read data from MongoDB
2
import certifi
3
import pprint 
4
import pymongo
5
import pymongoarrow 
6
from pymongo import MongoClient
7
8
client = MongoClient("URI ``sting``",tlsCAFile=certifi.where())
9
10
#Initialize database and collection
11
db = client.get_database("sample_geospatial")
12
col = db.get_collection("shipwrecks")
13
14
for doc in col.find({}): 
15
  pprint.pprint(doc)
16
17
from pymongoarrow.monkey import patch_all 
18
patch_all()
19
20
#Create Dataframe for data read from MongoDB
21
import pandas as pd 
22
df = col.find_pandas_all({})

1
#Transform the schema for required format. 
2
#e.g. the object id is not supported in dataframes can be removed or converted to string. 
3
4
del(df["_id"])

1
#Write the transformed data to BigQuery.
2
3
import pandas_gbq
4
5
pandas_gbq.to_gbq(df[0:100], "gcp-project-name.bigquery-dataset-name.bigquery-table-name", project_id="gcp-project-name")

1
!pip install bigframes

1
import bigframes.pandas as bpd
2
bigframes.options.bigquery.project = "GCP project ID"
3
4
# df = <pandas_df>
5
bdf = bpd.read_pandas(df)