Parallelizing mongoexport to dump large amounts of data from multiple collections

@Tudor_Palade , please modify the snippet to meet your needs . This seems working fine . Try not to execute all promise all once , it might bring up connection issue or whatsoever .

const { MongoClient } = require('mongodb');
// your connection uri
const uri = 'XXX;
const client = new MongoClient(uri);
const dbName = 'sample_vector_search';
const { exec } = require("child_process");
const { resolve } = require('path');

async function main() {
    await client.connect();
    console.log('Connected successfully to server');
    const db = client.db(dbName);
    const collectionName = 'restaurant_reviews';
    const restaurant_reviews = db.collection(collectionName);
    const count = await restaurant_reviews.find({}).count();
    const tasks = [];
    const limit = 1000;
    for (let index = 0; index < count; index =  index + limit) {
        tasks.push(spawnExport(uri, dbName, 'restaurant_reviews', `${dbName}_${collectionName}_batch${index}`, index + limit, limit))
    }

    // Use any library of your choice to better handle the tasks
    // process tasks array in chunk , ideal is chunk = number of core 
    Promise.allSettled(tasks).then(result => {
        console.log('Done')
    }).catch(error => {
        console.log('Error')
    })

    return 'done.';
}


function spawnExport(uri, db, col, filename, skip, limit) {

    return new Promise((resolve, reject) => {
        const command = `mongoexport --uri ${uri} -d ${db} -c ${col} -o ${filename}.json --skip=${skip} --limit=${limit}`;
        console.log(command)
        exec(command, (error, stdout, stderr) => {
            if (error) {
                console.log(`error: ${error.message}`);
                return reject(error.message);
            }
            if (stderr) {
                console.log(`stderr: ${stderr}`);
                return reject(stderr.message);
            }
            return resolve();
        });
    });



}

main()
    .then(console.log)
    .catch(console.error)
    .finally(() => client.close());