findOneAndUpdatemongoose需要太长的'唯一'20kasynchronous更新

我需要从Google表格中读取一个csv文件并更新一个mongodb数据库。 该表具有20k到25k行(每个大约30列)。 如果我将表的读取限制在10K行以下,我设法做得很好,但是如果我超越它,开始行为怪异并需要很长时间才能最终运行所有asynchronousFindOneAndUpdates。 这里是我正在使用的代码(我应该做的时候,我创build了一个callback到console.log的计数器):

const mongooseDeals = require('mongoose'); const dotenv = require('dotenv'); dotenv.load(); const googleAuth = require('./gAuth'); mongooseDeals.connect(process.env.MONGODB_URI_DEALS, { useMongoClient: true, socketTimeoutMS: 0, reconnectTries: 30 }); const fs = require('fs'); const readline = require('readline'); const google = require('googleapis'); //Models const Deals = require('./api/models/deals'); googleAuth.authorize() .then(readDeals) .catch((err) => { console.log('auth error', err); }); function readDeals(auth) { var sheets = google.sheets('v4'); sheets.spreadsheets.values.get({ auth: auth, spreadsheetId: process.env.SPREADSHEET_ID, range: 'A2:AB', }, function(err, response) { if (err) { console.log('The API returned an error: ' + err); throw err; } var rows = response.values; console.log(rows.length + ' records loaded'); if (rows.length == 0) { console.log('No data found.'); throw err; } else { for (var i = 0; i < rows.length; i++) { copyValues(rows[i]); } } }); } var counter = 0; function copyValues(fila){ var row = fila; counter += 1; var speed = 0.0; var lat = 0.0; var lng = 0.0; var nl_title = ''; var template_id = ''; var deal_id = ''; var country_code = ''; var merchant_name = ''; var cat1 = ''; speed = parseFloat(row[10])* parseFloat(row[19]); lat = parseFloat(row[24])/10; lng = parseFloat(row[25])/10; nl_title = row[5]; template_id = parseInt(row[3]); deal_id = parseInt(row[4]); country_code = row[0]; merchant_name = row[6]; cat1 = row[16]; var aux ={ template_id: template_id, deal_id: deal_id, country_code: country_code, position:{ type: "Point", coordinates: [lng,lat] }, sellSpeed: speed, nl_title: nl_title, merchant_name: merchant_name, cat1: cat1 }; var query = { deal_id: deal_id }; var data = { $set: aux }; Deals.findOneAndUpdate(query, data, { upsert: true }, function(err, object) { if (err) throw err; counter -= 1; if (!counter){ console.log('termine!'); } }); } 

不知道,但它可能是相关的,但在任何情况下,这是模式

 const mongooseDeals = require('mongoose'); const GeoJSON = require('mongoose-geojson-schema'); const DealSchema = new mongooseDeals.Schema({ country_code: String, product_type: String, city: String, template_id: Number, deal_id: { type: Number, required: true, unique: true }, nl_title: String, merchant_name: String, picture: String, value: Number, final_price: Number, start_date: Date, due_date: Date, cat_id: Number, cat3: String, cat2: String, cat1: String, url: String, stock: Number, sold: Number, position: mongooseDeals.Schema.Types.Point, merchant_id: Number, country_id: Number, sellSpeed: Number, idioma: String },{ timestamps: true }); DealSchema.index({ deal_id: 1 }); DealSchema.index({ position: "2dsphere" }); DealSchema.index( { merchant_name: "text", nl_title: "text", cat1: "text" }, { weights: { merchant_name: 10, nl_title: 5, cat1: 1 }, default_language: "spanish", language_override: "idioma", name: "TextIndex" } ); const Deal = module.exports = mongooseDeals.model('Deal', DealSchema);