每5分钟使用NodeJS将1000个数据中的10个数据添加/插入到MongoDB中
我目前正在尝试每5分钟插入10个新数据 ，而我目前在我的csv文件中有1000个数据 。
我怎么能这样做？ 帮助将不胜感激
以下是appendMongo.js代码
var fs= require('fs'); var csv = require('fast-csv'); var mongodb = require('mongodb'); var url = 'mongodb://localhost:27017/insertDB'; var MongoClient = mongodb.MongoClient; var data; MongoClient.connect(url, function (err, db) { if (err) { console.log('Unable to connect to the mongoDB server. Error:', err); } else { console.log('Connection established to', url); var collection = db.collection('airports'); readData=fs.createReadStream('1000data.csv').pipe(csv()) .on('data',function(data){ collection.insert({'data': data}); }) .on('end',function(data){ console.log('Read finished'); }) } }); //End of Mongo connect
一切为csv文件是相似的，因为我只是复制和粘贴文件，因为我需要做1000个数据，我被要求这样做。
这是1000data.csv的1000个数据中的部分数据
Machine Unit,Air Temperature °C,Water Temperature °C,Heat Temperature °C,Room Temperature °C,Date,Time 1,61,54,87,20,12/3/2016,8:39AM 2,41,57,92,21,29/9/2016,3:51PM 3,39,53,89,22,22/12/2016,5:30PM 4,56,27,87,23,9/7/2016,6:54AM 5,71,85,76,24,21/10/2016,7:29PM 6,82,64,85,25,22/3/2016,9:38PM 7,91,73,94,26,21/10/2016,1:30PM 8,35,79,89,27,30/4/2016,2:17PM 9,29,59,83,28,18/7/2016,7:49AM 10,19,61,80,29,27/9/2016,8:21PM
只要不断地向数组添加数据，并有一个单独的函数来执行每隔5分钟。 这个函数可以处理mongodb连接工作。
下面的准星
'use strict'; var fs= require('fs'); var csv = require('fast-csv'); var dataArray = []; var finishedReading = false; //set interval limit as you like var interval = 1000 * 10; //10secs fs.createReadStream('data.csv').pipe(csv()) .on('data',function(data){ console.log(data); dataArray.push(data) }) .on('end',function(data){ console.log('Read finished'); finishedReading = true; }) var intervalFn = setInterval(function(){ //you can have check for 10 since you want that many items if(dataArray.length >= 2){ var twoItems = dataArray.splice(0,2); console.log(new Date() + twoItems); //execute mongodb code from here }else if(finishedReading){ //clean up if items left in array are less than 10 and also clear this interval function console.log(new Date() + dataArray.length); clearInterval(intervalFn); } },interval)
这里是间隔函数的输出，表示数组每10秒后处理一次
Wed Dec 28 2016 14:46:09 GMT+0530 (India Standard Time)Machine Unit,Air Temperature C,Water Temperature C,Heat Temperature C,Room Temperature C,Date,Time,1,61,54,87,20,12/3/2016,8:39AM Wed Dec 28 2016 14:46:19 GMT+0530 (India Standard Time)2,41,57,92,21,29/9/2016,3:51PM,3,39,53,89,22,22/12/2016,5:30PM Wed Dec 28 2016 14:46:29 GMT+0530 (India Standard Time)4,56,27,87,23,9/7/2016,6:54AM,5,71,85,76,24,21/10/2016,7:29PM Wed Dec 28 2016 14:46:39 GMT+0530 (India Standard Time)6,82,64,85,25,22/3/2016,9:38PM,7,91,73,94,26,21/10/2016,1:30PM Wed Dec 28 2016 14:46:49 GMT+0530 (India Standard Time)8,35,79,89,27,30/4/2016,2:17PM,9,29,59,83,28,18/7/2016,7:49AM
Sikorski的代码是相当不错的，可以工作，但看起来像是刚接触node的代码，所以我将把代码分解成不同的组件，这不仅容易理解，而且也是一个很好的开发实践。
用于连接
function connectToMongo() { mongoose.connect('mongo_url'); }
断开连接
function disconnectToMongo() { mongoose.connection.close() }
用于保存数据
//Db is name of your mongo collection Db.create(data, function(err, data) { if (err) { console.log(err); return } });
阅读整个csv
function readDoc() { var doc_content = []; fs.createReadStream('data.csv').pipe(csv()) .on('data',function(data){ doc_content.push(data) }) .on('end',function(data){ console.log('Read finished'); return doc_content; }) }
现在让我们在一个函数中使用这些函数，让我们说init将运行
function init() { var numOfDocsToSave = 10; //just chnage here for num of docs you want to save in interval var interval = 1000 * 10; //10 secs, thanks Sikorski for this var data = readDoc(); var chunk = []; var intervalId = setInterval(function() { connectToMongo(); if(data.length >= numOfDocsToSave) { chunk = data.splice(0, numOfDocsToSave); saveInMongo(chunk); } else { saveInMongo(data); clearInterval(intervalId); } disconnectoToMongo(); //we should not open connection for 5 minutes when we're not doing any operation }, interval); console.log("Interval ended"); }
希望能帮助到你。
更新
正如你所说的Db未定义，可能是因为你还没有创build它。
所以你可以做到这一点，
〜model.js
var mongoose = require('mongoose'); mongoose.connect('mongodb://localhost/testDb'); module.exports = mongoose.model('Db', { name: String //change argument with your schema });
现在将其导入到主文件中
var Db = require('./model'); //assuming your model.js and main file are in same directory
看看下面的代码是否可以帮助你。
var fs= require('fs'); var csv = require('fast-csv'); var mongodb = require('mongodb'); var url = 'mongodb://localhost:27017/insertDB'; var MongoClient = mongodb.MongoClient; var data; MongoClient.connect(url, function (err, db) { if (err) { console.log('Unable to connect to the mongoDB server. Error:', err); } else { console.log('Connection established to', url); var collection = db.collection('airports'); readData=fs.createReadStream('1000data.csv').pipe(csv()) .on('data',function(data) { var count = 0; // Count initialised to 0 var timeout = setInterval(function() // Insert 10 record after 5 minutes { if(count >= 999) // If count is 1000 stop inserting. { clearInterval(timeout); } else { for(var i = count; i < count+10; i++) { collection.insert({'data': data[i]}, function(err) { console.log(err); }); //Insert one record } count += 10; //Increase the count } },300000); }) .on('end',function(data){ console.log('Read finished'); }) } }); //End of Mongo connect