MongoDB NodeJS进程内存不足
我正尝试在MongoDB集合中使用NodeJS程序插入一百万条logging（虚拟）。但不幸的是，我的过程耗尽内存：
这是我在JavaScript中编写并通过节点运行的代码
var MongoClient = require('mongodb').MongoClient; MongoClient.connect('mongodb://localhost:27017/course', function(err, db) { if(err) throw err; db.collection('students').drop(); var types = ['exam', 'quiz', 'homework', 'homework']; // For 1 Million Records for (var i = 0; i < 1000000; i++) { // Each student taking 10 classes for (var class_counter = 0; class_counter < 10; class_counter ++) { scores = []; // Each Class has 4 grades // and each class has 4 grades for (var j = 0; j < 4; j++) { scores.push({'type':types[j],'score':Math.random()*100}); } // // there are 500 different classes that they can take class_id = Math.floor(Math.random()*501); // get a class id between 0 and 500 record = {'student_id':i, 'scores':scores, 'class_id':class_id}; db.collection('students').insert(record); } } }); 这是我得到的错误跟踪：
AMAC02PC0PHG3QP:25_Indexes_Insert macadmin$ node app.js <--- Last few GCs ---> 28373 ms: Scavenge 1397.8 (1457.4) -> 1397.8 (1457.4) MB, 1.1 / 0 ms (+ 151.3 ms in 1 steps since last GC) [allocation failure] [incremental marking delaying mark-sweep]. 29444 ms: Mark-sweep 1397.8 (1457.4) -> 1397.7 (1457.4) MB, 1071.5 / 0 ms (+ 427.1 ms in 14 steps since start of marking, biggest step 202.5 ms) [last resort gc]. 30486 ms: Mark-sweep 1397.7 (1457.4) -> 1397.6 (1457.4) MB, 1041.4 / 0 ms [last resort gc]. <--- JS stacktrace ---> ==== JS stack trace ========================================= Security context: 0x23473037399 <JS Object> 1: /* anonymous */(aka /* anonymous */) [/Users/macadmin/Desktop/NodeJS_MongoDB/25_Indexes_Insert/app.js:~3] [pc=0x3f5d2b92c716] (this=0x23473004131 <undefined>,err=0x23473004131 <undefined>,db=0x1f851bb90029 <JS Object>) 2: /* anonymous */(aka /* anonymous */) [/Users/macadmin/Desktop/NodeJS_MongoDB/25_Indexes_Insert/node_modules/mongodb/lib/mongo_client.js:455] [pc=0x3f5d2b4da8bd] (thi... FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed - process out of memory Abort trap: 6
我的笔记本configuration：
// Macbook Pro // OS X 10.9.5 // 2.5 Ghz Intel Core i7 // 16 GB Ram DDR3 // SSD
如果你的MongoDB服务器是2.6或者更新的话，最好利用一个批量写入命令批量API来批量执行批量插入操作，这些批量插入操作只是简单的抽象到服务器上，以便于构build批量操作。 这些批量操作主要有两种：
- 有序批量操作 。 这些操作按顺序执行所有操作，并在第一次写入错误时出错。
- 无序的批量操作 。 这些操作并行地执行所有操作并聚合所有的错误。 无序批量操作不保证执行顺序。
请注意，对于比2.6更旧的服务器，API将下载转换操作。 但是，无法将其下变换为100％，因此可能会出现一些无法正确报告正确数字的边缘情况。
在你的情况下，你可以像这样实现Bulk API ：
var MongoClient = require('mongodb').MongoClient; MongoClient.connect("mongodb://localhost:27017/course", function(err, db) { // Handle error if(err) throw err; // Get the collection and bulk api artefacts var col = db.collection('students'), types = ['exam', 'quiz', 'homework', 'homework'], bulk = col.initializeOrderedBulkOp(), // Initialize the Ordered Batch counter = 0; // Drop the collection col.drop(); // Representing a long loop with 1 Million Records for (var i = 0; i < 1000000; i++) { var scores = [], class_id = 0, record = {}; // Each student taking 10 classes for (var class_counter = 0; class_counter < 10; class_counter ++) { // Each Class has 4 grades // and each class has 4 grades for (var j = 0; j < 4; j++) { scores.push({ 'type': types[j], 'score': Math.random()*100 }); } // there are 500 different classes that they can take class_id = Math.floor(Math.random() * 501); // get a class id between 0 and 500 record['student_id'] = i; record['scores'] = scores; record['class_id'] = class_id; } bulk.insert(record); counter++; if (counter % 1000 == 0 ) { bulk.execute(function(err, result) { // re-initialise batch operation bulk = col.initializeOrderedBulkOp(); }); } } if (counter % 1000 != 0 ){ bulk.execute(function(err, result) { // do something with result db.close(); }); } });
– 更新 –
荣誉@MarkusWMahlberg，为了生成虚拟的内容，你可能想试试包mgenerate 。
我已经在环境中成功testing了以下代码：
- 处理器：Intel（R）Core（TM）i5-3470 CPU @ 3.20GHz 64位
- 内存：4,9 GB
- 操作系统：Ubuntu 16.04 LTS
- NodeJS：4.4.2
- MongoDB：3.2.1
var mongodb = require('mongodb'); main(); function main() { doWork(onDoWork); function onDoWork(error, result) { if (error) { console.error(error); } else if (result) { console.log(result); } } } function doWork(callback) { mongodb.MongoClient.connect('mongodb://localhost:27017/course', onConnect); function onConnect(error, db) { if (error) return setImmediate(callback, error); var IMIN = 0; var IMAX = 1000000 - 1; var i = IMIN; var JMIN = 0; var JMAX = 10 - 1; var j = JMIN; insertOne(db, i, onInsertOne); function onInsertOne(error, result) { if (error) { db.close(); setImmediate(callback, error); } else if (i > IMAX) { db.close(); setImmediate(callback); } else { i++; j++; if (j > JMAX) j = JMIN; insertOne(db, i, onInsertOne); } } } } function insertOne(db, studentId, callback) { var types = ['exam', 'quiz', 'homework', 'homework']; var KMIN = 0; var KMAX = types.length - 1; var scores = []; for (var k = KMIN; k <= KMAX; k++) { scores.push({ type: types[k], score: Math.round(Math.random() * 100) }); } var record = { student_id: studentId, scores: scores, class_id: Math.floor(Math.random() * 500) + 1 }; db.collection('students').insertOne(record, {}, callback); }