求助:第 5 课,根据你写的 async 示例,我写了一个小爬虫来针对 cnode,我怎么才能证明它确实是限制在 5 个请求上?
var cheerio = require('cheerio'); var superagent = require('superagent'); var url = require('url'); var async = require('async');
var cnodeUrl = 'https://cnodejs.org/'; var topicUrls = [];
superagent.get(cnodeUrl)
.end(function(err,res){
if(err){
return console.error(err);
}
// var topicUrls = [];
var $ = cheerio.load(res.text);
$('#topic_list .topic_title').each(function(idx,element){
var $element = $(element);
var href = url.resolve(cnodeUrl,$element.attr('href'));
topicUrls.push(href);
});
async.mapLimit(topicUrls,5,function(url,callback){
console.time("耗时");
superagent.get(url)
.end(function(err,res){
if(err){
return console.error(err);
}
console.log(url + "1");
var $ = cheerio.load(res.text);
var content = [];
// var element;
// var $element = $(element);
content.push({
title: $('.topic_full_title').text().trim(),
href: url,
comment: $('.reply_content').eq(0).text().trim(),
});
// console.log(idx + "1111");
callback(null,content);
});
console.timeEnd("耗时");
},function(err,result){ console.log(err); console.log(result); });
});
内容来源: alsotang/node-lessons