Saturday 31 December 2016

scraping multiple pages

I'm having troubling looping over multiple sources to scrape, any suggestions on how I could make this work?var request = require('request'), cheerio = require('cheerio'), titles = []; msites = { site_1: 'site1link', site_2: 'site2link' }; for (msite in msites) {request(msites[msite], function(err, resp, body){ if(!err && resp.statusCode == 200) { var $ = cheerio.load(body); if(msites[msite] == 'site_1') { $('.pw-links-music ul li a div + div').each(function(){ var title = $(this).html(); titles.push(title); console.log(msites[msite]) console.log(titles); }); } else if (msites[msite] == 'site_2') { $('.penci-grid .list-posttt .grid-title a').each(function(){ var title = $(this).html(); titles.push(title); console.log(msites[msite]) console.log(titles); }); } } }); }

Submitted January 01, 2017 at 04:54AM by waveyrico

No comments:

Post a Comment