web scraping - CasperJS unable to scrape AJAX webpage -
i trying scrape http://www.snapdeal.com/offers/deal-of-the-day
loads json data using ajax call below page:
json_url = http://www.snapdeal.com/json/getproductbyid?*
the code block using below, log message waiting ajax request:
, not waiting ajax request:
, instead waitforresource
times out
casper.options.onresourcerequested = function (casper, requestdata){ // loop through our ajax urls // create list of ajax urls track var ajaxurls = [json_url]; ajaxurls.every(function(ajaxurl){ // request match ajax url if(requestdata.url.indexof(ajaxurl) !== -1){ // matches, we'll wait return (with 10s timeout) //console.log("waiting ajax request: " + requestdata.url); // print_object(requestdata); casper.waitforresource(requestdata.url, function(){ console.log("ajax request returned: " + requestdata.url); }, function(){ console.log("ajax request didn't return after wait period: " + requestdata.url) }, 10000); } }); }
to further debug, logged events , resource @ url json_url
received, not sure why waitforresource
times out.
casper.on('resource.received', function(resource) { if (resource.url.indexof('http://www.snapdeal.com/json/getproductbyid') != -1){ casper.echo('resource.received: ' + resource.url); } });
log after run:
waiting ajax request: http://www.snapdeal.com/json/getproductbyid?pogids=671556289429,649272180,60998,685755068805,677649317861,1239888775,661402031482,636966047361,1775140628,1822452791,439536 ,range,2042952975,1472100667,899358889,643129681532,668235859588,&lang=en resource.received: http://www.snapdeal.com/json/getproductbyid?pogids=671556289429,649272180,60998,685755068805,677649317861,1239888775,661402031482,636966047361,1775140628,1822452791,439536,range, 2042952975,1472100667,899358889,643129681532,668235859588,&lang=en resource.received: http://www.snapdeal.com/json/getproductbyid?pogids=671556289429,649272180,60998,685755068805,677649317861,1239888775,661402031482,636966047361,1775140628,1822452791,439536,range, 2042952975,1472100667,899358889,643129681532,668235859588,&lang=en ajax request didn't return after wait period: http://www.snapdeal.com/json/getproductbyid?pogids=671556289429,649272180,60998,685755068805,677649317861,1239888775,661402031482,636966047361,17751406 28,1822452791,439536,range,2042952975,1472100667,899358889,643129681532,668235859588,&lang=en got page no offeres found thestartin: ~/appli
what need is:
casper.waitforselector()
or
casper.waitfor(function() { // test here return true when page loaded })
Comments
Post a Comment