Portfolio Blog Snippets

Scrape(Walmart): Crawl Product SERPs

var items = [];
var page = 1;
var getData = async (url) => {
var res = await fetch(url, {
    "credentials": "include",
    "headers": {
        "Content-Type": "application/json"
    },
    "referrer": "https://www.walmart.com/search/?page=2&ps=40&query=3d+printer",
    "method": "GET",
    "mode": "cors"
});
var data = await res.json();
items = await [...items,...data.items.map(d => {
["fulfillment","imageProps","inventory","isHeartable","is_limited_qty","itemClassId","seeAllFacet", "enableAddToCart", "canAddToCart", "blitzItem", "cta", "isBluRay", "isDvd", "isVuduDigital", "preOrderAvailable", "shippingPassEligible" ].forEach(key => delete d[key]);
return d;
})]

console.log({items})
	if(data.pagination.next.url){
		getData("https://www.walmart.com/search/api/preso?prg=desktop&page="+(page++)+"&ps=40&query=3d%20printer")
	}else{
		console.log("DONE");
		console.log({items});
	}
   
}

(async () => {
	var data = await getData("https://www.walmart.com/search/api/preso?prg=desktop&page=1&ps=40&query=3d%20printer");
	
})()