Portfolio Blog Snippets

Scrape(Aliexpress|Single): Get Products From SERP

var collection = [];
var ids = new Set();
(async () => {
var maxScroll = (selector,percent=1) => Math.round(document.querySelector(selector).scrollHeight*percent);
var scrollPage = (scrollHeight) => window.scrollTo(0,scrollHeight);
var prevScroll = 0;
var maxTime = 8000; 
var start = performance.now();
var loopEvery = (seconds) => new Promise((resolve) => {
  var stop = () => clearInterval(timer);
	var timer = setInterval(() => {
    var ms = Math.round(performance.now()-start);
    var newMax = maxScroll(".list-items",(ms/maxTime));
    var toScroll = newMax > prevScroll && maxTime >= ms;
		console.log({toScroll,newMax,ms,prevScroll})
		if(toScroll){
      prevScroll = newMax;
      scrollPage(newMax);   
    }else{
     stop();
     var data = [...document.querySelectorAll(".product-card")].reduce((data,el,i) => {
      !el.querySelector(".item-ad-tag") ? data.products.push(el) : data.ads.push(el);
      return data;
     },{products:[],ads:[]});
     var products = data.products.forEach((el) => {
        var obj = {};
        var img = el.querySelector(".item-img");
				var info = el.querySelector(".product-info");
        obj.title = info.querySelector("a.item-title").title;
        obj.id = info.querySelector("a.item-title").href.split(".html?")[0].split("/item/")[1];
        obj.url = info.querySelector("a.item-title").href.split("?")[0];
        obj.price = info.querySelector(".price-current").innerText.split("$")[1].split("-").reduce((price,v,i) => {
         v = +(v.trim());
         i == 0 ? price.low = v : price.high = v;
         return price;
        },{low:0,high:0});
				obj.price.high = obj.price.low > obj.price.high ? obj.price.low : obj.price.high;
        obj.price.avg = Math.round((obj.price.low + obj.price.high)/2*100)/100;
        var shipping = info.querySelector(".shipping-value") || false;
        obj.price.shipping = shipping ? shipping.innerText.indexOf("Free Ship") > -1 ? 0 : +(info.querySelector(".shipping-value").innerText.split("$")[1]) : "NA";
			  if(!isNaN(obj.price.shipping)){
          obj.price.total = {
             low: obj.price.shipping + obj.price.low,
             high: obj.price.shipping + obj.price.high
          };
					obj.price.total.avg = Math.round((obj.price.total.low + obj.price.total.high)/2*100)/100;
        }
        var rating = info.querySelector(".rating-value") || false;
        obj.rating = rating ? +rating.innerText.trim() : "NA";
  
        var store = info.querySelector(".store-name") || false;
        obj.store = {
           name: store ? store.innerText.trim() : "NA",
           url: store ? store.href.split("?")[0].trim() : "NA"
        }

var sold = info.querySelector(".sale-value-link") || false;
        obj.sold = sold ? +sold.innerText.replace("Sold","").trim() : "NA";
        var prev = ids.size;
        ids.add(obj.id);
        if(prev < ids.size){
            collection.push(obj);
        }
			
     })
		 console.log({collection})
    }
    resolve
  }, seconds*1000)
})

await loopEvery(1.25);

})()