Scrape(coj): Crawl Property SERPs - Snippet Editor

var collection = [];
var page = 1;
(async () => {

var toNum = (s) => s.indexOf("$") > -1 || s.match(/^\d+$/) ? +(s.replace(/[\$,]+/gim,"")) : s;
var extract = el => ({
	nodeName:el.nodeName,
  id:el.id.replace("ctl00_cphBody_lbl","").replace("NoData","").split("_").slice(-1)[0].replace("lbl",""),
  text:el.innerText ? toNum(el.innerText.replace(/[ \n\r\t]+/gim," ").trim()) : ""
	});

var getProps = (doc) => {
var props = [...doc.querySelectorAll("span[id]")].map(extract).filter(el => el.text).slice(2).filter(d => !isNaN(d.text) || d.text.length < 80).reduce((report,el) => ({...report,[el.id]:el.text}),{});

var areaRows = [...doc.querySelectorAll("[id*='gridBuildingArea'] tr")];
var areaFields = [...areaRows[0].querySelectorAll("th")].slice(1).map(th => th.innerText);
var areaTotals = [...areaRows.slice(-1)[0].querySelectorAll("td")].slice(1).map(td => +td.innerText);

Object.entries(props).filter(prop => prop[0].indexOf("Header") == 0 || prop[1] == "See below").forEach(prop => delete props[prop[0]])
delete props.SaleNote2;
delete props.SaleNote3;
delete props.LegalDescription1;
delete props.LegalDescription2;
delete props.ValueMethodCertified;
delete props.ValueMethodInProgress;
delete props.Version;

var attributes = [...doc.querySelectorAll("[id*='gridBuildingAttributes'] tr")].slice(1).map(row => [...row.querySelectorAll("th,td")].slice(0,2).map(d => d.innerText)).reduce((o,arr) => {
o[arr[0].replace(/ /gim,"")] = +arr[1];
return o;
},{});

return {
    ...props,
    ...attributes,
  	[areaFields[0]]:areaTotals[0],
		[areaFields[1]]:areaTotals[1],
		[areaFields[2]]:areaTotals[2]
}
}

const download = (data, fileName) => {
  var jsonDUMP = JSON.stringify(data);
  var exportFilename = fileName + ".json";
  var csvData = new Blob([jsonDUMP], {
    type: 'text/csv;charset=utf-8;'
  });
  var link = document.createElement('a');
  link.href = window.URL.createObjectURL(csvData);
  link.setAttribute('download', exportFilename);
  document.body.appendChild(link);
  link.click();
  document.body.removeChild(link);
};

let res = await fetch(a.href);
			let text = await res.text();
			let parser = await new DOMParser();
			let doc = await parser.parseFromString(text, "text/html");

record.props = getProps(doc);
    	
    record.salesHistory = getSalesHistory(doc);
		collection.push(record)
		}catch(e){
			console.log(e)
		}	
		if(collection.length == 2000){
			await download(collection,"data")
			page++;
			collection = [];
		 }
		
	}
		if(collection.length){
			await download(collection,"data_final")
			collection = [];
		 }

})()