Scrape(coj|Multi): Crawl Property SERPs

var collection = [];

(() => {
  var options = {
    scripts: ["https://d3js.org/d3.v6.min.js"]
  }
  const downloadCSV = (data, fileName) => {
		var d = document;
    var a = d.createElement('a');
    a.href = window.URL.createObjectURL(new Blob([d3.csvFormat(data)],{type:'text/csv;charset=utf-8;'}));
    a.setAttribute('download', `${fileName}.csv`);
    d.body.appendChild(a);
    a.click();
    d.body.removeChild(a);
  };
	var run = () => {
  //Scrape sales history and property meta from browser results page
// https://paopropertysearch.coj.net/Sales/Results.aspx

(async () => {

var salesHistoryCollection = [];
	var page = 1;
const download = (data, fileName) => {
  var jsonDUMP = JSON.stringify(data);
  var exportFilename = fileName + ".json";
  var csvData = new Blob([jsonDUMP], {
    type: 'text/csv;charset=utf-8;'
  });
  var link = document.createElement('a');
  link.href = window.URL.createObjectURL(csvData);
  link.setAttribute('download', exportFilename);
  document.body.appendChild(link);
  link.click();
  document.body.removeChild(link);
};

buildingInfo = buildingInfo ? buildingInfo.slice(1).map(row => ({[row.innerText.match(/[a-z]+/i)]:+row.innerText.match(/\d/)})).reduce((o,d) => ({...o,...d}),{}) : null;

var values = [...doc.querySelectorAll("[id*='BuildingAttributes'] .col_code")].map(el => isNaN(el.innerText) ? el.innerText : +el.innerText);

var elements = [...doc.querySelectorAll("[id*='BuildingAttributes'] .col_element")].reduce((elements,el,i) => {
		var key = el.innerText.replace(/[ \/]+/gim,"").trim()
		elements[key] = values[i];
		return elements;
},{})

var values = [...doc.querySelectorAll("[id*='BuildingElements'] .col_detail")].map(el => isNaN(el.innerText) ? el.innerText : +el.innerText);

elements = {...elements, ...[...doc.querySelectorAll("[id*='BuildingElements'] .col_element")].reduce((elements,el,i) => {
		var key = el.innerText.replace(/[ \/]+/gim,"").trim()
		elements[key] = values[i].split(/\d+ /gim)[1].toLowerCase();
		return elements;
},{})};

var buildingArea = [...doc.querySelectorAll("[id*='gridBuildingArea'] tr:last-of-type td")].map(el => el.innerText).slice(1);
			buildingArea = {
				grossArea:+buildingArea[0] || "",
				heatedArea:+buildingArea[1] || "",
				EffectiveArea:+buildingArea[2] || ""
			}

var toNum = (s) => s.indexOf("$") > -1 || s.match(/^\d+$/) ? +(s.replace(/[\$,]+/gim,"")) : s;
var extract = el => ({
	nodeName:el.nodeName,
  id:el.id.replace("ctl00_cphBody_lbl","").replace("NoData","").split("_").slice(-1)[0].replace("lbl",""),
  text:el.innerText ? toNum(el.innerText.replace(/[ \n\r\t]+/gim," ").trim()) : ""
	});

var props = [...doc.querySelectorAll("span[id]")].map(extract).filter(el => el.text).slice(2).filter(d => !isNaN(d.text) || d.text.length < 80).reduce((report,el) => ({...report,[el.id]:el.text}),{})

Object.entries(props).filter(prop => prop[0].indexOf("Header") == 0 || prop[1] == "See below").forEach(prop => delete props[prop[0]])
delete props.SaleNote2;
delete props.SaleNote3;
delete props.LegalDescription1;
delete props.LegalDescription2;
delete props.ValueMethodCertified;
delete props.ValueMethodInProgress;
delete props.Version;

record = {...props ,...record , ...buildingArea ,...props , ...elements}

delete record["Cap Diff/Portability Amt_2019_certfied"];
		delete record["Cap Diff/Portability Amt_2020_inProgress"];
 		record.salesHistory = getSalesHistory(doc);
		collection.push(record)
		
		}catch(e){
			console.log(e)
		}	
		if(collection.length == 2000){
			await download(collection,"salesHistory")
			page++;
			collection = [];
		 }
		
	}
		if(collection.length){
			await download(collection,"salesHistory")
			collection = [];
		 }

})()
  }
  const loadScripts = ({scripts}) => {
    console.log(`Loading External Scripts`);
    var scriptCountdown = scripts.length;
    var loadScript = (url) => {
      var scriptsLoaded = () => scriptCountdown == 0 ? run() : null;
      var imported = document.createElement('script');
      imported.src = url;
      imported.addEventListener("load", () => {
        scriptCountdown--;
        scriptsLoaded();
      });
      document.head.appendChild(imported);
    }
    scripts.forEach(loadScript)
  }
  loadScripts(options);
})();