Portfolio Blog Snippets

Scrape(General|Multi): Tiny Link Crawler + Delay & Node Reports

(async function(){
var links = new Set();
var docs = [];
var crawled = new Set();
var crawlLimit = 1000;
var delay = ms => new Promise(r => setTimeout(r, ms))

var settings = [
 { css:"title", props:["innerText"]},
 { css:"meta", props:["name","content"]},
 { css:"h1,h2,h3,h4,h5,h6", props:["innerText","nodeName"]},
 { css:"a",props:["id","className","href","innerText"]}
],
report = ({css,props}) => {
  const select = q => document.querySelectorAll(q),
  mapProps = (el) => props.reduce((o,p) => { o[p] = el[p] || ""; return o},{});
  return [...select(css)].map(mapProps);
},
processReport = opts => ({...opts,results:report(opts)});
function getLinks(doc){
 [...doc.querySelectorAll("a")].forEach(function(a){
  a.href ? links.add(a.href.split("#")[0]) : null;
 })
}
async function parseDoc(response){
 var htmlString = await response.text();
 var parser = new DOMParser();
 return parser.parseFromString(htmlString, "text/html");
}
await getLinks(document);
var linkIterator = links.values();
for await (var link of linkIterator){
      try{
      if(link.indexOf(window.location.origin) == 0 && link.indexOf("http") == 0){
    var ms = Math.round(Math.random()*3000);
    console.log({waiting:ms})
    await delay(ms);
		var response = await fetch(link,{redirect:"manual"});
		if (response.status == 200 && response.redirected == false) {
            if(crawled.has(link+"/") || crawled.has(link)){
            
            console.log("skipping...",)
             
             }else{

crawled.add(link);
			var doc = await parseDoc(response);
            var {type,url,status,ok,redirected} = response;
            docs.push({
             reports:settings.map(processReport),
             type,
             url,
             status,
             ok,
             redirected,
             title:doc.title
            });
			await getLinks(doc);
            }
		} 
		else {console.log("ERROR: Server returned:", response);} //end if/else response.status == 200
	  } //end if window.location.origin
      }catch(err){console.log({err})}
      crawled.size % 10 == 0 ? console.log({crawlLimit,crawled:crawled.size}) : null; //log progess every 10 links
      if(crawlLimit <= crawled.size){break}
   }
 console.log({links,docs})
})()