Portfolio Blog Snippets

Scrape(General|Multi|Node): Grouped Node Crawler

(async () => {
  //SETTINGS
  var collection = [];
  var CRAWLERSETTINGS = {
    LIMIT: 10,
    INDEX: 0
  }
  var fetchSettings = {
    url: document.URL,
    options: {
      method: 'GET', // *GET, POST, PUT, DELETE, etc.
      //cache: 'no-cache', // *default, no-cache, reload, force-cache, only-if-cached
      credentials: 'same-origin', // include, *same-origin, omit
      redirect: 'follow', // manual, *follow, error
    }
  };
  var links = new Set([fetchSettings.url]);
  //END SETTINGS

//Node EXTRACTION FUNCTIONS
  var extract = (el,t="text",arr) => {
  //console.log(`extracting data from el`)
    var getNodeData = (el) => {
      var text = getNodeText(el);
      var attributes = getNodeAttributes(el);
      return {text,...attributes,tag: el.nodeName}
    }

if(t == "text") {

var nodeObj = getNodeData(el);
            //  console.log({arr,nodeObj})
          Object.keys(nodeObj).length > 0 ? arr.push(nodeObj) : null;
    }

return arr;
  }
  const getNodeText = (el,clean=true) => {
     //select the node text, starting with innerText, then trying other possible alternatives (depends on the selected html element)
     var text = el.innerText ? el.innerText : el.value ? el.value : el.content ? el.content : "";
     return clean ? text.replace(/\.+/gim,".").replace(/\?+/gim,"?").replace(/!+/gim,"!") //clean up multiple puncuation marks
       .replace(/[ \t]+/gim," ") //remove extra white space, replacing tabs with space
       .replace(/[\n\r]+/gim,"\n").replace(/( ?\n ?)+/gim,"\n").trim()  //remove extra line breaks, replacing returns with new lines
     : text; //if clean is false, just return the raw text;
 }
  const getNodeAttributes = (el) => {
    try {
      return el.attributes ? [...el.attributes].reduce((atts, att) => {
          atts[att.nodeName] = att.value
          return atts
        }, {}) : {};
    } catch (e) { //if there is an error with selection, then log the error and return an empty array
      console.log({
        e
      })
      return {};
    }
  }
  //END EXTRACTION FUNCTIONS
  //GET DOCUMENT
  var getDoc = async (settings) => {
    const isValid = (response) => response.ok; //simple check to see if response is good
    var response = await fetch(settings.url, settings.options); //fetch the doc, await response object
    return isValid(response) ? await response.text() : "Not Valid"; //return the document text if the reponse is valid
  }
  //PARSE DOCUMENT
  var parseDoc = (docText) => { //input = html string output = virtual dom
    var parser = new DOMParser(); //new parser instance
    return parser.parseFromString(docText, "text/html") //return parsed doc
  }

var generateReport = ({link,doc}) => {
    var domain = doc.domain;
    var cleanText = (str) => str.replace(/[\s\t\n\r]+/gim, " ").trim(); //utility function to clean whitespace
    var report = {};
    try {
      var getElData = (selector,type="text") => {
        return [...doc.querySelectorAll(selector)].reduce((arr,el) => extract(el,type,arr), []);
      }
      report = {
        title: doc.querySelector("title").innerText,
        url: link,
        meta: getElData("meta"),
        resources: [...doc.scripts, ...doc.styleSheets].reduce((arr,el) => extract(el,"text",arr), []),
        links: [...doc.links].reduce((links, a) => {
          var linkObject = {
            text: cleanText(a.innerText),
            href: a.href
          };
          linkObject.href.indexOf(domain) > -1 ? links.internal.push(linkObject) : links.external.push(linkObject);
          return links;
        },{
          internal: [],
          external: []
        }),
        headingText: getElData("h1,h2,h3,h4,h5,h6"),
        paragraphs: getElData("p"),
        semanticTags: getElData("article,details,section,summary,figcaption,figure,data,dialog,main"),
        lists: getElData("ul,ol,li,dir,dl,dt,dd"),
        io: getElData("form,input,textarea,button,select,optgroup,option,label,fieldset,legend,datalist,output"),
        formatted: getElData("acronym,abbr,b,bdi,bdo,big,center,code,del,dfn,em,font,i,ins,kbd,mark,meter,pre,progress,rp,rt,ruby,s,samp,small,strike,strong,sub,sup,template,tt,u,var,wbr"),
        references: getElData("address,blockquote,cite,q,time"),
        images: getElData("img,map,area,canvas,figcaption,figure,picture,svg"),
        video: getElData("video"),
        audio: getElData("audio"),
        headerFooterAsideNav: getElData("header,footer,nav,aside"),
        navigation: getElData("a,nav"),
        divs: getElData("div")
      }
    } catch (e) {
      console.log(e);
    }
    return report;
  }
  //loop through links. By default, the crawler starts on the current page.
  for await (let link of links) {
    CRAWLERSETTINGS.INDEX++; //track the link index
    fetchSettings.url = link; //replace the old url with a new one.
    try {
      //STEP 1 - get document via fetch
      var docText = await getDoc(fetchSettings);
      //STEP 2 - parse document via DOM Parser API
      var doc = await parseDoc(docText);
      //Step 3 - extract data from doc
      var report = await generateReport({link,doc});
      //STEP 4 - add doc data to bulk collection
      await collection.push(report);
      if (CRAWLERSETTINGS.INDEX < CRAWLERSETTINGS.LIMIT) { //STEP 5 - check if limits have been reached
        await report.links.internal.forEach(link => { //loop through each of the current doc's links so they can be added to the crawl list
          links.size < CRAWLERSETTINGS.LIMIT ? // ...until limit is reached or search exhausted
            links.add(link.href) : null; //keep finding & adding links to the crawl list...
        });
        await console.log({ //log progress to the console
          ...CRAWLERSETTINGS, //limit & index
          reports: collection.length, //# of page-level reports
          links: links.size //# of unique links found
        });
      } else {
        //Check if crawler limits have been reached.
        var limitReached = CRAWLERSETTINGS.INDEX >= CRAWLERSETTINGS.LIMIT;
        var linkSizeReached = links.size <= CRAWLERSETTINGS.INDEX;
        if (limitReached || linkSizeReached) {
          console.log({limitReached,linkSizeReached,collection})
        }
      }
    } catch (e) {
      await console.log({
        failedToGetPage: e
      });
    }
  } // END FOR AWAIT LOOP - link of links
})()