Scrape(General|Single|NLP): Compromise RiTa D3 - Snippet Editor

Save Snippet View Snippet

(() => {
  /* For this to work for some websites, you might need to disable a security feature in your browser. More info here: https://stackoverflow.com/questions/27323631/how-to-override-content-security-policy-while-including-script-in-browser-js-con
  */
  var options = {
  	scripts: [
      "https://unpkg.com/compromise",
      "https://rednoise.org/rita/download/rita.min.js",
      "https://d3js.org/d3.v5.min.js"
    ],
    cssSelector:"body",
    multiNodeTarget:false,
    stripTags:true,
    doc:document,
  }
  const stripTags = (el,tagList) => {
      tagList = !tagList ? "script,style,noscript,textarea,form,input,nav,aside" : tagList;
      [...el.querySelectorAll(tagList)].forEach(el => el.remove())
  }
  const getNodeText = (el,clean=true) => {
      var text = el.innerText ? el.innerText : el.value ? el.value : el.content ? el.content : "";
      return clean ? text.replace(/\.+/gim,".").replace(/\?+/gim,"?").replace(/!+/gim,"!") 
        .replace(/[\s\t]+/gim," ")
        .replace(/[\n\r]+/gim,"\n").trim()
      : text; //if clean is false, just return the raw text;
  }
  const getNodeAttributes = (el) => {
    try{
    return [...el.attributes]
    .reduce((atts,att) => {
         atts[att.nodeName] = att.value
         return atts
    },{})
    }catch(e){ //if there is an error with selection, then log the error and return an empty array
      console.log({e})
      return {};
    }
  }
  const select = ({doc,cssSelector,multiNodeTarget}) => {
    try{
      return multiNodeTarget ? [...doc.querySelectorAll(cssSelector)] : [doc.querySelector(cssSelector)];
    }catch(e){ //if there is an error with selection, then log the error and return an empty array
      console.log({e})
      return [];
    }
  }
  const processNodes = (nodes) => {
    var results = [];
    if(nodes.length > 0){
      for(var node of nodes){
        results.push({
		      nodeName:node.nodeName,
		      text:getNodeText(node), 
		      ...getNodeAttributes(node)
        });
      }
    }
    return results;
  }
var getWordCount = (tagReport,v) => {
  tagReport[v.word] ? tagReport[v.word]++ : tagReport[v.word] = 1;
  return tagReport;
}
  var getWordPOV = (word) => {
  var wordLow = word.toLowerCase();
  var pointsOfView = {
    first:["i","me","my","mine","we","us","our","ours","myself","ourselves"],
    second:["you","your","yours","yourself","yourselves"],
    third:["he","it","she","her","his","their","they","them","everyone","anybody","anyone","him", "hers", "himself", "herself", "itself", "they", "them", "theirs", "themselves"]
  }
  var povLabel = Object.keys(pointsOfView).find(key => pointsOfView[key].indexOf(wordLow) > -1);
  return povLabel ? povLabel : "third";
}

const run = (options) => {
   options.stripTags ? stripTags(document.body) : null;
   var targets = select(options);
   var results = processNodes(targets)
   console.log({results}) ;

var pov =  fpov && spov ? "first and second" : fpov && spov ? "first and second" :
      fpov ? "first" : spov ? "second" : "third"
      posReport.push({
        sentence:s.text,
        pov,
        tags,
        ...matchTests
      })
        }
      return posReport;
    },[]);

var report = {
      nestedSentencePOSReport:d3.nest()
      .key(d => d.pov)
      .key(d => d.tags.indexOf("QuestionWord") > -1 ? "Has Question Word" : "No Question Words")
      .key(d => d.sentence.length > 130 ? "Long Sentences" : d.sentence.length > 80 ? "Medium Sentences" : "Short Sentences")
      .key(d => d.tags.indexOf("Conjunction") > -1 ? "Has Conjunction" : "No Conjunctions")
      .key(d => d.tags.indexOf("Preposition") > -1 ? "Has Preposition" : "No Prepositions")
      .key(d => d.tags.indexOf("Noun") > -1 || d.tags.indexOf("Determiner") > -1 ? "Has Noun" : "No Nouns")
      .key(d => d.tags.indexOf("Verb") > -1 ? "Has Verb" : "No Verbs")
      .key(d => d.tags.some(tag => tag.indexOf("Tense") > -1 || tag.indexOf("Future") > -1) ? "Has Tense Verb" : "No Tense Verbs")
      .key(d => d.tags.indexOf("Adjective") > -1 ? "Has Adjective" : "No Adjectives")
            .key(d => d.tags.indexOf("Adverb") > -1 ? "Has Adverb" : "No Adverbs")
      .key(d => d.tags.filter(tag => tag == "Noun").length >= 3 ? "3+ Nouns" : "Less Than 3 Nouns").map(sentences.posReport),
      sentences,
      allPOSTags
    }
    console.log({report})
  }

})()