Portfolio Blog Snippets

Scrape(Amazon|Single): Product Review NLP

(() => {

var options = {
  	scripts: [
      "https://unpkg.com/compromise",
      "https://cdnjs.cloudflare.com/ajax/libs/rita/1.3.94/rita-full.min.js"
    ]
  }
  const run = (options) => {
var collection = [];
var lib = {
  getAttributes:(el) => {
     try{
     return [...el.attributes]
     .reduce((atts,att) => {
          atts[att.nodeName] = att.value
          return atts
     },{})
     }catch(e){
       return false;
     }
   },
  getText:(el,clean=true) => {
    var text = el.innerText ? el.innerText : el.value ? el.value : el.content ? el.content : el.nodeValue ? el.nodeValue : "";
       return clean ? text
                      .replace(/\.+/gim,".")
                      .replace(/\?+/gim,"?")
                      .replace(/!+/gim,"!")
                      .replace(/[ \t]+/gim," ")
                      .replace(/[\n\r]+/gim,"\n").trim()
       : text;
     },
  isValid: (text,obj) => Object.keys(obj).length || text.length,
 }
var skipElements = ["[rel*='sheet']","[property]","body","head","html","script","style","noscript"].reduce((s,d) => s+=`:not(${d})`, "");
for (let el of document.querySelectorAll("[class*='review-text']"+skipElements)){
   var obj = lib.getAttributes(el);
   var text = lib.getText(el);
    lib.isValid(obj,text) ? collection.push({
        ...obj,
        text,
        nodeName: el.nodeName.toLowerCase(),
        children: el.childElementCount,
        textLength:text.length
      }) : null;
}
var set = new Set();
collection = collection.reduce((items,d) => {
	var prevLen = set.size;
	set.add(d.text);
	set.size > prevLen ? items.push(d) : null;
	return items;
},[]);
var nlpReport = {
	nouns:[],
	adjectives:[],
	verbs:[],
	adverbs:[],
	clauses:[],
};
collection = collection.map(item => {
  let doc = nlp(item.text);
	item.sentences = doc.sentences().out('array').filter(s => s.length);
	item.filteredSentences = item.sentences.map(s => s
.match(/[A-Z0-9][^\.!\?]{2,200}\.?( [a-z]+?\.+?)*((\w.){1,})?[^\.!\?][a-z]*[.?!]/gm)).filter(s => s);
	item.nouns = [];
	item.adjectives = [];
	item.verbs = [];
	item.adverbs = [];
  item.clauses = [];
	item.report = item.filteredSentences.map(s => {
    var obj = {};
		let doc = nlp(s);
		
    obj.clauses = doc.clauses().out('array')
		
		let nouns = doc.nouns();
				obj.nouns = nouns.normalize().out('array').filter(s => s.length)
				obj.adjectives = nouns.adjectives().normalize().out('array').filter(s => s.length)

let verbs = doc.verbs();
				obj.verbs = verbs.normalize().out('array').filter(s => s.length)
				obj.adverbs = verbs.adverbs().normalize({verbs:true,parentheses:true}).out('array').filter(s => s.length)
				item.nouns = [...item.nouns, ...obj.nouns]
				item.adjectives = [...item.adjectives, ...obj.adjectives]
				item.verbs = [...item.verbs, ...obj.verbs]
				item.adverbs = [...item.adverbs, ...obj.adverbs]
				item.clauses = [...item.clauses, ...obj.clauses]
			
		obj.sentence = s;
		return obj;
	})

nlpReport.nouns = [...nlpReport.nouns, ...item.nouns]
				nlpReport.adjectives = [...nlpReport.adjectives, ...item.adjectives]
				nlpReport.verbs = [...nlpReport.verbs, ...item.verbs]
				nlpReport.adverbs = [...nlpReport.adverbs, ...item.adverbs]
				nlpReport.clauses = [...nlpReport.clauses, ...item.clauses]

delete item.text;
  return item;
})
				args = {
 ignoreCase: false,
 ignoreStopWords: true
};
var sortObj = (obj,columns = []) => {
		return Object.entries(obj).sort(function(a, b) {
			return b[1] - a[1];
		})
	}

nlpReport.wordCount = {
	nouns:sortObj(RiTa.concordance(nlpReport.nouns, null, args)),
	adjectives:sortObj(RiTa.concordance(nlpReport.adjectives, null, args)),
	verbs:sortObj(RiTa.concordance(nlpReport.verbs, null, args)),
	adverbs:sortObj(RiTa.concordance(nlpReport.adverbs, null, args))
}
		
		console.log({nlpReport,collection})

}

const loadScripts = (scripts) => {
    var scriptCountdown = scripts.length;
    var loadScript = (url) => {
    	var scriptsLoaded = () => {
      	console.log("scriptsLoaded",{scriptCountdown});
      	scriptCountdown == 0 ? run(options) : null;
      	return true;
    	}
      console.log("loadScript",{url})
      var imported = document.createElement('script');
          imported.src = url;
          imported.addEventListener("load", () => {
            scriptCountdown--;
            scriptsLoaded();
          });
          document.head.appendChild(imported);
    }
    scripts.forEach(loadScript)
  }
  loadScripts(options.scripts);

})()