Scrape(General|Single|Node): Node Recon (CSV) - Snippet Editor

Save Snippet View Snippet

(() => {
  var options = {
    scripts: ["https://d3js.org/d3.v6.min.js"],
		skip:["iframe","br","hr","clippath","path","[rel*='sheet']","body","header","footer","head", "html","script","style","noscript","svg","picture", "figcaption","table", "tbody"]
		.reduce((s, d) => s += `:not(${d})`, "")
  }
  const downloadCSV = (data, fileName) => {
		var d = document;
    var a = d.createElement('a');
    a.href = URL.createObjectURL(new Blob([d3.csvFormat(data)],{type:'text/csv;charset=utf-8;'}));
    a.setAttribute('download', `${fileName}.csv`);
    d.body.appendChild(a);
    a.click();
    d.body.removeChild(a);
  };
	var run = () => {
			const getText = el => ((el.innerText || el.value || el.content || el.nodeValue || "")
											 .replace(/\.+/gim,".").replace(/\?+/gim,"?").replace(/!+/gim,"!")
											 .replace(/(\")+/gim," ")
		                	 .replace(/(["'])|\[edit\]/gim, "").replace(/\[[\W\d]+\]/gim," ").replace(/^[\W]+/gim, "")
											 .replace(/[ \t]+/gim," ")
											 .replace(/[\n\r]+/gim,"\n")
											 .trim())
			const getElAttributes = el => [...el.attributes].reduce((atts,att) => {
							if(!["aria-label","videopayload"].some(stop => att.nodeName.toLowerCase().indexOf(stop) > -1))
		          	atts[att.nodeName] = att.value;
		          return atts;
		  },{});
			const getAttributes = (report,el,i) => {
				try{
					var attributes = getElAttributes(el);
							attributes.text = getText(el);
							attributes.length = attributes.text.length || false;
					if(attributes.length && attributes.length < 600){
						var nodeName = el.nodeName.toLowerCase();
						
		
						var getCategory = nodeName => Object.entries({
								headingText:"h1,h2,h3,h4,h5,h6",
								formattedText:"b,strong,em,i,abbr,blockquote,cite,dfn,sub,sup,small,q,bdi",
								form:"textarea,input,form,button,select,optgroup,option,label,fieldset,legend,datalist,output",
								image:"img,svg,canvas,area,map,figure,picture,figcaption,clippath,path,defs",
								table:"table,tr,td,th,tbody",
								time:"time",
								address:"address",
								title:"title",
								list:"ul,ol,li"
							}).find(entry => entry[1].indexOf(nodeName) > -1 || "");
						var childElements = el.childElementCount || 0;
			      var childNodes = el.childNodes ? el.childNodes.length : 0;
						attributes.category = getCategory(nodeName)[0] || "";
			      attributes.childTextNodes = childNodes - childElements;
						attributes.childElements = childElements;
						attributes.index = i;
						attributes.nodeName = nodeName;
						
						report[nodeName] ? report[nodeName].push(attributes) : report[nodeName]= [attributes];
					}
				}catch(e){}
				return report;
		   };
		var reports = Object.entries([...document.querySelectorAll("*"+options.skip)].reduce(getAttributes,{})).reduce((report,entry) => [...report,...entry[1]],[]);
		//var reports = [...document.querySelectorAll("*")].reduce(getAttributes,{});
			var filename = `report`;
			console.log({filename,reports})
      downloadCSV(reports,filename)
  }
  const loadScripts = ({scripts}) => {
    var scriptCountdown = scripts.length;
    var loadScript = (url) => {
      var scriptsLoaded = () => scriptCountdown == 0 ? run() : null;
      var imported = document.createElement('script');
      imported.src = url;
      imported.addEventListener("load", () => {
        scriptCountdown--;
        scriptsLoaded();
      });
      document.head.appendChild(imported);
    }
    scripts.forEach(loadScript)
  }
  loadScripts(options);
})();