Scrape(Twitter): Crawl Post Feed (infinite scroll)

var collection = [];
var index = new Set();
const download = (data, fileName) => {
  var jsonDUMP = JSON.stringify(data);
  var exportFilename = fileName + ".json";
  var csvData = new Blob([jsonDUMP], {
    type: 'text/csv;charset=utf-8;'
  });
  var link = document.createElement('a');
  link.href = window.URL.createObjectURL(csvData);
  link.setAttribute('download', exportFilename);
  document.body.appendChild(link);
  link.click();
  document.body.removeChild(link);
	collection = [];
	i = 0;
};
var dip = async () => {
	return await [...document.querySelectorAll(`[lang="en"]:not(html)`)]
		.map(el => ({
time:el.parentElement.parentElement.parentElement.querySelector("time").getAttribute("datetime"),
metrics:[...el.parentElement.parentElement.parentElement.querySelectorAll("[data-testid]")].map(testEl => ({att:testEl.attributes ? [...testEl.attributes].reduce((obj,a) => {
										if(a.nodeName && a.textContent){
											
if(a.nodeName == "aria-label" && ["Retweet","Like","Reply"].some(str => a.textContent.indexOf(str) > -1)){
	var metric = a.textContent.split(". ")[1] || a.textContent;
  obj[metric.toLowerCase().trim()] = +(a.textContent.replace(/\D+/gim,""))
}
										}
										
										return obj;
							},{}) : []})).filter(d => Object.keys(d.att).length).reduce((m,d) => {
Object.keys(d.att).forEach(key => m[key] = d.att[key])
return m;},{}),
links:[...el.parentElement.parentElement.parentElement.querySelectorAll("a")].map(a => ({ 
	text: a.innerText, url: a.href
})),images:[...el.parentElement.parentElement.parentElement.querySelectorAll("img")].map(img => ({
									att:img.attributes ? [...img.attributes].reduce((obj,a) => {
										if(a.nodeName && a.textContent){
											obj[a.nodeName] = a.textContent;
										}
										
										return obj;
							},{}) : []})),
				text:el.innerText.replace(/[\n\r]+/gim," ")
                			 .replace(/[ \t]+/gim," "),
				att:[...el.attributes]
						.reduce((obj,a) => 	{
							obj[a.nodeName] = a.textContent;
							return obj;
						},{})
		}))}
var i = 0;
function rand(max) {
  return Math.floor(Math.random() * Math.floor(max));
}

function sz() {
  setInterval(function(){
		window.scrollBy(0,75+rand(50));
 	}, 600);
	
	setInterval(async function(){
    i++;
		var data = await dip();
		for await (let d of data){
			var prev = +(""+index.size);
			index.add(d.text);
			prev == index.size ? console.log("dupe detected") : collection.push(d);
		}
		if(i == 100) download(collection,"tweetsCollected2020-10-05")
		console.log({collection});
 	}, 2500);
}
sz();