Portfolio Blog Snippets

Scrape(Thingiverse API): Get Thing Batches by Id via DnD

var downloadCSV = (data, fileName) => {
		var d = document,
    a = d.createElement('a');
    a.href = window.URL.createObjectURL(new Blob([d3.csvFormat(data)],{type:'text/csv;charset=utf-8;'}));
    a.setAttribute('download', `${fileName}.csv`);
    d.body.appendChild(a);
    a.click();
    d.body.removeChild(a);

finalResults = [];
  };
var finalResults = [];
(() => {
  var options = {
    scripts: ["https://d3js.org/d3.v6.min.js"],
  }

var run = (data) => {
    console.log(`Import Data by dropping files anywhere on the browser window. Export Data: downloadCSV(data, "fileName") `);
  }

const fileToString = (inputFile) => {
	const fr = new FileReader();
	return new Promise((resolve, reject) => {
		fr.onerror = () => {fr.abort(); reject({error:"file reader error"})}
		fr.onload = () => resolve(fr.result); 
		fr.readAsText(inputFile);
})}
var apiFetch = (d) => fetch(` https://api.thingiverse.com/things/${d.id}`, {
    "headers": {
        "Authorization": "Bearer 56edfc79ecf25922b98202dd79a291aa"
    }
	})

var requestBatch = async (arr) => {
 var batch = [];
 for (let d of arr){
   try{
    batch.push(apiFetch(d));
	 }catch(e){
		console.log({e});
	}
 }
 var responses = await Promise.all(batch);
 var collection = [];
 for (let res of responses){
   collection.push(res.json())
 }
 var data = await Promise.all(collection);
 return await data;
}
const dropHandler = async (e) => {
	e.preventDefault(); //disable default drop behavior
  for await (let file of e.dataTransfer.files) { //loop through dropped files
		let r = {format:file.name.slice(file.name.lastIndexOf(".")+1)}
		const str = await fileToString(file); //convert dropped file to string
		var parse = () => {
			var stringParser = () => {
				var parseStr = (delimiter) => str.split(delimiter);
				var testStr = (delimiter) => str.indexOf(delimiter) > -1
				var t = {hasLines:testStr("\n"),hasComma:testStr(","),hasTab:testStr("\t")};
				return t.hasLines ? parseStr("\n") : t.hasTab ? parseStr("\t") : t.hasComma ? parseStr(",") : [];
			}
			return r.format == "json" ? JSON.parse(str) :
			 r.format == "txt" ? stringParser() :
			 r.format == "csv" ? d3.csvParse(str) : []
		};
		r.name = file.name.replace("."+r.format,""),
		r.data = parse();
		var count = r.data.length;
		console.log({r});
			async function* ag() {
   let i = 0;
    while (i < Math.ceil(count/75)) {
     yield i++;
    }
  }
for await (let n of ag()) {
	n = n * 75;
	var batchData = await requestBatch(r.data.slice(n,(n+75)));
  for await (let item of batchData){

finalResults.push(Object.entries(item).reduce((o,e)=> {
		e[0].indexOf("_count") > -1 || e[0] == "id" ? o[e[0]] = e[1] : null;
		return o;
	},{}));
  count--;
 
  if(count <= 0 || finalResults.length >= 10000){
     downloadCSV(finalResults, "thingsById");
  }
    }

}

}
 }
window.ondragover = (e) => e.preventDefault(); //Prevent default browser behavior
window.ondrop = (e) => dropHandler(e); //Handle dropped files
const loadScripts = (scripts) => {
	console.log(`Loading External Scripts`);
	var scriptCountdown = scripts.length;
	var loadScript = (url) => {
		var scriptsLoaded = () => {
			scriptCountdown == 0 ? run() : null;
			return true;
		}
		var imported = document.createElement('script');
				imported.src = url;
				imported.addEventListener("load", () => {scriptCountdown--; scriptsLoaded();});
				document.head.appendChild(imported);
	}
	scripts.forEach(loadScript)
}
loadScripts(options.scripts);
})();