{
    "subcategory": "",
    "sid": "",
    "category": "Models",
    "cid": 70
  },
  {
    "subcategory": "Biology",
    "sid": 106,
    "category": "Learning",
    "cid": 69
  },
  {
    "subcategory": "Engineering",
    "sid": 104,
    "category": "Learning",
    "cid": 69
  },
  {
    "subcategory": "Math",
    "sid": 105,
    "category": "Learning",
    "cid": 69
  },
  {
    "subcategory": "Physics & Astronomy",
    "sid": 148,
    "category": "Learning",
    "cid": 69
  },
  {
    "subcategory": "",
    "sid": "",
    "category": "Learning",
    "cid": 69
  },
  {
    "subcategory": "Bathroom",
    "sid": 147,
    "category": "Household",
    "cid": 67
  },
  {
    "subcategory": "Containers",
    "sid": 146,
    "category": "Household",
    "cid": 67
  },
  {
    "subcategory": "Decor",
    "sid": 97,
    "category": "Household",
    "cid": 67
  },
  {
    "subcategory": "Household Supplies",
    "sid": 99,
    "category": "Household",
    "cid": 67
  },
  {
    "subcategory": "Kitchen & Dining",
    "sid": 100,
    "category": "Household",
    "cid": 67
  },
  {
    "subcategory": "Office Organization",
    "sid": 101,
    "category": "Household",
    "cid": 67
  },
  {
    "subcategory": "Outdoor & Garden",
    "sid": 98,
    "category": "Household",
    "cid": 67
  },
  {
    "subcategory": "Pets",
    "sid": 103,
    "category": "Household",
    "cid": 67
  },
  {
    "subcategory": "",
    "sid": "",
    "category": "Household",
    "cid": 67
  },
  {
    "subcategory": "Automotive",
    "sid": 155,
    "category": "Hobby",
    "cid": 66
  },
  {
    "subcategory": "DIY",
    "sid": 93,
    "category": "Hobby",
    "cid": 66
  },
  {
    "subcategory": "Electronics",
    "sid": 92,
    "category": "Hobby",
    "cid": 66
  },
  {
    "subcategory": "Music",
    "sid": 94,
    "category": "Hobby",
    "cid": 66
  },
  {
    "subcategory": "R/C Vehicles",
    "sid": 95,
    "category": "Hobby",
    "cid": 66
  },
  {
    "subcategory": "Robotics",
    "sid": 96,
    "category": "Hobby",
    "cid": 66
  },
  {
    "subcategory": "Sport & Outdoors",
    "sid": 140,
    "category": "Hobby",
    "cid": 66
  },
  {
    "subcategory": "",
    "sid": "",
    "category": "Hobby",
    "cid": 66
  },
  {
    "subcategory": "Audio",
    "sid": 141,
    "category": "Gadgets",
    "cid": 65
  },
  {
    "subcategory": "Camera",
    "sid": 86,
    "category": "Gadgets",
    "cid": 65
  },
  {
    "subcategory": "Computer",
    "sid": 87,
    "category": "Gadgets",
    "cid": 65
  },
  {
    "subcategory": "Mobile Phone",
    "sid": 88,
    "category": "Gadgets",
    "cid": 65
  },
  {
    "subcategory": "Tablet",
    "sid": 90,
    "category": "Gadgets",
    "cid": 65
  },
  {
    "subcategory": "Video Games",
    "sid": 91,
    "category": "Gadgets",
    "cid": 65
  },
  {
    "subcategory": "",
    "sid": "",
    "category": "Gadgets",
    "cid": 65
  },
 
  {
    "subcategory": "Accessories",
    "sid": 81,
    "category": "Fashion",
    "cid": 64
  },
  {
    "subcategory": "Bracelets",
    "sid": 82,
    "category": "Fashion",
    "cid": 64
  },
  {
    "subcategory": "Costume",
    "sid": 142,
    "category": "Fashion",
    "cid": 64
  },
  {
    "subcategory": "Earrings",
    "sid": 139,
    "category": "Fashion",
    "cid": 64
  },
  {
    "subcategory": "Glasses",
    "sid": 83,
    "category": "Fashion",
    "cid": 64
  },
  {
    "subcategory": "Jewelry",
    "sid": 84,
    "category": "Fashion",
    "cid": 64
  },
  {
    "subcategory": "Keychains",
    "sid": 130,
    "category": "Fashion",
    "cid": 64
  },
  {
    "subcategory": "Rings",
    "sid": 85,
    "category": "Fashion",
    "cid": 64
  },
 {
    "subcategory": "",
    "sid": "",
    "category": "Fashion",
    "cid": 64
  },
  {
    "subcategory": "",
    "sid": "",
    "category": "Art",
    "cid": 63
  },
  {
    "subcategory": "Art Tools",
    "sid": 75,
    "category": "Art",
    "cid": 63
  },
  {
    "subcategory": "Coins & Badges",
    "sid": 143,
    "category": "Art",
    "cid": 63
  },
  {
    "subcategory": "Interactive Art",
    "sid": 78,
    "category": "Art",
    "cid": 63
  },
  {
    "subcategory": "Math Art",
    "sid": 79,
    "category": "Art",
    "cid": 63
  },
  {
    "subcategory": "Scans & Replicas",
    "sid": 145,
    "category": "Art",
    "cid": 63
  },
  {
    "subcategory": "Sculptures",
    "sid": 80,
    "category": "Art",
    "cid": 63
  },
  {
    "subcategory": "Signs & Logos",
    "sid": 76,
    "category": "Art",
    "cid": 63
  },
  {
    "subcategory": "2D Art",
    "sid": 144,
    "category": "Art",
    "cid": 63
  },
  {
    "subcategory": "3D Printer Accessories",
    "sid": 127,
    "category": "3D Printing",
    "cid": 73
  },
  {
    "subcategory": "3D Printer Extruders",
    "sid": 152,
    "category": "3D Printing",
    "cid": 73
  },
  {
    "subcategory": "3D Printer Parts",
    "sid": 128,
    "category": "3D Printing",
    "cid": 73
  },
  {
    "subcategory": "3D Printers",
    "sid": 126,
    "category": "3D Printing",
    "cid": 73
  },
  {
    "subcategory": "3D Printing Tests",
    "sid": 129,
    "category": "3D Printing",
    "cid": 73
  },

{
    "subcategory": "",
    "sid": "",
    "category": "3D Printing",
    "cid": 73
  },
  {
    "subcategory": "",
    "sid": "",
    "category": "Other",
    "cid": 0
  }
];
var run = async () => {
  var recordCount = 10000,
		perPage = 50,
		batchSize = categories.length,
		pageCount = Math.ceil(recordCount / perPage),
		batchCount = 100,
		requestCount = Math.ceil(pageCount / batchCount);
	console.log({recordCount,perPage,batchSize,batchCount});
  
	async function* ag() {
   let i = 0;
    while (i < batchCount) {
     yield i++;
    }
  }
var apiFetch = (cat,n) => {
	 n++;
	 reqTotal++;
	 //console.log({page:n,reqTotal});
   return fetch(`https://api.thingiverse.com/search/?q=&page=${n}&per_page=${perPage}&category_id=${cat}&sort=popular&type=things`, { //&posted_after=now-365d //&posted_before=now-365d
    "headers": {
        "Authorization": "Bearer 56edfc79ecf25922b98202dd79a291aa"
    }
	})
 }
var requestBatch = async (cat,n) => {
 var batch = [];
 for (let i = n; i < (n+batchSize); i++){
   try{
    batch.push(apiFetch(cat,i));
	 }catch(e){
		console.log({e});
	}
 }
 var responses = await Promise.all(batch);
 var collection = [];
 for (let res of responses){
   collection.push(res.json())
 }
 var data = await Promise.all(collection);
 return await data;
}
var downloadReport = () => {
console.log({records:records.length})
 		records = records.reduce((a,d) => {
			var prev = uniq.size;
			uniq.add(d.id);
			if(prev < uniq.size){
				d.tags = Array.isArray(d.tags) ? d.tags.map(tag => tag.name).join("|") : "";
				var creator = {...d.creator};
				d.creatorId = creator.id;
				d.lastname = creator.last_name;
				d.firstname = creator.first_name;
        d.creatorName = creator.name;
				a.push([
  "id",
  "category",
  "subcategory",
  "creatorName",
  "name",
  "public_url",
  "created_at",
  "creatorId",
  "comment_count",
  "like_count",
  "tags",
  "lastname",
  "firstname"
].reduce((o,k) => ({...o, [k]:d.hasOwnProperty(k) ? k.indexOf("_count") > -1 && isNaN(d[k]) ? 0 : d[k] : ""}),{}));		
			} 
			return a;
		},[])
 console.log({reduced:records.length});
	downloadCSV(records,`${records[0].category}_${records[0].subcategory}`);
	records = [];
	return true;
}
var done = false;
for await (let n of ag()) {
var cat = categories[n];
if(!cat){
	cat = categories[categories.length-1];
	done = true;
}
 var catId = cat.sid !== "" ? cat.sid : cat.cid;
  console.log("Category: "+cat.category + cat.subcategory);
  try{
	 var data = await requestBatch(catId,n);
	 for (let d of data){
		records = [...records, ...(d.hits || []).map(t => ({...t, ...cat}))];
	 }
    
	 }catch(e){
		console.log({e});
	}
  if(catId != cat.sid || !cat.sid || cat.id == 0){
		 console.log("Downloading "+records.length);
     await downloadReport(cat.category,cat.subcategory);
	}
	if(done){ break;}
}
   console.log("DONE")
}