Portfolio Blog Snippets

Scrape(Rate My Professor|Multi): Crawl Prof SERPs

var collection = [];
(async () => {
const download = (data, fileName) => {
  var jsonDUMP = JSON.stringify(data);
  var exportFilename = fileName + ".json";
  var csvData = new Blob([jsonDUMP], {
    type: 'text/csv;charset=utf-8;'
  });
  var link = document.createElement('a');
  link.href = window.URL.createObjectURL(csvData);
  link.setAttribute('download', exportFilename);
  document.body.appendChild(link);
  link.click();
  document.body.removeChild(link);
};

var run = async () => {
var skip = 1000;
for (i = 0; i < 1000; i++){
var res = await fetch("https://www.ratemyprofessors.com/search.jsp?query=&queryoption=HEADER&stateselect=&country=united+states&dept=&queryBy=teacherName&facetSearch=true&schoolName=&offset="+((i*20)+skip)+"&max=20");
var text = await res.text();
var parser = await new DOMParser();
var doc = await parser.parseFromString(text, "text/html");
var links = [...doc.querySelectorAll(`[href*='/ShowRatings']`)].map(a => {
	return {
		name:a.querySelector(".main").innerText.trim(),
		description:a.querySelector(".sub").innerText.trim(),
		category:a.querySelector(".listing-cat").innerText.trim().toLowerCase(),
		url:a.href
	}
});
collection = [...collection, ...links];
}
}
await run();
download(collection,"ratemyprofessor-20k_results-teachers-us") 
})()