Portfolio Blog Snippets

Userscript(Youtube): Scrape Channel Videos

// ==UserScript==
// @name         Tab Drone
// @version      0.1
// @include      https://www.youtube.com/watch?v=*&tab=1*
// @grant    GM_openInTab
// @grant unsafeWindow
// @grant window.close
// @grant window.focus
// @grant        GM_addValueChangeListener
// @grant        GM_setValue
// @grant        GM_getValue
// @grant        GM_deleteValue
// @grant     GM_listValues
// @run-at document-end
// ==/UserScript==
(() => {
  var options = {
    scripts: ["https://d3js.org/d3.v6.min.js"]
  }

var run = async () => {

var delay = ms => new Promise(r => setTimeout(r, ms))

function downloadCSV(data) {

var csvData = new Blob([d3.csvFormat([data])], {
				type: 'text/csv;charset=utf-8;'
			});
			var link = document.createElement('a');
			link.href = window.URL.createObjectURL(csvData);
			link.setAttribute('download', channelName + ".csv");
			document.body.appendChild(link);
			link.click();
			document.body.removeChild(link);
            window.close();
};
function treatAsUTC(date) {
    var result = new Date(date);
    result.setMinutes(result.getMinutes() - result.getTimezoneOffset());
    return result;
}

function daysBetween(startDate, endDate) {
    var millisecondsPerDay = 24 * 60 * 60 * 1000;
    return (treatAsUTC(endDate) - treatAsUTC(startDate)) / millisecondsPerDay;
}
await delay(1800);
window.scrollTo(0,document.querySelector(":root").scrollHeight*0.12);
await delay(1600);
window.scrollTo(0,document.querySelector(":root").scrollHeight*0.18);
document.querySelector("video").pause();
await delay(2000);
window.scrollTo(0,document.querySelector(":root").scrollHeight*0.24);
await delay(2500);
var toNum = (s) => {
  return s.indexOf("K") > -1 ? +(s.replace("K",""))*1000 :
         s.indexOf("M") > -1 ? +(s.replace("M",""))*1000000 :
         isNaN(+s) ? 0 : +s;
}
try{
var info = [...document.querySelectorAll("#info yt-formatted-string")].reduce((arr,el) => {
    el.innerText ? arr.push(el.innerText) : null;
      return arr;
},[]).slice(0,4);
document.querySelector("video").pause();
var channelName = document.querySelector("#container.ytd-channel-name a");
channelName = channelName ? channelName.innerText.replace(/[\s\n\t\r]+/gim," ").trim() : "";

var description = document.querySelector("#description");
description = description ? description.innerText : "";

var subscribers = [...document.querySelectorAll("#owner-sub-count")].map(el => el.innerText)[0].split(" ")[0];

var jsData = JSON.parse(ytplayer.config.args.player_response).microformat.playerMicroformatRenderer;

var category = jsData.category;
var lengthSeconds = +jsData.lengthSeconds;
var title = jsData.title.simpleText;
var publishDate = jsData.publishDate;
var isFamilySafe = jsData.isFamilySafe;
var embedURL = jsData.embed.iframeUrl;
var height = jsData.embed.height;
var width = jsData.embed.width;
var channel = "https://www.youtube.com/channel/"+ jsData.externalChannelId;
var uploaded = new Date(publishDate);
var uploadedFormatted = publishDate;
var uploadYear = uploaded.getFullYear();
var uploadMonth = uploaded.getMonth()+1;
var uploadDate = uploaded.getDate();
var uploadDay = uploaded.getDay()+1;
var obj = {
    title: info[0],
    category,
    lengthSeconds,
    uploaded:uploadedFormatted,
    uploadYear,
    uploadMonth,
    uploadDate,
    uploadDay,
    description,
    channelName,
    channelURL:channel,
    subscribers:toNum(subscribers),
    category,
    width,
    height,
    embedURL,
    isFamilySafe
}
info = document.querySelector("ytd-video-primary-info-renderer");
info = info ? info.__data.data : {error:"Error"};
var data = {
	published: new Date(info.dateText.simpleText).toISOString().slice(0,10),
	views:+(info.viewCount.videoViewCountRenderer.viewCount.simpleText.replace(/[, a-z]/gim,"")) || ""
};
var sent = info.sentimentBar.sentimentBarRenderer.tooltip;
sent = sent ? sent.split(" / ").map(s => s.replace(/,/gim,"")) : {error:"error"};

data.likes = +sent[0] || "";
data.dislikes = +sent[1] || "";
data.url = document.URL;

var comments = () => document.querySelector(".count-text.style-scope.ytd-comments-header-renderer");
if(!comments()){
 await run();
}else{

data.comments = comments() ? +(comments().innerText.replace(/\D/gim,"")) : comments();

data.snapshotDate = new Date().toISOString().slice(0,10);

data.ageDays = daysBetween(data.published,data.snapshotDate);
data.ageYears = Math.round(data.ageDays/365*1000)/1000;
data.ageMonths = Math.round(data.ageDays/30*1000)/1000;

data.viewsPerDay = data.views / data.ageDays;
data.viewsPerMonth = data.views / data.ageMonths;
data.likeRatio = Math.round(data.likes/ (data.likes + data.dislikes)*1000)/1000;

downloadCSV({...data, ...obj});
}
}catch(e){window.close();}
};

const loadScripts = ({scripts}) => {
    console.log(`Loading External Scripts`);
    var scriptCountdown = scripts.length;
    var loadScript = (url) => {
      var scriptsLoaded = () => scriptCountdown == 0 ? run() : null;
      var imported = document.createElement('script');
      imported.src = url;
      imported.addEventListener("load", () => {
        scriptCountdown--;
        scriptsLoaded();
      });
      document.head.appendChild(imported);
    }
    scripts.forEach(loadScript)
  }
window.addEventListener('DOMContentLoaded', (event) => {
     loadScripts(options);
});
 
})();