philshem’s gists

philshem / all.csv

Last active February 24, 2022 04:00

getting the youtube stats from a channel (john oliver's last week tonight - hbo)

index	upload_date	duration	webpage_url	view_count	like_count	dislike_count	average_rating	series	season_number	episode_number	release_date	thumbnail	fulltitle
Cy-O4myeUzg	20161107	1949	https://www.youtube.com/watch?v=Cy-O4myeUzg	2801183			4.8850956					https://i.ytimg.com/vi_webp/Cy-O4myeUzg/maxresdefault.webp	Mercadeo Multinivel: Last Week Tonight With John Oliver (HBO)
ekoETowzmAo	20170118	84	https://www.youtube.com/watch?v=ekoETowzmAo	5724473			4.8252573					https://i.ytimg.com/vi/ekoETowzmAo/maxresdefault.jpg	Season 4 Official Trailer: Last Week Tonight with John Oliver (HBO)
eAFnby2184o	20141229	225	https://www.youtube.com/watch?v=eAFnby2184o	5421913			4.8301988					https://i.ytimg.com/vi_webp/eAFnby2184o/maxresdefault.webp	New Year's Eve (Web Exclusive): Last Week Tonight with John Oliver (HBO)
FVFdsl29s_Q	20170521	1446	https://www.youtube.com/watch?v=FVFdsl29s_Q	14654842			4.8198462					https://i.ytimg.com/vi_webp/FVFdsl29s_Q/maxresdefault.webp	Stupid Watergate: Last Week Tonight with John Oliver (HBO)

philshem / get_data.py

Created August 17, 2020 14:51

scraper for vehicle counts from autoscout.ch https://twitter.com/philshem/status/1295371670321070080

	import requests
	from bs4 import BeautifulSoup
	import json
	from collections import defaultdict
	from random import randint
	from time import sleep

	'''
	scraper for counts of each make of car found at autoscout.ch
	https://twitter.com/philshem/status/1295371670321070080

philshem / bikes.ipynb

Created July 8, 2020 05:42

bikes.ipynb

Sorry, something went wrong. Reload?

Sorry, we cannot display this file.

Sorry, this file is invalid so it cannot be displayed.

philshem / traffic.ipynb

Created July 7, 2020 07:28

traffic.ipynb

Sorry, something went wrong. Reload?

Sorry, we cannot display this file.

Sorry, this file is invalid so it cannot be displayed.

philshem / make_gsg.sh

Last active May 2, 2020 10:48

ghana says goodbye to coronavirus protestors https://twitter.com/philshem/status/1256535940442591232

	# made by @philshem
	# https://twitter.com/philshem/status/1256535940442591232
	# the unlicense https://unlicense.org/
	# requires youtube-dl and ffmpeg (easily installed with homebrew or conda)

	# download the video
	youtube-dl jS0bjDbWydw --output raw.mp4

	# get dimensions
	# 1920 × 1080 in this case

philshem / _get_swissgrid_data.py

Last active May 20, 2020 07:14

get swiss energy consumption open data, combine with weather measurements https://twitter.com/philshem/status/1252535904918478850

	import os
	import pandas as pd
	from datetime import datetime

	url='https://www.swissgrid.ch/dam/dataimport/energy-statistic/EnergieUebersichtCH-{YEAR}.xls'

	dfs = []
	for y in range(2009,datetime.now().year+1):
	# generate url with year in it
	url_y = url.format(YEAR=str(y))

philshem / scrape_the_scrapers.sh

Last active April 4, 2020 15:47

	# fetches raw data from https://shopsensor-272815.appspot.com/
	#
	# requires multiple queries for each city's bounding box
	#
	# if you actually run this, pipe each curl statement to a seperate file

	# bern, 27 results
	curl 'https://shopsensor-272815.appspot.com/searchresults' -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:75.0) Gecko/20100101 Firefox/75.0' -H 'Accept: application/json, text/javascript, /; q=0.01' -H 'Accept-Language: en-US,en;q=0.5' --compressed -H 'Content-Type: application/json; charset=utf-8' -H 'X-Requested-With: XMLHttpRequest' -H 'Origin: https://shopsensor-272815.appspot.com' -H 'DNT: 1' -H 'Connection: keep-alive' -H 'Referer: https://shopsensor-272815.appspot.com/' -H 'Pragma: no-cache' -H 'Cache-Control: no-cache' --data '{"_southWest":{"lat":46.92687938388263,"lng":7.421092987060548},"_northEast":{"lat":46.96941802389087,"lng":7.471647262573243}}'
	# > bern.json

philshem / covi19_gmaps_crowds_zurich

Last active March 20, 2020 16:35

	#!/usr/bin/env python

	'''
	Post processing the scrape job
	requires `pip install pandas gcsfs`
	'''

	import pandas as pd
	import numpy as np
	from datetime import datetime

philshem / covid19_kanton_zurich.py

Created March 12, 2020 11:35

get kanton zurich coronavirus cases and do some basic calcs

	import pandas as pd
	import numpy as np
	from datetime import datetime

	url = 'https://raw.githubusercontent.com/openZH/covid_19/master/COVID19_Fallzahlen_Kanton_ZH_alter_geschlecht.csv'

	# define date parsing format
	dateparse = lambda dates: [datetime.strptime(d, '%d.%m.%Y') for d in dates]

	# read csv from web

philshem / get_swiss_coronavirus_cases.py

Created March 3, 2020 15:57

scraper for count of Swiss coronavirus confirmed cases

	#!/usr/bin/python3

	# pip3 install pandas matplotlib
	import pandas as pd
	from matplotlib import pyplot as plt

	# this gets the worldwide confirmed cases from JHU
	url = 'https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_19-covid-Confirmed.csv'

	# read/download csv

@philshem philshem