Add API v4, which does not provide `rawraw` by default. Move changelog (f1196787) · Commits · git-mirror / Youtubevideofinder

app.py

+10 −5

Original line number	Diff line number	Diff line
		@@ -10,7 +10,7 @@ with open('config.yml', 'r') as file:
		@app.route("/robots.txt")
		async def robots():
		return """
		# I'm 100% fine with crawlers, just don't fuck up my servers.
		# Please be courteous and have at least a second or two of delay between requests.
		User-Agent: *
		Crawl-delay: 2
		Disallow:
		@@ -23,11 +23,11 @@ async def youtubev2(id):
		"""
		return (await lostmediafinder.YouTubeResponse.generate(id)).coerce_to_api_version(2).json()

		async def wrapperYT(id):
		async def wrapperYT(id, includeRaw):
		"""
		Wrapper for generateAsync
		"""
		return await lostmediafinder.YouTubeResponse.generate(id)
		return await lostmediafinder.YouTubeResponse.generate(id, includeRaw)

		@app.route("/api/v<int:v>/<site>/<id>")
		@app.route("/api/v<int:v>/<id>")
		@@ -35,12 +35,17 @@ async def youtube(v, id, site="youtube", json=True):
		"""
		Wrapper around lostmediafinder
		"""
		includeRaw = True
		if v == 1:
		return "This API version is no longer supported.", 410
		if v not in (2, 3):
		if v not in (2, 3, 4):
		return "Unrecognised API version", 404
		if site == "youtube":
		r = (await wrapperYT(id)).coerce_to_api_version(v)
		includeRaw = True
		if v >= 4:
		# Versions 4 and higher only provide `rawraw` if you ask for it
		includeRaw = "includeRaw" in request.args
		r = (await wrapperYT(id, includeRaw=includeRaw)).coerce_to_api_version(v)
		if json:
		return r.json()
		return r

lostmediafinder/finder.py

+16 −16

Original line number	Diff line number	Diff line
		@@ -15,12 +15,12 @@ class YouTube(YouTubeService):
		configId = "youtube"

		@classmethod
		async def _run(cls, id, session: aiohttp.ClientSession, includeRaw=True) -> T:
		async def _run(cls, id, session: aiohttp.ClientSession) -> T:
		lien = f"https://i.ytimg.com/vi/{id}/hqdefault.jpg"
		async with session.head(lien, allow_redirects=False, timeout=15) as response:
		code = response.status

		rawraw = code if includeRaw else None
		rawraw = code
		archived = None
		link = f"https://youtu.be/{id}"

		@@ -45,7 +45,7 @@ class WaybackMachine(YouTubeService):
		configId = "ia_wayback"

		@classmethod
		async def _run(cls, id, session: aiohttp.ClientSession, includeRaw=True) -> T:
		async def _run(cls, id, session: aiohttp.ClientSession) -> T:
		ismeta = False
		lien = f"https://web.archive.org/web/2oe_/http://wayback-fakeurl.archive.org/yt/{id}"
		async with session.head(lien, allow_redirects=False, timeout=15) as response:
		@@ -66,7 +66,7 @@ class WaybackMachine(YouTubeService):
		ismeta = True
		lien = response2["archived_snapshots"]["closest"]["url"]

		rawraw = (redirect, response2) if includeRaw else None
		rawraw = (redirect, response2)
		return cls(
		archived=archived, capcount=int(archived), rawraw=rawraw,
		available=lien, lastupdated=time.time(), name=cls.getName(),
		@@ -86,7 +86,7 @@ class ArchiveOrgDetails(YouTubeService):
		]

		@classmethod
		async def _run(cls, id, session: aiohttp.ClientSession, includeRaw=True) -> T:
		async def _run(cls, id, session: aiohttp.ClientSession) -> T:
		responses = []
		is_dark = False
		for template in cls.items_tried:
		@@ -100,7 +100,7 @@ class ArchiveOrgDetails(YouTubeService):
		is_dark = False
		break
		archived = bool(metadata)
		rawraw = responses if includeRaw else None
		rawraw = responses
		lien = f"https://archive.org/details/{ident}" if archived else None
		note = ""
		if not archived:
		@@ -132,7 +132,7 @@ class ArchiveOrgCDX(YouTubeService):
		configId = "ia_cdx"

		@classmethod
		async def _run(cls, id, session: aiohttp.ClientSession, includeRaw=True) -> T:
		async def _run(cls, id, session: aiohttp.ClientSession) -> T:
		cdx_urls = [
		f"https://web.archive.org/cdx/search/cdx?url=i.ytimg.com/vi/{id}*&collapse=digest&filter=statuscode:200&mimetype:image/jpeg&output=json",
		f"https://web.archive.org/cdx/search/cdx?url=i1.ytimg.com/vi/{id}*&collapse=digest&filter=statuscode:200&mimetype:image/jpeg&output=json",
		@@ -200,11 +200,11 @@ class GhostArchive(YouTubeService):
		configId = "ghostarchive"

		@classmethod
		async def _run(cls, id, session: aiohttp.ClientSession, includeRaw=True) -> T:
		async def _run(cls, id, session: aiohttp.ClientSession) -> T:
		link = f"https://ghostarchive.org/varchive/{id}"
		async with session.get(link) as resp:
		code = resp.status
		rawraw = code if includeRaw else None
		rawraw = code
		archived = None
		with Switch(code) as case:
		if case(200):
		@@ -236,7 +236,7 @@ class HackintYa(YouTubeService):
		configId = "hackint_ya"

		@classmethod
		async def _run(cls, id, session: aiohttp.ClientSession, includeRaw=True):
		async def _run(cls, id, session: aiohttp.ClientSession):
		username = methods[cls.configId]["username"]
		password = methods[cls.configId]["password"]

		@@ -252,7 +252,7 @@ class HackintYa(YouTubeService):
		commentcount = await resp.text()
		archived = (count > 0)
		comments = [i for i in commentcount.split("\n") if i.strip("∅\n") and i.strip() != "0"]
		rawraw = (count, commentcount) if includeRaw else None
		rawraw = (count, commentcount)
		return cls(
		archived=archived, capcount=count, comments=(len(comments) > 0), lastupdated=time.time(), name=cls.getName(),
		note=cls.note if archived else "", rawraw=rawraw, metaonly=False
		@@ -268,7 +268,7 @@ class DistributedYoutubeArchive(YouTubeService):
		configId = "distributed_youtube_archive"

		@classmethod
		async def _run(cls, id, session: aiohttp.ClientSession, includeRaw=True):
		async def _run(cls, id, session: aiohttp.ClientSession):
		token = methods[cls.configId]['key']
		user_agent = FYT_UA
		lastupdated = time.time()
		@@ -310,7 +310,7 @@ class Hobune(YouTubeService):
		cooldown = 0.5

		@classmethod
		async def _run(cls, id, session: aiohttp.ClientSession, includeRaw=True):
		async def _run(cls, id, session: aiohttp.ClientSession):
		while time.time() - cls.lastretrieved < cls.cooldown:
		await asyncio.sleep(0.1)
		user_agent = "FindYoutubeVideo/1.0 operated by thetechrobo@proton.me"
		@@ -352,7 +352,7 @@ class Filmot(YouTubeService):
		configId = "filmot"

		@classmethod
		async def _run(cls, id, session: aiohttp.ClientSession, includeRaw=True) -> T:
		async def _run(cls, id, session: aiohttp.ClientSession) -> T:
		key = methods[cls.configId]["api_key"]

		while time.time() - cls.lastretrieved < cls.cooldown:
		@@ -361,7 +361,7 @@ class Filmot(YouTubeService):
		cls.lastretrieved = time.time()
		async with session.get(f"https://filmot.com/api/getvideos?key={key}&id={id}&flags=1") as resp:
		metadata = await resp.json(content_type=None)
		rawraw = metadata if includeRaw else None
		rawraw = metadata
		if len(metadata) > 0: # pylint: disable=simplifiable-if-statement
		archived = True
		else:
		@@ -385,7 +385,7 @@ class Playboard(YouTubeService):
		configId = "playboard_co"

		@classmethod
		async def _run(cls, id, session: aiohttp.ClientSession, includeRaw=True):
		async def _run(cls, id, session: aiohttp.ClientSession):
		user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.%s.0.0 Safari/537.36"
		user_agent = user_agent % random.randint(0, 100)
		url = f"https://playboard.co/en/video/{id}"

lostmediafinder/types.py

+21 −6

Original line number	Diff line number	Diff line
		@@ -59,7 +59,7 @@ class Service(JSONDataclass):
		configId = None

		@classmethod
		async def _run(cls, id, session: aiohttp.ClientSession, includeRaw=True) -> T:
		async def _run(cls, id, session: aiohttp.ClientSession) -> T:
		raise NotImplementedError("Subclass Service and impl the _run function")

		@classmethod
		@@ -79,7 +79,10 @@ class Service(JSONDataclass):
		includeRaw (bool): Whether or not to include the raw data as sent from the service. If you don't need this data, turn this off; it's only the default for compatibility.
		"""
		try:
		return await cls._run(id, session, includeRaw=includeRaw, **kwargs)
		res = await cls._run(id, session, **kwargs)
		if not includeRaw:
		res.rawraw = None
		return res
		except Exception as ename: # pylint: disable=broad-except
		note = f"An error occured while retrieving data from {cls.getName()}."
		print(ename)
		@@ -124,12 +127,17 @@ class YouTubeResponse(JSONDataclass):
		keys (list[YouTubeService]): An array with all the server responses. THIS IS DIFFERENT THAN BEFORE! Before, this would be an array of strings. You'd use the strings as keys. Now, this array has the data directly!
		api_version (int): The API version. Breaking API changes are made by incrementing this.
		verdict (dict): The verdict of the response. Has video, metaonly, and comments field, that are set to true if any archive was found where that was saved. Also has human_friendly field that has a simple verdict that can be used by people.

		=Changelog=
		API VERSION 3 -> 4:
		The `rawraw` field is now only provided if you provide the `includeRaw` parameter and set it to True.
		Example in a URL: <code>/api/v4/dQw4w9WgXcQ?includeRaw=true</code>
		"""
		id: str
		status: str
		keys: list[YouTubeService]
		verdict: dict
		api_version: int = 3
		api_version: int = 4

		def coerce_to_api_version(selfNEW, target): # pylint: disable=no-self-argument
		"""
		@@ -150,6 +158,12 @@ class YouTubeResponse(JSONDataclass):
		assert self.api_version == target
		return self

		# There were no changes to the data structure between v3 and v4
		def _convert_v4_to_v3(selfNEW): # pylint: disable=no-self-argument
		assert self.api_version == 4
		self.api_version = 3
		return copy.deepcopy(selfNEW)

		def _convert_v3_to_v2(selfNEW): # pylint: disable=no-self-argument
		self = copy.deepcopy(selfNEW)
		assert self.api_version == 3
		@@ -194,11 +208,12 @@ class YouTubeResponse(JSONDataclass):
		return verdict

		@classmethod
		async def generate(cls, id):
		async def generate(cls, id: str, includeRaw=False):
		"""
		Runs all the Services.
		Arguments:
		id: The video ID
		id (str): The video ID
		includeRaw (bool): Whether or not to include the raw data in the `rawraw` field. If you don't need it, disable this.
		"""
		if not cls.verifyId(id):
		return cls(status="bad.id", id=id, keys=[], verdict={"video":False,"comments":False,"metaonly":False,"human_friendly":"Invalid video ID. "})
		@@ -207,7 +222,7 @@ class YouTubeResponse(JSONDataclass):
		coroutines = []
		async with aiohttp.ClientSession() as session:
		for service in services:
		coroutines.append(service.run(id, session))
		coroutines.append(service.run(id, session, includeRaw=includeRaw))
		results = await asyncio.gather(*coroutines)
		for result in results:
		keys.append(result)

templates/api.html

+25 −38

Original line number	Diff line number	Diff line
		@@ -29,32 +29,24 @@
		</div>
		<div class="container">
		<div class="banner">
		Please don't spam my servers. Anyone that spams too much may be banned. Contact me on Discord
		(username: thetechrobo) or IRC (TheTechRobo on hackint, libera.chat, and OFTC) to discuss.
		Please don't spam my servers. Anyone who makes too many requests may be banned. Contact me on Discord
		(username: thetechrobo), IRC (TheTechRobo on hackint, libera.chat, and OFTC), or email (thetechrobo@proton.me) to discuss.
		</div>
		<br>
		<h4>API Documentation</h4>
		<h6>Call: GET <code>/api/:version/:videoid</code></h6>
		<p>Current versions available: v2, v3</p>
		<b>Fields</b>
		{% if changelog[0] %}
		<div id="Fcl">
		<h6>Accepted parameters: <code>includeRaw</code> (set to include the <code>rawraw</code> field)</h6>
		<p>Current versions available: v2, v3, v4. Documentation below only applies to the latest version.</p>
		<u>Changelog</u>
		<div id="changelog">
		<ul>
		{% for log, logitems in changelog[0].items() %}
		{% if log %}
		<li>Api version {{ log }}</li>
		<ul>
		{% for logitem in logitems %}
		<li>{{ logitem }}</li>
		{% endfor %}
		<li>API version v3 -> 4:</li>
		<ul id="root_3to4">
		<li>The <code>rawraw</code> field is now only provided if you set the <code>includeRaw</code> parameter.</li>
		</ul>
		{% endif %}
		{% endfor %}
		</ul>
		</div>
		{% endif %}
		<br>
		<b>Fields</b>
		<dl>
		{% if fields is mapping %}
		{% for field, value in fields.items() %}
		@@ -67,24 +59,19 @@
		Fail
		{% endif %}
		</dl>
		<u>NOTE:</u> The <code>rawraw</code> field is only set if you set the <code>includeRaw</code> parameter. Example: <code>/api/v4/dQw4w9WgXcQ?includeRaw=true</code>. Please only enable this if you have to, as it will slow down loading.
		</div>
		<div class="container">
		<b>Service Objects</b>
		{% if changelog[1] %}
		<div id="Scl">
		<div id="changelog2">
		<u>Changelog</u>
		<ul>
		{% for log, logitems in changelog[1].items() %}
		{% if log %}
		<li>Api version {{ log }}</li>
		<ul>
		{% for logitem in logitems %}
		<li>{{ logitem }}</li>
		{% endfor %}
		<li>API version 2 -> 3:</li>
		<ul id="service_2to3">
		<li>The <code>error</code> field is no longer a boolean; it now contains an error message if an error occured and null otherwise</li>
		</ul>
		{% endif %}
		{% endfor %}
		</ul>
		</div>
		{% endif %}
		{% if services is mapping %}
		<dl>
		{% for field, value in services.items() %}