Commit 66c7c718 authored by TheTechRobo's avatar TheTechRobo
Browse files

Consistent ordering; only works on JavaScript frontend for now

Partial fix for #75
parent 8c98ceca
Loading
Loading
Loading
Loading
+1 −1
Original line number Diff line number Diff line
@@ -46,7 +46,7 @@ Some scrapers require further configuration (typically authentication) and are d

## Contributing

If creating a new scraper, just make a new class in the finder.py file, and update the example config file.
If creating a new scraper, just make a new class in the finder.py file, and update the example config file. Ensure you register it with the proper type (the `@registry.something`) line. Take a look at the other classes for all the somethings.

Scrapers should try to yield the best or near-best link first, for best compatibility with APIv4 and lower.

+3 −4
Original line number Diff line number Diff line
@@ -102,10 +102,9 @@ def coerce_to_id(vid):

def get_enabled_methods():
    titles = []
    for key in config_yml["methods"]:
        method = config_yml["methods"][key]
        if method["enabled"]:
            titles.append(method["title"])
    for service in findyoutubevideo.types.registry.get_services():
        if service.enabled():
            titles.append(config_yml['methods'][service.configId]['title'])
    return titles

@app.route("/noscript_load.html")
+17 −1
Original line number Diff line number Diff line
@@ -4,7 +4,7 @@ All the Service implementations live here.

import random, time, aiohttp, asyncio
import typing_extensions as typing
from .types import FytSession, Link, LinkContains, Service, methods, experiment_base_url
from .types import FytSession, Link, LinkContains, Service, methods, experiment_base_url, registry
from yarl import URL

async def submit_experiment(session: FytSession, experiment_name: str, video_id: str, **report):
@@ -18,6 +18,7 @@ async def submit_experiment(session: FytSession, experiment_name: str, video_id:
        except Exception:
            pass

@registry.youtube
class YouTube(Service):
    """
    Checks if the video is still available on YouTube.
@@ -58,6 +59,7 @@ class YouTube(Service):
        )


@registry.ia
class WaybackMachine(Service):
    name = methods["ia_wayback"]["title"]
    configId = "ia_wayback"
@@ -208,6 +210,7 @@ class WaybackMachine(Service):
        )


@registry.ia
class ArchiveOrgDetails(Service):
    name = methods["ia_details"]["title"]
    configId = "ia_details"
@@ -270,6 +273,7 @@ class ArchiveOrgDetails(Service):
        )


@registry.ia
class ArchiveOrgCDX(Service):
    """
    Queries the Archive.org CDX for an archived video thumb
@@ -340,6 +344,7 @@ class ArchiveOrgCDX(Service):
        )


@registry.misc
class GhostArchive(Service):
    name = methods["ghostarchive"]["title"]
    configId = "ghostarchive"
@@ -373,6 +378,7 @@ class GhostArchive(Service):
            metaonly=False, classname=cls.__name__
        )

@registry.on_request
class HackintYa(Service):
    name = methods["hackint_ya"]["title"]
    note = ("Video retrieval is currently not available for technical reasons. "
@@ -410,6 +416,7 @@ class HackintYa(Service):
        )


@registry.on_request
class DistributedYoutubeArchive(Service):
    name = methods['distributed_youtube_archive']['title']
    configId = "distributed_youtube_archive"
@@ -450,6 +457,7 @@ class DistributedYoutubeArchive(Service):
            classname=cls.__name__
        )

@registry.public_archives
class Hobune(Service):
    name = methods["hobune_stream"]["title"]
    configId = "hobune_stream"
@@ -498,6 +506,7 @@ class Hobune(Service):
            note="", rawraw=raw, metaonly=False, classname=cls.__name__
        )

@registry.public_archives
class removededm(Service):
    name = methods["removededm"]["title"]
    configId = "removededm"
@@ -594,6 +603,7 @@ class removededm(Service):
                    print("Login failure for removededm", j, flush = True)
                    raise RuntimeError("Login failure")

@registry.metadata
class Filmot(Service):
    name = methods["filmot"]["title"]
    lastretrieved: int = 0
@@ -627,6 +637,7 @@ class Filmot(Service):
                classname=cls.__name__
        )

@registry.metadata
class Playboard(Service):
    """
    Playboard is metadata-only as far as I know.
@@ -666,6 +677,7 @@ class Playboard(Service):
                classname=cls.__name__
        )

@registry.public_archives
class AltCensored(Service):
    """
    altCensored does not store any videos. Instead, it links to archived versions.
@@ -697,6 +709,7 @@ class AltCensored(Service):
                rawraw=None, metaonly=False, classname=cls.__name__
        )

@registry.misc
class Odysee(Service):
    """
    Queries the LBRY YouTube Sync API to find out whether the video has been mirrored to Odysee.
@@ -737,6 +750,7 @@ class Odysee(Service):
            classname=cls.__name__
        )

@registry.public_archives
class PreserveTube(Service):
    name = methods["preservetube"]["title"]
    note = ""
@@ -786,6 +800,7 @@ class PreserveTube(Service):
                rawraw=None, metaonly=metaonly, classname=cls.__name__
        )

@registry.public_archives
class NyaneOnline(Service):
    name = methods['nyaneonline']['title']
    note = ""
@@ -816,6 +831,7 @@ class NyaneOnline(Service):
                   rawraw=None, metaonly=False, classname=cls.__name__
        )

@registry.metadata
class LetsPlayIndex(Service):
    name = methods['letsplayindex']['title']
    note = ""
+43 −10
Original line number Diff line number Diff line
@@ -3,6 +3,7 @@ The classes that are used to store the response data.
"""
import copy
import dataclasses
import enum
import time
import typing_extensions as typing
import re
@@ -40,16 +41,8 @@ class FytSession:

    @classmethod
    def _get_services(cls) -> list[type['BaseService']]:
        serviceClasses = BaseService.__subclasses__()
        potentialServices = []
        for sc in serviceClasses:
            potentialServices.extend(sc.__subclasses__())
        services = []
        for potentialService in potentialServices:
            if not potentialService.enabled():
                continue
            services.append(potentialService)
        return services
        potentialServices = registry.get_services()
        return [service for service in potentialServices if service.enabled()]

    @classmethod
    async def new(cls, batching = False):
@@ -488,3 +481,43 @@ class Response(JSONDataclass):
        return string

Response.__doc__ = Response.__doc__.replace("%s", str(Response.api_version))

class ServiceCategory(enum.Enum):
    YOUTUBE = enum.auto()
    IA = enum.auto()
    MISC = enum.auto()
    PUBLIC_ARCHIVES = enum.auto()
    ON_REQUEST = enum.auto()
    METADATA = enum.auto()

class ServiceRegistry:
    services: dict[ServiceCategory, list[Service]]

    def __init__(self):
        self.services = {k: [] for k in ServiceCategory}

    def add_service(self, category: ServiceCategory, service: Service):
        self.services[category].append(service)

    def get_services(self):
        print(self.services)
        return [j for i in self.services.values() for j in i]

    def youtube(self, service: Service):
        self.add_service(ServiceCategory.YOUTUBE, service)

    def ia(self, service: Service):
        self.add_service(ServiceCategory.IA, service)

    def misc(self, service: Service):
        self.add_service(ServiceCategory.MISC, service)

    def public_archives(self, service: Service):
        self.add_service(ServiceCategory.PUBLIC_ARCHIVES, service)

    def on_request(self, service: Service):
        self.add_service(ServiceCategory.ON_REQUEST, service)

    def metadata(self, service: Service):
        self.add_service(ServiceCategory.METADATA, service)
registry = ServiceRegistry()