Set up Pylint

[MASTER]

# Pickle collected data for later comparisons

persistent=yes

# List of plugins (as comma separated values of python modules names) to load,

# usually to register additional checkers.

load-plugins=

# Use multiple processes to speed up Pylint.

jobs=4

# Allow loading of arbitrary C extensions. Extensions are imported into the

# active Python interpreter and may run arbitrary code.

unsafe-load-any-extension=no

# A comma-separated list of package or module names from where C extensions may

# be loaded. Extensions are loading into the active Python interpreter and may

# run arbitrary code

extension-pkg-whitelist=

[MESSAGES CONTROL]

# Only show warnings with the listed confidence levels. Leave empty to show

# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED

confidence=

# Disable the message, report, category or checker with the given id(s). You

# can either give multiple identifiers separated by comma (,) or put this

# option multiple times (only on the command line, not in the configuration

# file where it should appear only once).You can also use "--disable=all" to

# disable everything first and then reenable specific checks. For example, if

# you want to run only the similarities checker, you can use "--disable=all

# --enable=similarities". If you want to run only the classes checker, but have

# no Warning level messages displayed, use"--disable=all --enable=classes

# --disable=W"

disable=redefined-builtin,line-too-long

enable=

[REPORTS]

# Set the output format. Available formats are text, parseable, colorized, msvs

# (visual studio) and html. You can also give a reporter class, eg

# mypackage.mymodule.MyReporterClass.

output-format=text

# Put messages in a separate file for each module / package specified on the

# command line instead of printing them on stdout. Reports (if any) will be

# written in a file name "pylint_global.[txt|html]". This option is deprecated

# and it will be removed in Pylint 2.0.

files-output=no

# Tells whether to display a full report or only the messages

reports=no

# Python expression which should return a note less than 10 (10 is the highest

# note). You have access to the variables errors warning, statement which

# respectively contain the number of errors / warnings messages and the total

# number of statements analyzed. This is used by the global evaluation report

# (RP0004).

evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)

# Template used to display messages. This is a python new-style format string

# used to format the message information. See doc for all details

#msg-template=

[BASIC]

# Good variable names which should always be accepted, separated by a comma

# good-names=i,j,k,_,id

# Bad variable names which should always be refused, separated by a comma

bad-names=foo,bar,baz,toto,tutu,tata,stuff,myvar,var,variable,thing,thingy,sus,su,item,lol,data

# Colon-delimited sets of names that determine each other's naming style when

# the name regexes allow several styles.

name-group=

# Include a hint for the correct naming format with invalid-name

include-naming-hint=true

# List of decorators that produce properties, such as abc.abstractproperty. Add

# to this list to register other decorators that produce valid properties.

property-classes=abc.abstractproperty

# Regular expression matching correct function names

function-rgx=[a-z_][a-z0-9_]{2,30}$

# Naming hint for function names

function-name-hint=[a-z_][a-z0-9_]{2,30}$

# Regular expression matching correct variable names

variable-rgx=[a-z_][a-zA-Z0-9_]{2,30}$

# Naming hint for variable names

variable-name-hint=[a-z_][a-z0-9_]{2,30}$

# Regular expression matching correct constant names

const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$

# Naming hint for constant names

const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$

# Regular expression matching correct attribute names

attr-rgx=[a-z_][a-z0-9A-Z_]{1,30}$

# Naming hint for attribute names

attr-name-hint=[a-z_][a-z0-9_]{2,30}$

# Regular expression matching correct argument names

argument-rgx=[a-z_][a-zA-Z0-9_]{1,30}$

# Naming hint for argument names

argument-name-hint=[a-z_][a-z0-9_]{2,30}$

# Regular expression matching correct class attribute names

class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{1,30}|(__.*__))$

# Naming hint for class attribute names

class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$

# Regular expression matching correct inline iteration names

inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$

# Naming hint for inline iteration names

inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$

# Regular expression matching correct class names

class-rgx=[A-Z_][a-zA-Z0-9]+$

# Naming hint for class names

class-name-hint=[A-Z_][a-zA-Z0-9]+$

# Regular expression matching correct module names

module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$

# Naming hint for module names

module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$

# Regular expression matching correct method names

method-rgx=[a-z_][a-zA-Z0-9_]{2,30}$

# Naming hint for method names

method-name-hint=[a-z_][a-z0-9_]{2,30}$

# Regular expression which should only match function or class names that do

# not require a docstring.

no-docstring-rgx=^_

# Minimum line length for functions/classes that require docstrings, shorter

# ones are exempt.

docstring-min-length=-1

[ELIF]

# Maximum number of nested blocks for function / method body

max-nested-blocks=5

[FORMAT]

# Maximum number of characters on a single line.

max-line-length=159

# Regexp for a line that is allowed to be longer than the limit.

ignore-long-lines=^\s*(# )?<?https?://\S+>?$

# Allow the body of an if to be on the same line as the test if there is no

# else.

single-line-if-stmt=no

# List of optional constructs for which whitespace checking is disabled. `dict-

# separator` is used to allow tabulation in dicts, etc.: {1  : 1,\n222: 2}.

# `trailing-comma` allows a space between comma and closing bracket: (a, ).

# `empty-line` allows space-only lines.

no-space-check=trailing-comma,dict-separator

# Maximum number of lines in a module

max-module-lines=1000

# String used as indentation unit. This is usually "    " (4 spaces) or "\t" (1

# tab).

indent-string='    '

# Number of spaces of indent required inside a hanging  or continued line.

indent-after-paren=4

# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.

expected-line-ending-format=

[LOGGING]

# Logging modules to check that the string format arguments are in logging

# function parameter format

logging-modules=logging

[MISCELLANEOUS]

# List of note tags to take in consideration, separated by a comma.

notes=FIXME,XXX,TODO

[SIMILARITIES]

# Minimum lines number of a similarity.

min-similarity-lines=4

# Ignore comments when computing similarities.

ignore-comments=yes

# Ignore docstrings when computing similarities.

ignore-docstrings=yes

# Ignore imports when computing similarities.

ignore-imports=no

[SPELLING]

# Spelling dictionary name. To make it working

# install python-enchant package.

spelling-dict=

# List of comma separated words that should not be checked.

spelling-ignore-words=

# A path to a file that contains private dictionary; one word per line.

spelling-private-dict-file=

# Tells whether to store unknown words to indicated private dictionary in

# --spelling-private-dict-file option instead of raising a message.

spelling-store-unknown-words=no

[TYPECHECK]

# Tells whether missing members accessed in mixin class should be ignored. A

# mixin class is detected if its name ends with "mixin" (case insensitive).

ignore-mixin-members=yes

# List of module names for which member attributes should not be checked

# (useful for modules/projects where namespaces are manipulated during runtime

# and thus existing member attributes cannot be deduced by static analysis. It

# supports qualified module names, as well as Unix pattern matching.

ignored-modules=

# List of class names for which member attributes should not be checked (useful

# for classes with dynamically set attributes). This supports the use of

# qualified names.

ignored-classes=optparse.Values,thread._local,_thread._local

# List of members which are set dynamically and missed by pylint inference

# system, and so shouldn't trigger E1101 when accessed. Python regular

# expressions are accepted.

generated-members=

# List of decorators that produce context managers, such as

# contextlib.contextmanager. Add to this list to register other decorators that

# produce valid context managers.

contextmanager-decorators=contextlib.contextmanager

[VARIABLES]

# Tells whether we should check for unused import in __init__ files.

init-import=no

# A regular expression matching the name of dummy variables (i.e. expectedly

# not used).

dummy-variables-rgx=(_+[a-zA-Z0-9]*?$)|dummy

# List of additional names supposed to be defined in builtins. Remember that

# you should avoid to define new builtins when possible.

additional-builtins=

# List of strings which can identify a callback function by name. A callback

# name must start or end with one of those strings.

# callbacks=cb_,_cb

# List of qualified module names which can have objects that can redefine

# builtins.

redefining-builtins-modules=six.moves,future.builtins

[CLASSES]

# List of method names used to declare (i.e. assign) instance attributes.

defining-attr-methods=__init__,__new__,setUp

# List of valid names for the first argument in a class method.

valid-classmethod-first-arg=cls

# List of valid names for the first argument in a metaclass class method.

valid-metaclass-classmethod-first-arg=mcs

# List of member names, which should be excluded from the protected access

# warning.

exclude-protected=_asdict,_fields,_replace,_source,_make

[DESIGN]

# Maximum number of arguments for function / method

max-args=5

# Argument names that match this expression will be ignored. Default to name

# with leading underscore

ignored-argument-names=_.*

# Maximum number of locals for function / method body

max-locals=15

# Maximum number of return / yield for function / method body

max-returns=6

# Maximum number of branch for function / method body

max-branches=12

# Maximum number of statements in function / method body

max-statements=60

# Maximum number of parents for a class (see R0901).

max-parents=7

# Maximum number of attributes for a class (see R0902).

max-attributes=15

# Minimum number of public methods for a class (see R0903).

min-public-methods=2

# Maximum number of public methods for a class (see R0904).

max-public-methods=20

# Maximum number of boolean expressions in a if statement

max-bool-expr=5

[IMPORTS]

# Deprecated modules which should not be used, separated by a comma

deprecated-modules=regsub,TERMIOS,Bastion,rexec

# Create a graph of every (i.e. internal and external) dependencies in the

# given file (report RP0402 must not be disabled)

import-graph=

# Create a graph of external dependencies in the given file (report RP0402 must

# not be disabled)

ext-import-graph=

# Create a graph of internal dependencies in the given file (report RP0402 must

# not be disabled)

int-import-graph=

# Force import order to recognize a module as part of the standard

# compatibility libraries.

known-standard-library=

# Force import order to recognize a module as part of a third party library.

known-third-party=enchant

# Analyse import fallback blocks. This can be used to support both Python 2 and

# 3 compatible code, which means that the block might have code that exists

# only in one or another interpreter, leading to false positives when analysed.

analyse-fallback-blocks=no

[EXCEPTIONS]

# Exceptions that will emit a warning when being caught. Defaults to

# "Exception"

overgeneral-exceptions=Exception,BaseException

 No newline at end of file

"""

LostMediaFinder

"""

from .types import *

from .finder import *

import sys

"""

The CLI interface of LostMediaFinder.

"""

from . import Response

import sys

def main() -> int:

    """

    Parses CLI arguments and returns the Response.

    """

    print("Using LostMediaFinder from the command-line is not yet supported.")

    return 255

from snscrape.base import _JSONDataclass as JSONDataclass

"""

All the Service implementations live here.

"""

import config

import dataclasses

import time

import requests

import urllib.parse

from switch import Switch

import requests

from requests.auth import HTTPBasicAuth

from switch import Switch

from .types import *

import config

from .types import Service, T

class WaybackMachine(Service):

    """

    Queries the Wayback Machine for the video you requested.

    """

    name = "Wayback Machine"

    @classmethod

    def _run(cls, id) -> T:

    def _run(cls, id, includeRaw=True) -> T:

        ismeta = False

        lien = f"https://web.archive.org/web/2oe_/http://wayback-fakeurl.archive.org/yt/{id}"

        response = requests.get(lien, allow_redirects=False, timeout=15)

        archived = True if response.headers.get("location") else False

        archived = bool(response.headers.get("location")) # if there's a redirect, it's archived

        response2 = None

        if not archived:

            lien = None

                ismeta = True

                lien = response2["archived_snapshots"]["closest"]["url"]

        rawraw = (response.headers.get("location"), response2)

        rawraw = (response.headers.get("location"), response2) if includeRaw else None

        return cls(

                archived=archived, capcount=int(archived), rawraw=rawraw,

                available=lien, lastupdated=time.time(), name=cls.getName(),

        )

class InternetArchive(Service):

    """

    Queries the Internet Archive for the video you requested.

    """

    name = "Internet Archive/archive.org"

    items_tried = [

        "youtube-%s",

    ]

    @classmethod

    def _run(cls, id) -> T:

        datas = []

    def _run(cls, id, includeRaw=True) -> T:

        responses = []

        is_dark = False

        for template in cls.items_tried:

            ident = template % id

            data = requests.get(f"https://archive.org/metadata/{ident}", timeout=12).json()

            datas.append(data)

            if data.get("is_dark"):

            metadata = requests.get(f"https://archive.org/metadata/{ident}", timeout=12).json()

            responses.append(metadata)

            if metadata.get("is_dark"):

                is_dark = True

            if data and (not data.get("is_dark")):

            if metadata and (not metadata.get("is_dark")):

                is_dark = False

                break

        archived = bool(data)

        rawraw = datas

        archived = bool(metadata)

        rawraw = responses if includeRaw else None

        lien = f"https://archive.org/details/{ident}" if archived else None

        note = ""

        if not archived:

            note = "Even if it isn't found here, it might still be in the Internet Archive. This site only checks for certain item identifiers."

            if is_dark:

                note = "This item is currently unavailable to the general public.<br>" + note

                note = "An item was found, but it is currently unavailable to the general public.<br>" + note

        capcount = int(archived)

        return cls(

            archived=archived, capcount=capcount, available=lien, lastupdated=time.time(), name=cls.getName(), note=note,

        )

class GhostArchive(Service):

    """

    Queries GhostArchive for the video you requested.

    """

    @classmethod

    def _run(cls, id) -> T:

    def _run(cls, id, includeRaw=True) -> T:

        link = f"https://ghostarchive.org/varchive/{id}"

        code = requests.get(link).status_code

        archived: bool = False

        rawraw = code if includeRaw else None

        archived = None

        with Switch(code) as case:

            if case(200):

                archived = True

            if case(404):

            elif case(404):

                archived = False

            if case.default:

            elif case.default:

                raise AssertionError(f"bad status code (expected one of (200, 404), got {code})")

            else:

                raise RuntimeError("We should never be here!")

        capcount = int(archived)

        available = link if archived else None

        lastupdated = time.time()

        return cls(

            archived=archived, available=available, capcount=capcount, lastupdated=lastupdated, name=cls.getName(), note="", rawraw=code,

            archived=archived, available=available, capcount=capcount, lastupdated=lastupdated, name=cls.getName(), note="", rawraw=rawraw,

            metaonly=False, comments=False

        )

class Ya(Service):

    """

    Queries #youtubearchive for the video you requested.

    """

    name = "#youtubearchive"

    note = ("To retrieve a video from #youtubearchive, join #youtubearchive on hackint IRC and ask for help. "

        "Remember <a href='https://wiki.archiveteam.org/index.php/Archiveteam:IRC#How_do_I_chat_on_IRC?'>IRC etiquette</a>!"

    password = config.ya.password

    @classmethod

    def _run(cls, id):

    def _run(cls, id, includeRaw=True):

        vid = id

        assert cls.enabled, "#youtubearchive API access is not enabled"

        auth = HTTPBasicAuth(cls.username, cls.password)

        comments = False

        data = requests.get("https://ya.borg.xyz/cgi-bin/capture-count?v=" + vid, auth=auth, timeout=5).text

        if not data:

        count = requests.get("https://ya.borg.xyz/cgi-bin/capture-count?v=" + vid, auth=auth, timeout=5).text

        if not count:

            raise ValueError("Server returned empty response!")

        commentcount = requests.get("https://ya.borg.xyz/cgi-bin/capture-comment-counts?v="+vid, auth=auth).text

        count = int(data)

        count = int(count)

        archived = (count > 0)

        comments = [i for i in commentcount.split("\n") if i.strip("∅\n") and i.strip() != "0"]

        rawraw = (data, commentcount)

        rawraw = (count, commentcount) if includeRaw else None

        return cls(

            archived=archived, capcount=count, comments=(len(comments) > 0), lastupdated=time.time(), name=cls.getName(),

            note=cls.note if archived else "", rawraw=rawraw, metaonly=False

        )

class Filmot(Service):

    """

    Queries Filmot for the video you requested.

    """

    key = config.filmot.key

    enabled = getattr(config.filmot, "enabled", False)

    cooldown: int = 2

    @classmethod

    def _run(cls, id) -> T:

    def _run(cls, id, includeRaw=True) -> T:

        while time.time() - cls.lastretrieved < cls.cooldown:

            time.sleep(0.1)

        cls.lastretrieved = time.time()

        lastupdated = time.time()

        assert cls.enabled, "Filmot API access is not enabled."

        data = requests.get(f"https://filmot.com/api/getvideos?key={cls.key}&id={id}&flags=1").json()

        metadata = requests.get(f"https://filmot.com/api/getvideos?key={cls.key}&id={id}&flags=1").json()

        rawraw = metadata if includeRaw else None

        if len(metadata) > 0: # pylint: disable=simplifiable-if-statement

            archived = True

        if not data:

        else:

            archived = False

        capcount = int(archived)

        available = f"https://filmot.com/video/{id}" if archived else None

        return cls(

                archived=archived, capcount=capcount, error=False,

                lastupdated=lastupdated, name=cls.getName(), note="",

                rawraw=data, metaonly=True, comments=False,

                rawraw=rawraw, metaonly=True, comments=False,

                available=available

        )

from snscrape.base import _JSONDataclass as JSONDataclass

"""

The classes that are used to store the response data.

"""

import asyncio as aio

import cachetools.func

import dataclasses

import time

import typing

import re

import urllib.parse

import cachetools.func

import nest_asyncio

nest_asyncio.apply()

from snscrape.base import _JSONDataclass as JSONDataclass

T = typing.TypeVar("T", bound="Service")

nest_asyncio.apply()

T = typing.TypeVar("T", bound="Service") # pylint: disable=invalid-name

# (this name is fine)

@dataclasses.dataclass

class Service(JSONDataclass):

    error: bool = False

    @classmethod

    def _run(cls, id) -> T:

    def _run(cls, id, includeRaw=True) -> T:

        raise NotImplementedError("Subclass Service and impl the _run function")

    """

    Do not use this function!

    """

    @classmethod

    def Run(cls, id, includeRaw=True) -> T:

    def __run(cls, id, includeRaw=True) -> T:

        try:

            return cls._run(id)

        except Exception as ename:

            return cls._run(id, includeRaw=includeRaw)

        except Exception as ename: # pylint: disable=broad-except

            note = f"An error occured while retrieving data from {cls.getName()}."

            print(ename)

            rawraw = f"{type(ename)}{repr(ename)}" if includeRaw else None

            return cls(

                    archived=False, capcount=0, error=True,

                    lastupdated=time.time(), name=cls.getName(), note=note,

                    rawraw=str(ename), metaonly=False, comments=False,

                    rawraw=rawraw, metaonly=False, comments=False,

                    available=None

            )

    """

    Retrieves the data from the service.

    Arguments:

        id (str): The video ID.

        includeRaw (bool): Whether or not to include the raw data as sent from the service. If you don't need this data, turn this off; it's only the default for compatibility.

    """

    @classmethod

    # cache has a max of 128 items; items are cached for 600 seconds (10min)

    # important settings:

    # might add this to config.py later

    @cachetools.func.ttl_cache

    def run(cls, id: str, includeRaw=True):

        return cls.Run(id, includeRaw)

        """

    Runs cls.run(...) but it's async.

        Retrieves the data from the service.

        Arguments:

            id (str): The video ID.

            includeRaw (bool): Whether or not to include the raw data as sent from the service. If you don't need this data, turn this off; it's only the default for compatibility.

        """

        return cls.__run(id, includeRaw)

    @classmethod

    async def runAsync(cls, id, includeRaw=True):

        """

        Runs cls.run(...) but it's async.

        This currently still uses blocking networking (requests)!

        """

        return cls.run(id, includeRaw)

    @classmethod

    def getName(cls) -> str:

        """

        Gets the name of the service.

        """

    @classmethod

    def getName(cls) -> str:

        return getattr(cls, "name", cls.__name__)

    def __str__(self):

        lien = f"\n  Link: {self.available}" if self.available else ""

        m = "(metadata only)" if self.metaonly else ""

        m = m + " (incl. comments)" if self.comments else m

        meta = "(metadata only)" if self.metaonly else ""

        meta = meta + " (incl. comments)" if self.comments else meta

        string = f"""- Service Name: {self.name}

  Archived? {self.archived} {m} {lien}

  Archived? {self.archived} {meta} {lien}

  \t{self.note.strip()}

"""

        return string

    def _get_services(cls):

        return Service.__subclasses__()

    @classmethod

    def generate(cls, id, asyncio=False):

        """

        Runs all the Services.

        Arguments:

            id: The video ID

            asyncio: Whether or not to use asyncio.run_until_complete; this is implied if you use generateAsync

        """

    @classmethod

    def generate(cls, id, asyncio=False):

        if not re.match(r"^[A-Za-z0-9_-]{10}[AEIMQUYcgkosw048]$", id):

            return cls(status="bad.id", id=id, keys=[])

        keys = []

        services = cls._get_services()

        for subclass in services:

            data = None

            result = None

            if asyncio:

                data = aio.get_event_loop().run_until_complete(subclass.runAsync(id))

                result = aio.get_event_loop().run_until_complete(subclass.runAsync(id))

            else:

                data = subclass.run(id)

            keys.append(data)

                result = subclass.run(id)

            keys.append(result)

        return cls(id=id, status="ok", keys=keys)

    @classmethod

    async def generateAsync(cls, *args, **kwargs):

        """

        Runs all the Services asynchronously.

        """

    @classmethod

    async def generateAsync(cls, *args, **kwargs):

        kwargs['asyncio'] = True

        return cls.generate(*args, **kwargs)