Commit 2b71da8d authored by John's avatar John
Browse files

Added auto source downloads

Added an option to download manga from sources and then translate them.
Currently supported websites:
+ nhentai.net
+ more to come
For more information on how to use it look into: ballontranslator\ui\pagesources\__init__.py and read help
parent 1b0a8ec0
Loading
Loading
Loading
Loading
+5 −0
Original line number Diff line number Diff line
@@ -6,7 +6,12 @@ ballontranslator/data/models
ballontranslator/data/testpacks/eng_dontupload
ballontranslator/data/testpacks/testpacks
ballontranslator/data/*.png
ballontranslator/ui/pagesources/projects
release
.venv
ballontranslator/.venv
.idea


tmp.py
dummy_scripts.py
+28 −2
Original line number Diff line number Diff line
from qtpy.QtWidgets import QLayout, QHBoxLayout, QVBoxLayout, QTreeView, QWidget, QLabel, QSizePolicy, QSpacerItem, QCheckBox, QSplitter, QScrollArea, QGroupBox
from qtpy.QtWidgets import QLayout, QHBoxLayout, QVBoxLayout, QTreeView, QWidget, QLabel, QSizePolicy, QSpacerItem, QCheckBox, QSplitter, QScrollArea, QGroupBox, QLineEdit
from qtpy.QtCore import Qt, QModelIndex, Signal, QSize
from qtpy.QtGui import QStandardItem, QStandardItemModel, QMouseEvent, QFont, QColor, QPalette
from PyQt5 import QtCore
@@ -59,6 +59,7 @@ class ConfigSubBlock(Widget):

class ConfigBlock(Widget):
    sublock_pressed = Signal(int, int)

    def __init__(self, header: str, *args, **kwargs) -> None:
        super().__init__(*args, **kwargs)
        self.header = ConfigTextLabel(header, CONFIG_FONTSIZE_HEADER)
@@ -72,7 +73,16 @@ class ConfigBlock(Widget):
    def setIndex(self, index: int):
        self.index = index

    def addTextLabel(self, text: str):
    def addTextBox(self, discription: str = None):
        if discription is not None:
            self.vlayout.addWidget(ConfigTextLabel(discription, CONFIG_FONTSIZE_CONTENT))
        textbox = QLineEdit()
        textbox.setFixedWidth(CONFIG_COMBOBOX_MIDEAN)
        textbox.setFixedHeight(45)
        self.vlayout.addWidget(textbox)
        return textbox

    def addTextLabel(self, text: str = None):
        label = ConfigTextLabel(text, CONFIG_FONTSIZE_HEADER)
        self.vlayout.addWidget(label)
        self.label_list.append(label)
@@ -257,6 +267,7 @@ class ConfigPanel(Widget):
        label_inpaint = self.tr('Inpaint')
        label_translator = self.tr('Translator')
        label_startup = self.tr('Startup')
        label_sources = self.tr('Sources')
        label_lettering = self.tr('Lettering')
    
        dltableitem.appendRows([
@@ -267,6 +278,7 @@ class ConfigPanel(Widget):
        ])
        generalTableItem.appendRows([
            TableItem(label_startup, CONFIG_FONTSIZE_TABLE),
            TableItem(label_sources, CONFIG_FONTSIZE_TABLE),
            TableItem(label_lettering, CONFIG_FONTSIZE_TABLE)
        ])

@@ -290,6 +302,14 @@ class ConfigPanel(Widget):
        self.open_on_startup_checker = generalConfigPanel.addCheckBox(self.tr('Reopen last project on startup'))
        self.open_on_startup_checker.stateChanged.connect(self.on_open_onstartup_changed)

        generalConfigPanel.addTextLabel(label_sources)
        src_manual_str = self.tr('manual')
        src_nhentai_str = self.tr('nhentai')
        self.src_choice_combox = generalConfigPanel.addCombobox([src_manual_str, src_nhentai_str], self.tr('source'))
        self.src_choice_combox.currentIndexChanged.connect(self.on_source_flag_changed)
        self.src_link_textbox = generalConfigPanel.addTextBox('source url')
        self.src_link_textbox.textChanged.connect(self.on_source_link_changed)

        generalConfigPanel.addTextLabel(label_lettering)
        dec_program_str = self.tr('decide by program')
        use_global_str = self.tr('use global setting')
@@ -340,6 +360,12 @@ class ConfigPanel(Widget):
    def on_fontcolor_flag_changed(self):
        self.config.let_fntcolor_flag = self.let_fntcolor_combox.currentIndex()

    def on_source_flag_changed(self):
        self.config.src_choice_flag = self.src_choice_combox.currentIndex()

    def on_source_link_changed(self):
        self.config.src_link_flag = self.src_link_textbox.text()

    def focusOnTranslator(self):
        idx0, idx1 = self.trans_sub_block.idx0, self.trans_sub_block.idx1
        self.configTable.setCurrentItem(idx0, idx1)
+24 −4
Original line number Diff line number Diff line
@@ -18,9 +18,10 @@ dl.translators.SYSTEM_LANG = QLocale.system().name()
from .stylewidgets import ProgressMessageBox
from .configpanel import ConfigPanel
from .misc import ProjImgTrans, DLModuleConfig
from .pagesources import SourceBase, nhentai
from .constants import PROGRAM_PATH

class ModuleThread(QThread):

    exception_occurred = Signal(str, str, str)
    finish_set_module = Signal()

@@ -109,7 +110,6 @@ class InpaintThread(ModuleThread):


class TextDetectThread(ModuleThread):
    
    finish_detect_page = Signal(str)
    def __init__(self, dl_config: DLModuleConfig, *args, **kwargs) -> None:
        super().__init__(dl_config, 'textdetector', TEXTDETECTORS, *args, **kwargs)
@@ -124,7 +124,6 @@ class TextDetectThread(ModuleThread):


class OCRThread(ModuleThread):

    finish_ocr_page = Signal(str)
    def __init__(self, dl_config: DLModuleConfig, *args, **kwargs) -> None:
        super().__init__(dl_config, 'ocr', OCR, *args, **kwargs)
@@ -214,7 +213,6 @@ class TranslateThread(ModuleThread):
        num_pages = len(self.imgtrans_proj.pages)
        while not self.pipeline_finished():
            if len(self.pipeline_pagekey_queue) == 0:
                
                time.sleep(0.1)
                continue
            
@@ -397,6 +395,7 @@ class DLManager(QObject):
        super().__init__(*args, **kwargs)
        self.dl_config = dl_config
        self.imgtrans_proj = imgtrans_proj
        self.config = config_panel.config

        self.textdetect_thread = TextDetectThread(self.dl_config)
        self.textdetect_thread.finish_set_module.connect(self.on_finish_setdetector)
@@ -512,7 +511,28 @@ class DLManager(QObject):
            return
        self.inpaint_thread.inpaint(img, mask, img_key, inpaint_rect)

    def source(self):
        url = self.config.src_link_flag
        SOURCEMAP = {
            0: 'manual',
            1: 'nhentai'
        }
        src = SOURCEMAP[self.config.src_choice_flag]
        if src == 'manual':
            LOGGER.info('Source download set to manual')
            return
        elif src == 'nhentai':
            LOGGER.info('Source download set to nhentai')
            doujin = nhentai()
            LOGGER.info('Downloading images...')
            doujin.run(url)
            gallery_number = doujin.ReturnGalleryNumber()
            self.imgtrans_proj.load(rf'{PROGRAM_PATH}\ui\pagesources\projects\{gallery_number}')
        else:
            raise NotImplementedError

    def runImgtransPipeline(self):
        self.source()
        if self.imgtrans_proj.is_empty:
            LOGGER.info('proj file is empty, nothing to do')
            self.progress_msgbox.hide()
+7 −1
Original line number Diff line number Diff line
@@ -452,6 +452,8 @@ class ProgramConfig:
        self.let_fntsize_flag = 0
        self.let_fntstroke_flag = 0
        self.let_fntcolor_flag = 0
        self.src_choice_flag = 0
        self.src_link_flag = 0
        if config_dict is not None:
            self.load_from_dict(config_dict)

@@ -471,6 +473,8 @@ class ProgramConfig:
            self.let_fntsize_flag = config_dict['let_fntsize_flag']
            self.let_fntstroke_flag = config_dict['let_fntstroke_flag']
            self.let_fntcolor_flag = config_dict['let_fntcolor_flag']
            self.src_choice_flag = config_dict['src_choice_flag']
            self.src_link_flag = config_dict['src_link_flag']
        except Exception as e:
            raise InvalidProgramConfigException(e)

@@ -488,7 +492,9 @@ class ProgramConfig:
            'open_recent_on_startup': self.open_recent_on_startup,
            'let_fntsize_flag': self.let_fntsize_flag,
            'let_fntstroke_flag': self.let_fntstroke_flag,
            'let_fntcolor_flag': self.let_fntcolor_flag
            'let_fntcolor_flag': self.let_fntcolor_flag,
            'src_choice_flag': self.src_choice_flag,
            'src_link_flag': self.src_link_flag
        }


+83 −0
Original line number Diff line number Diff line
import time
import requests
import re
import urllib.request
from .constants import SOURCE_DOWNLOAD_PATH
from .exceptions import SourceNotImplemented
import os

PROXY = urllib.request.getproxies()


class SourceBase:
    def __init__(self):
        self.link = ''
        self.SOURCEMAP = {
            0: 'manual',
            1: 'nhentai'
        }
        self.source: str = ''
        self.number_of_pages: int = 0

    def SetLink(self, link):
        self.link = link

    def CheckLink(self):
        if 'https://' not in self.link:
            self.link = 'https://' + self.link
        for v in self.SOURCEMAP.items():
            if v[1] in self.link.lower():
                self.source = v[1]
        if not self.source:
            self.link = ''
            raise SourceNotImplemented


class nhentai(SourceBase):
    def __init__(self):
        super().__init__()
        self.gallery_number: int = 0
    def Help(self):
        print("""Please use gallery links. To get link follow these steps:
        + Go to desired doujin
        + click on first image
        + right click it and select open image in new tab
        + in the url bar you can see gallery link
        These steps are necessary as I don't know of any methods to bypass cloudflare""")

    def FetchImages(self):
        size = len(self.link)
        url = self.link[:size - 5]
        number = (re.search('galleries/(.*)/', url)).group(1)
        self.gallery_number = number
        i = 1
        path = fr'{SOURCE_DOWNLOAD_PATH}\{number}'
        if not os.path.exists(path):
            os.makedirs(path)
        while True:
            try:
                img_data = requests.get(f'{url}{i}.jpg').content
                if '404 Not Found' in str(img_data):
                    break
                with open(rf'{path}\{i}.jpg', 'wb') as image:
                    image.write(img_data)
                i += 1
                time.sleep(1)  # Avoiding anti ddos ban
            except Exception as e:
                print(e)
                break
        self.number_of_pages = i

    def ReturnGalleryNumber(self):
        return self.gallery_number

    def run(self, url):
        self.SetLink(url)
        self.CheckLink()
        self.FetchImages()





Loading