Commit 2b71da8d authored Jul 23, 2022 by John

Added auto source downloads

Added an option to download manga from sources and then translate them.
Currently supported websites:
+ nhentai.net
+ more to come
For more information on how to use it look into: ballontranslator\ui\pagesources\__init__.py and read help

parent 1b0a8ec0

.gitignore

+5 −0

Original line number	Diff line number	Diff line
		@@ -6,7 +6,12 @@ ballontranslator/data/models
		ballontranslator/data/testpacks/eng_dontupload
		ballontranslator/data/testpacks/testpacks
		ballontranslator/data/*.png
		ballontranslator/ui/pagesources/projects
		release
		.venv
		ballontranslator/.venv
		.idea


		tmp.py
		dummy_scripts.py

ballontranslator/ui/configpanel.py

+28 −2

Original line number	Diff line number	Diff line
		from qtpy.QtWidgets import QLayout, QHBoxLayout, QVBoxLayout, QTreeView, QWidget, QLabel, QSizePolicy, QSpacerItem, QCheckBox, QSplitter, QScrollArea, QGroupBox
		from qtpy.QtWidgets import QLayout, QHBoxLayout, QVBoxLayout, QTreeView, QWidget, QLabel, QSizePolicy, QSpacerItem, QCheckBox, QSplitter, QScrollArea, QGroupBox, QLineEdit
		from qtpy.QtCore import Qt, QModelIndex, Signal, QSize
		from qtpy.QtGui import QStandardItem, QStandardItemModel, QMouseEvent, QFont, QColor, QPalette
		from PyQt5 import QtCore
		@@ -59,6 +59,7 @@ class ConfigSubBlock(Widget):

		class ConfigBlock(Widget):
		sublock_pressed = Signal(int, int)

		def __init__(self, header: str, args, *kwargs) -> None:
		super().__init__(args, *kwargs)
		self.header = ConfigTextLabel(header, CONFIG_FONTSIZE_HEADER)
		@@ -72,7 +73,16 @@ class ConfigBlock(Widget):
		def setIndex(self, index: int):
		self.index = index

		def addTextLabel(self, text: str):
		def addTextBox(self, discription: str = None):
		if discription is not None:
		self.vlayout.addWidget(ConfigTextLabel(discription, CONFIG_FONTSIZE_CONTENT))
		textbox = QLineEdit()
		textbox.setFixedWidth(CONFIG_COMBOBOX_MIDEAN)
		textbox.setFixedHeight(45)
		self.vlayout.addWidget(textbox)
		return textbox

		def addTextLabel(self, text: str = None):
		label = ConfigTextLabel(text, CONFIG_FONTSIZE_HEADER)
		self.vlayout.addWidget(label)
		self.label_list.append(label)
		@@ -257,6 +267,7 @@ class ConfigPanel(Widget):
		label_inpaint = self.tr('Inpaint')
		label_translator = self.tr('Translator')
		label_startup = self.tr('Startup')
		label_sources = self.tr('Sources')
		label_lettering = self.tr('Lettering')

		dltableitem.appendRows([
		@@ -267,6 +278,7 @@ class ConfigPanel(Widget):
		])
		generalTableItem.appendRows([
		TableItem(label_startup, CONFIG_FONTSIZE_TABLE),
		TableItem(label_sources, CONFIG_FONTSIZE_TABLE),
		TableItem(label_lettering, CONFIG_FONTSIZE_TABLE)
		])

		@@ -290,6 +302,14 @@ class ConfigPanel(Widget):
		self.open_on_startup_checker = generalConfigPanel.addCheckBox(self.tr('Reopen last project on startup'))
		self.open_on_startup_checker.stateChanged.connect(self.on_open_onstartup_changed)

		generalConfigPanel.addTextLabel(label_sources)
		src_manual_str = self.tr('manual')
		src_nhentai_str = self.tr('nhentai')
		self.src_choice_combox = generalConfigPanel.addCombobox([src_manual_str, src_nhentai_str], self.tr('source'))
		self.src_choice_combox.currentIndexChanged.connect(self.on_source_flag_changed)
		self.src_link_textbox = generalConfigPanel.addTextBox('source url')
		self.src_link_textbox.textChanged.connect(self.on_source_link_changed)

		generalConfigPanel.addTextLabel(label_lettering)
		dec_program_str = self.tr('decide by program')
		use_global_str = self.tr('use global setting')
		@@ -340,6 +360,12 @@ class ConfigPanel(Widget):
		def on_fontcolor_flag_changed(self):
		self.config.let_fntcolor_flag = self.let_fntcolor_combox.currentIndex()

		def on_source_flag_changed(self):
		self.config.src_choice_flag = self.src_choice_combox.currentIndex()

		def on_source_link_changed(self):
		self.config.src_link_flag = self.src_link_textbox.text()

		def focusOnTranslator(self):
		idx0, idx1 = self.trans_sub_block.idx0, self.trans_sub_block.idx1
		self.configTable.setCurrentItem(idx0, idx1)

ballontranslator/ui/dl_manager.py

+24 −4

Original line number	Diff line number	Diff line
		@@ -18,9 +18,10 @@ dl.translators.SYSTEM_LANG = QLocale.system().name()
		from .stylewidgets import ProgressMessageBox
		from .configpanel import ConfigPanel
		from .misc import ProjImgTrans, DLModuleConfig
		from .pagesources import SourceBase, nhentai
		from .constants import PROGRAM_PATH

		class ModuleThread(QThread):

		exception_occurred = Signal(str, str, str)
		finish_set_module = Signal()

		@@ -109,7 +110,6 @@ class InpaintThread(ModuleThread):


		class TextDetectThread(ModuleThread):

		finish_detect_page = Signal(str)
		def __init__(self, dl_config: DLModuleConfig, args, *kwargs) -> None:
		super().__init__(dl_config, 'textdetector', TEXTDETECTORS, args, *kwargs)
		@@ -124,7 +124,6 @@ class TextDetectThread(ModuleThread):


		class OCRThread(ModuleThread):

		finish_ocr_page = Signal(str)
		def __init__(self, dl_config: DLModuleConfig, args, *kwargs) -> None:
		super().__init__(dl_config, 'ocr', OCR, args, *kwargs)
		@@ -214,7 +213,6 @@ class TranslateThread(ModuleThread):
		num_pages = len(self.imgtrans_proj.pages)
		while not self.pipeline_finished():
		if len(self.pipeline_pagekey_queue) == 0:

		time.sleep(0.1)
		continue

		@@ -397,6 +395,7 @@ class DLManager(QObject):
		super().__init__(args, *kwargs)
		self.dl_config = dl_config
		self.imgtrans_proj = imgtrans_proj
		self.config = config_panel.config

		self.textdetect_thread = TextDetectThread(self.dl_config)
		self.textdetect_thread.finish_set_module.connect(self.on_finish_setdetector)
		@@ -512,7 +511,28 @@ class DLManager(QObject):
		return
		self.inpaint_thread.inpaint(img, mask, img_key, inpaint_rect)

		def source(self):
		url = self.config.src_link_flag
		SOURCEMAP = {
		0: 'manual',
		1: 'nhentai'
		}
		src = SOURCEMAP[self.config.src_choice_flag]
		if src == 'manual':
		LOGGER.info('Source download set to manual')
		return
		elif src == 'nhentai':
		LOGGER.info('Source download set to nhentai')
		doujin = nhentai()
		LOGGER.info('Downloading images...')
		doujin.run(url)
		gallery_number = doujin.ReturnGalleryNumber()
		self.imgtrans_proj.load(rf'{PROGRAM_PATH}\ui\pagesources\projects\{gallery_number}')
		else:
		raise NotImplementedError

		def runImgtransPipeline(self):
		self.source()
		if self.imgtrans_proj.is_empty:
		LOGGER.info('proj file is empty, nothing to do')
		self.progress_msgbox.hide()

ballontranslator/ui/misc.py

+7 −1

Original line number	Diff line number	Diff line
		@@ -452,6 +452,8 @@ class ProgramConfig:
		self.let_fntsize_flag = 0
		self.let_fntstroke_flag = 0
		self.let_fntcolor_flag = 0
		self.src_choice_flag = 0
		self.src_link_flag = 0
		if config_dict is not None:
		self.load_from_dict(config_dict)

		@@ -471,6 +473,8 @@ class ProgramConfig:
		self.let_fntsize_flag = config_dict['let_fntsize_flag']
		self.let_fntstroke_flag = config_dict['let_fntstroke_flag']
		self.let_fntcolor_flag = config_dict['let_fntcolor_flag']
		self.src_choice_flag = config_dict['src_choice_flag']
		self.src_link_flag = config_dict['src_link_flag']
		except Exception as e:
		raise InvalidProgramConfigException(e)

		@@ -488,7 +492,9 @@ class ProgramConfig:
		'open_recent_on_startup': self.open_recent_on_startup,
		'let_fntsize_flag': self.let_fntsize_flag,
		'let_fntstroke_flag': self.let_fntstroke_flag,
		'let_fntcolor_flag': self.let_fntcolor_flag
		'let_fntcolor_flag': self.let_fntcolor_flag,
		'src_choice_flag': self.src_choice_flag,
		'src_link_flag': self.src_link_flag
		}

ballontranslator/ui/pagesources/init.py

0 → 100644

+83 −0

Original line number	Diff line number	Diff line
		import time
		import requests
		import re
		import urllib.request
		from .constants import SOURCE_DOWNLOAD_PATH
		from .exceptions import SourceNotImplemented
		import os

		PROXY = urllib.request.getproxies()


		class SourceBase:
		def __init__(self):
		self.link = ''
		self.SOURCEMAP = {
		0: 'manual',
		1: 'nhentai'
		}
		self.source: str = ''
		self.number_of_pages: int = 0

		def SetLink(self, link):
		self.link = link

		def CheckLink(self):
		if 'https://' not in self.link:
		self.link = 'https://' + self.link
		for v in self.SOURCEMAP.items():
		if v[1] in self.link.lower():
		self.source = v[1]
		if not self.source:
		self.link = ''
		raise SourceNotImplemented


		class nhentai(SourceBase):
		def __init__(self):
		super().__init__()
		self.gallery_number: int = 0
		def Help(self):
		print("""Please use gallery links. To get link follow these steps:
		+ Go to desired doujin
		+ click on first image
		+ right click it and select open image in new tab
		+ in the url bar you can see gallery link
		These steps are necessary as I don't know of any methods to bypass cloudflare""")

		def FetchImages(self):
		size = len(self.link)
		url = self.link[:size - 5]
		number = (re.search('galleries/(.*)/', url)).group(1)
		self.gallery_number = number
		i = 1
		path = fr'{SOURCE_DOWNLOAD_PATH}\{number}'
		if not os.path.exists(path):
		os.makedirs(path)
		while True:
		try:
		img_data = requests.get(f'{url}{i}.jpg').content
		if '404 Not Found' in str(img_data):
		break
		with open(rf'{path}\{i}.jpg', 'wb') as image:
		image.write(img_data)
		i += 1
		time.sleep(1) # Avoiding anti ddos ban
		except Exception as e:
		print(e)
		break
		self.number_of_pages = i

		def ReturnGalleryNumber(self):
		return self.gallery_number

		def run(self, url):
		self.SetLink(url)
		self.CheckLink()
		self.FetchImages()