diff --git a/src/custom.txt b/src/custom.txt index 30e4c9a..c87bc4b 100644 --- a/src/custom.txt +++ b/src/custom.txt @@ -1 +1 @@ -{"Caiyun(Token Required)": {"url": "https://fanyi.caiyunapp.com/", "key_edit": true, "secret_edit": false, "is_queue": true, "target": "caiyun.target.rst", "source": "caiyun.source.rst", "file_name": "_caiyun.py"}, "Baidu(Token Required)": {"url": "https://fanyi-api.baidu.com/", "key_edit": true, "secret_edit": true, "is_queue": false, "target": "baidu.target.rst", "source": "baidu.source.rst", "file_name": "_baidu.py"}, "Deepl(Token Required)": {"url": "https://www.deepl.com/account/?utm_source=github&utm_medium=github-python-readme", "key_edit": true, "secret_edit": false, "is_queue": true, "target": "deepl.target.rst", "source": "deepl.source.rst", "file_name": "_deepl.py"}, "OpenAI(Token Required)": {"url": "https://platform.openai.com/api-keys", "key_edit": true, "secret_edit": false, "is_queue": true, "target": "openai.target.rst", "source": "openai.source.rst", "file_name": "_openai.py"}} \ No newline at end of file +{"Caiyun(Token Required)": {"url": "https://fanyi.caiyunapp.com/", "key_edit": true, "secret_edit": false, "is_queue": true, "target": "caiyun.target.rst", "source": "caiyun.source.rst", "file_name": "_caiyun.py"}, "Baidu(Token Required)": {"url": "https://fanyi-api.baidu.com/", "key_edit": true, "secret_edit": true, "is_queue": false, "target": "baidu.target.rst", "source": "baidu.source.rst", "file_name": "_baidu.py"}, "Deepl(Token Required)": {"url": "https://www.deepl.com/account/?utm_source=github&utm_medium=github-python-readme", "key_edit": true, "secret_edit": false, "is_queue": true, "target": "deepl.target.rst", "source": "deepl.source.rst", "file_name": "_deepl.py"}, "OpenAI(Token Required)": {"url": "https://platform.openai.com/api-keys", "key_edit": true, "secret_edit": false, "is_queue": true, "target": "openai.target.rst", "source": "openai.source.rst", "file_name": "_openai.py"}, "Gemini": {"url": "", "key_edit": true, "secret_edit": false, "is_queue": true, "target": "openai.target.rst", "source": "openai.source.rst", "file_name": "_gemini.py"}} \ No newline at end of file diff --git a/src/custom_engine/_gemini.py b/src/custom_engine/_gemini.py new file mode 100644 index 0000000..fd2629b --- /dev/null +++ b/src/custom_engine/_gemini.py @@ -0,0 +1,122 @@ +import threading +import json +import os +import io +import time +import concurrent.futures +import traceback +import requests + +limit_time_span_dic = dict() +lock = threading.Lock() +count = 0 +api_key = "" +rpm = 0 +rps = 0 +tpm = 0 +model = "" +base_url = "" +proxies = None +time_out = 0 +max_length = 0 +gemini_template_file = 'openai_template.json' + + +def translate_queue(app_key, app_secret, source, target, proxies, q): + # write import inside the function , otherwise will cause NameError + + global gemini_template_file + def translate_gemini_batch(api_key_to_use, source_lang, target_lang, proxy_settings, text_list): + + global model, time_out + if not model: + model = 'gemini-2.0-flash' + if not time_out or time_out <= 0: + time_out = 120 + + gemini_api_url = f"https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent?key={api_key_to_use}" + + ori_dic = {str(i): text for i, text in enumerate(text_list)} + json_to_translate = json.dumps(ori_dic, ensure_ascii=False) + + messages = [] + try: + with io.open(gemini_template_file, 'r', encoding='utf-8') as f: + template_content = f.read() + template_content = template_content.replace('#SOURCE_LANGUAGE_ID!@$^#', source) + template_content = template_content.replace('#TARGET_LANGAUGE_ID!@$^#', target) + messages = json.loads(template_content) + except Exception as e: + print(f"Error reading or parsing {gemini_template_file}: {e}") + return None + + if not messages: + print(f'{gemini_template_file} is not a valid json template or is empty.') + return None + full_prompt_text = "" + for message in messages: + full_prompt_text += message.get("content", "") + "\n" + + full_prompt_text = full_prompt_text.replace('#JSON_DATA_WAITING_FOR_TRANSLATE_ID!@$^#', json_to_translate) + + payload = { + "contents": [ + { + "role": "user", + "parts": [ + { + "text": full_prompt_text + } + ] + } + ], + "generationConfig": { + "temperature": 0.5, + "topP": 1, + "topK": 1, + "maxOutputTokens": 8192, + "response_mime_type": "application/json" + } + } + + try: + response = requests.post( + gemini_api_url, + json=payload, + proxies=proxy_settings, + timeout=time_out + ) + response.raise_for_status() + + response_json = response.json() + content_text = response_json['candidates'][0]['content']['parts'][0]['text'] + translated_dic = json.loads(content_text) + + if len(translated_dic) != len(ori_dic): + print("Warning: Mismatch between original and translated item count.") + + l = [] + for key, translated_text in translated_dic.items(): + if key in ori_dic: + item = { + 'untranslatedText': ori_dic[key], + 'translatedText': translated_text + } + l.append(item) + + return l + + except Exception: + if 'response' in locals(): + print(response.status_code) + print(response.text) + msg = traceback.format_exc() + print(msg) + return [] + + api_key_to_use = app_key if app_key else api_key + if not api_key_to_use: + print("Gemini API key is missing.") + return [] + + return translate_gemini_batch(api_key_to_use, source, target, proxies, q) \ No newline at end of file diff --git a/src/engine.txt b/src/engine.txt new file mode 100644 index 0000000..881c785 --- /dev/null +++ b/src/engine.txt @@ -0,0 +1 @@ +{"engine": "Gemini", "key": "", "secret": "", "Google(Free)_key": "", "Google(Free)_secret": "", "Google(Free)": {"target": "Chinese(Simplified)", "source": "Auto Detect"}, "Gemini_key": "", "Gemini_secret": "", "rpm": "3", "rps": "3", "tpm": "40000", "openai_model": "gpt-3.5-turbo", "openai_base_url": "", "openai_model_index": 0, "time_out": "120", "max_length": "5000", "Gemini": {"target": "Chinese(Simplified)", "source": "Auto Detect"}, "tl": "Chinese"} \ No newline at end of file diff --git a/src/one_key_translate_form.py b/src/one_key_translate_form.py index b58bc80..7463303 100644 --- a/src/one_key_translate_form.py +++ b/src/one_key_translate_form.py @@ -18,7 +18,7 @@ from custom_engine_form import sourceDic, targetDic from my_log import log_print from renpy_translate import engineDic, language_header, translateThread, translate_threads, get_translated_dic, \ - web_brower_export_name, rpy_info_dic, get_rpy_info, web_brower_translate, engineList + web_brower_export_name, rpy_info_dic, get_rpy_info, web_brower_translate, engineList, translate_file_single from engine_form import MyEngineForm from game_unpacker_form import finish_flag from extract_runtime_form import extract_finish @@ -36,6 +36,7 @@ from error_repair_form import repairThread from translated_form import MyTranslatedForm +import concurrent.futures class MyQueue(queue.Queue): def peek(self): @@ -43,6 +44,31 @@ def peek(self): with self.mutex: return self.queue[0] +# Limit max threads using Thread Pool, default 5 +class MyTranslationPoolWorker(QThread): + finished = Signal() + progress = Signal(str) + + def __init__(self, tasks, max_workers = 5, parent = None): + super().__init__(parent) + self.tasks = tasks + self.max_workers = max_workers + + def run(self): + with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as executor: + + future_to_task = {executor.submit(translate_file_single, **task_params): task_params for task_params in self.tasks} + + for future in concurrent.futures.as_completed(future_to_task): + task_info = future_to_task[future] + try: + future.result() + self.progress.emit(f"Successfully processed: {task_info['p']}") + except Exception as exc: + self.progress.emit(f"Error processing {task_info['p']}: {exc}") + + self.finished.emit() + class MyOneKeyTranslateForm(QDialog, Ui_OneKeyTranslateDialog): def __init__(self, parent=None): @@ -102,6 +128,8 @@ def __init__(self, parent=None): self.overwriteCheckBox.setChecked(not is_skip_if_exist) _thread.start_new_thread(self.update, ()) + self.translation_pool_worker = None + def on_tl_path_changed(self): if os.path.isfile('engine.txt'): json_file = open('engine.txt', 'r',encoding='utf-8') @@ -226,46 +254,64 @@ def translate(self): target_language = targetDic[self.targetComboBox.currentText()] if self.sourceComboBox.currentText() != '': source_language = sourceDic[self.sourceComboBox.currentText()] + + # create a local thread list instead of the global one + tasks = [] for path, dir_lst, file_lst in paths: for file_name in file_lst: i = os.path.join(path, file_name) if not file_name.endswith("rpy"): continue - t = translateThread(cnt, i, target_language, source_language, - True, - False, self.local_glossary, True, - True, self.filterCheckBox_2.isChecked(), self.filterLengthLineEdit_2.text(), True) - translate_threads.append(t) + + # ------------ pack into params dict ----------------- + task_params = { + 'p': i, + 'lang_target': target_language, + 'lang_source': source_language, + 'is_gen_bak': False, + 'local_glossary': self.local_glossary, + 'is_translate_current': True, + 'is_skip_translated': True, + 'is_open_filter': self.filterCheckBox_2.isChecked(), + 'filter_length': int(self.filterLengthLineEdit_2.text()), + 'is_replace_special_symbols': True + } + + tasks.append(task_params) + cnt = cnt + 1 - if len(translate_threads) > 0: + + translate_threads.clear() + + + if len(tasks) > 0: is_finished, is_executed = self.qDic[self.translate] is_finished = False self.qDic[self.translate] = is_finished, is_executed - log_print('start translate...') - for t in translate_threads: - t.start() + log_print("Starting translation workers") + self.setDisabled(True) - _thread.start_new_thread(self.translate_threads_over, ()) + + # UI might need a update for max workers + self.translation_pool_worker = MyTranslationPoolWorker(tasks) + self.translation_pool_worker.finished.connect(self.on_translate_finished) + self.translation_pool_worker.progress.connect(log_print) + self.translation_pool_worker.start() else: is_finished, is_executed = self.qDic[self.translate] is_finished = True self.qDic[self.translate] = is_finished, is_executed + + else: is_finished, is_executed = self.qDic[self.translate] is_finished = True self.qDic[self.translate] = is_finished, is_executed - def translate_threads_over(self): - while True: - threads_len = len(translate_threads) - if threads_len > 0: - for t in translate_threads: - if t.is_alive(): - t.join() - translate_threads.remove(t) - else: - break + + def on_translate_finished(self): log_print('translate all complete!') + self.setDisabled(False) is_finished, is_executed = self.qDic[self.translate] is_finished = True self.qDic[self.translate] = is_finished, is_executed diff --git a/src/renpy_translate.py b/src/renpy_translate.py index c8da0dd..b8c3c96 100644 --- a/src/renpy_translate.py +++ b/src/renpy_translate.py @@ -101,126 +101,22 @@ def run(self): def TranslateFile(self, p, lang_target, lang_source, is_gen_bak, local_glossary, is_translate_current, is_skip_translated, is_open_filter, filter_length, is_replace_special_symbols): - global rpy_info_dic - client = init_client() - if client is None: - return - transList = [] - trans_ori_dic = [] - ret, unmatch_cnt, p = get_rpy_info(p) - if len(ret) == 0: - log_print(p + ' unable to get translated info') - rpy_info_dic.pop(p) - return - for dic in ret: - original = dic['original'] - current = dic['current'] - if is_translate_current: - target = current - else: - target = original - dic['target'] = target - if is_skip_translated and original != current: - continue - if local_glossary is not None and len(local_glossary) > 0: - for original, replace in local_glossary.items(): - target = target.replace(original, replace) - if is_replace_special_symbols: - d = EncodeBrackets(target) - else: - d = dict() - d['en_1'] = [] - d['en_2'] = [] - d['en_3'] = [] - d['encoded'] = target - strip_i = target - for j in (d['en_1']): - strip_i = strip_i.replace(j, '') - for j in (d['en_2']): - strip_i = strip_i.replace(j, '') - for j in (d['en_3']): - strip_i = strip_i.replace(j, '') - _strip_i = replace_all_blank(strip_i) - if is_open_filter: - if len(_strip_i) < filter_length: - # log_print(len(strip_i),i) - continue - if not isAllPunctuations(d['encoded'].strip('"')): - if is_replace_special_symbols: - transList.append(d['encoded'].strip('"')) - dic['d'] = d - else: - dic['d'] = None - transList.append(target) - trans_ori_dic.append((dic, d['encoded'].strip('"'))) - - if client.__class__.__name__ == 'Translate' and local_glossary is not None and len(local_glossary) > 0: - fmt = 'html' - else: - fmt = 'text' - if len(transList) == 0: - log_print(p + ' translate skip!') - rpy_info_dic.pop(p) - return - if isinstance(client, str) and client == 'web_brower': - plain_text_to_html_from_list(transList, web_brower_export_name, is_replace_special_symbols) - return - trans_dic = TranslateToList(client, transList, lang_target, lang_source, fmt=fmt) - f = io.open(p, 'r', encoding='utf-8') - _read_lines = f.readlines() - f.close() - if is_gen_bak: - f = io.open(p + '.bak', 'w', encoding='utf-8') - f.writelines(_read_lines) - f.close() - for dic, trans_key in trans_ori_dic: - line = dic['line'] - 1 - ori_line = dic['ori_line'] - 1 - original = dic['original'] - current = dic['current'] - target = dic['target'] - d = dic['d'] - if is_replace_special_symbols: - translated = get_translated(trans_dic, d) - else: - translated = trans_dic[target] - if translated is None: - translated = '' - encoded = d['encoded'].strip('"') - if encoded in trans_dic: - translated = trans_dic[encoded] - log_print(f'{p} Error in line:{str(line)}\n{target}\n{encoded}\n{translated}\nError') - else: - if target == current: - if _read_lines[line].startswith(' new '): - header = _read_lines[line][:7] - content = _read_lines[line][7:] - _read_lines[line] = header + content.replace(target, translated, 1) - else: - _read_lines[line] = _read_lines[line].replace(target, translated, 1) - else: - _read_lines[line] = ' ' + _read_lines[ori_line].replace(target, translated).lstrip().lstrip( - '#').lstrip() - if _read_lines[line].startswith(' old '): - _read_lines[line] = _read_lines[line].replace(' old ', ' new ', 1) - f = io.open(p, 'w', encoding='utf-8') - f.writelines(_read_lines) - f.close() - log_print(p + ' translate success!') - rpy_info_dic.clear() + translate_file_single(p, lang_target, lang_source, is_gen_bak, local_glossary, is_translate_current, + is_skip_translated, is_open_filter, filter_length, is_replace_special_symbols) def TranslateToList(cli, inList, lang_target, lang_source, fmt='text'): dic = dict() + if cli.__class__.__name__ != 'Translate': texts = cli.translate(inList, target=lang_target, source=lang_source) else: texts = cli.translate(inList, target=lang_target, source=lang_source, fmt=fmt) if isinstance(texts, list): for i, e in enumerate(texts): - if cli.__class__.__name__ == 'OpenAITranslate': - if hasattr(e, 'untranslatedText'): - dic[e.untranslatedText] = e.translatedText + # This should be inside api, otherwise the order will messed up + if hasattr(e, 'untranslatedText'): + dic[e.untranslatedText] = e.translatedText else: dic[inList[i]] = e.translatedText else: @@ -467,14 +363,15 @@ def get_rpy_info(p): dic['is_match'] = is_match infoList.append(dic) # sorted(infoList, key=lambda x: x['line']) - rpy_info_dic[p] = infoList, unmatch_cnt, p + # rpy_info_dic[p] = infoList, unmatch_cnt, p return infoList, unmatch_cnt, p except: f.close() msg = traceback.format_exc() log_print(msg) - rpy_info_dic[p] = infoList, 0, p - return infoList, 0, p + # rpy_info_dic[p] = infoList, 0, p + # return infoList, 0, p + return [], 0, p def get_translated_dic(html_path, translated_path): @@ -575,3 +472,120 @@ def web_brower_translate(is_open_filter, filter_length, is_current, is_replace_s f = io.open(path, 'w', encoding='utf-8') f.writelines(_read_lines) f.close() + +def translate_file_single(p, lang_target, lang_source, is_gen_bak, local_glossary, is_translate_current, + is_skip_translated, is_open_filter, filter_length, is_replace_special_symbols): + + try: + client = init_client() + if client is None: + return + transList = [] + trans_ori_dic = [] + ret, unmatch_cnt, p = get_rpy_info(p) + if len(ret) == 0: + log_print(p + ' unable to get translated info') + return + + for item_dic in ret: + original = item_dic['original'] + current = item_dic['current'] + + target_text = current if is_translate_current else original + item_dic['target'] = target_text + + if is_skip_translated and original != current: + continue + + if local_glossary: + for orig, repl in local_glossary.items(): + target_text = target_text.replace(orig, repl) + + encoded_info = None + if is_replace_special_symbols: + encoded_info = EncodeBrackets(target_text) + text_to_translate = encoded_info['encoded'].strip('"') + item_dic['d'] = encoded_info + else: + text_to_translate = target_text + item_dic['d'] = None + + _strip_i = replace_all_blank(target_text) + if is_open_filter and len(_strip_i) < filter_length: + continue + if isAllPunctuations(text_to_translate): + continue + + transList.append(text_to_translate) + trans_ori_dic.append({'item': item_dic, 'key': text_to_translate}) + + if not transList: + log_print(f"[{os.path.basename(p)}] No text to translate after filtering.") + return + + if client.__class__.__name__ == 'Translate' and local_glossary is not None and len(local_glossary) > 0: + fmt = 'html' + else: + fmt = 'text' + if len(transList) == 0: + log_print(p + ' translate skip!') + # rpy_info_dic.pop(p) + return + if isinstance(client, str) and client == 'web_brower': + plain_text_to_html_from_list(transList, web_brower_export_name, is_replace_special_symbols) + return + trans_dic = TranslateToList(client, transList, lang_target, lang_source, fmt=fmt) + f = io.open(p, 'r', encoding='utf-8') + _read_lines = f.readlines() + f.close() + if is_gen_bak: + f = io.open(p + '.bak', 'w', encoding='utf-8') + f.writelines(_read_lines) + f.close() + for task in trans_ori_dic: + item_dic = task['item'] + trans_key = task['key'] + + line = item_dic['line'] - 1 + ori_line = item_dic['ori_line'] - 1 + original = item_dic['original'] + current = item_dic['current'] + target = item_dic['target'] + d = item_dic['d'] + + if trans_key not in trans_dic: + log_print(f"[{os.path.basename(p)}] Key '{trans_key}' not found in translation results. Line: {line+1}") + continue + + translated_raw = trans_dic[trans_key] + + if is_replace_special_symbols: + translated = get_translated({'': translated_raw}, d) + translated = get_translated({trans_key: translated_raw}, d) + else: + translated = translated_raw + + if translated is None: + log_print(f"[{os.path.basename(p)}] Failed to decode translation. Original: '{target}', Translated: '{translated_raw}'. Line: {line+1}") + continue + + if target == current: + if _read_lines[line].startswith(' new '): + header = _read_lines[line][:7] + content = _read_lines[line][7:] + _read_lines[line] = header + content.replace(target, translated, 1) + else: + _read_lines[line] = _read_lines[line].replace(target, translated, 1) + else: + _read_lines[line] = ' ' + _read_lines[ori_line].replace(target, translated).lstrip().lstrip('#').lstrip() + if _read_lines[line].startswith(' old '): + _read_lines[line] = _read_lines[line].replace(' old ', ' new ', 1) + + with io.open(p, 'w', encoding='utf-8') as f: + f.writelines(_read_lines) + log_print(p + ' translate success!') + rpy_info_dic.clear() + except Exception: + log_print(f"Error processing {p}:\n{traceback.format_exc()}") + + \ No newline at end of file diff --git a/src/requirements.txt b/src/requirements.txt index 80193d5..2ec267b 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -10,4 +10,5 @@ openpyxl==3.1.5 qt-material==2.14 beautifulsoup4==4.12.3 pywin32==306 -ping3==4.0.8 \ No newline at end of file +ping3==4.0.8 +requests==2.32.4 \ No newline at end of file