Compare commits
7 Commits
9437d5ed7a
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
af0934ea5a | ||
|
|
8efebfb5ed | ||
|
|
b30c8de0b4 | ||
|
|
04fd4cef89 | ||
|
|
977bb422c0 | ||
|
|
9299020738 | ||
|
|
5c78c2dad6 |
10
README.md
10
README.md
@@ -14,9 +14,17 @@ creates anki hsk decks from a list of words
|
|||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
```console
|
```console
|
||||||
git clone https://github.com/resemble-ai/chatterbox
|
install qwen models https://github.com/QwenLM/Qwen3-TTS
|
||||||
|
modelscope download --model Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice --local_dir ./Qwen3-TTS-12Hz-0.6B-CustomVoice
|
||||||
git clone https://gitea.wolfang.info.ve/wolfang/anki-hsk-creator
|
git clone https://gitea.wolfang.info.ve/wolfang/anki-hsk-creator
|
||||||
|
install cuda 13.2 https://developer.nvidia.com/cuda-downloads
|
||||||
|
install pythorch for cuda 13.2 https://pytorch.org/get-started/locally/
|
||||||
git clone https://gitea.wolfang.info.ve/wolfang/anki-hsk-creator-data
|
git clone https://gitea.wolfang.info.ve/wolfang/anki-hsk-creator-data
|
||||||
|
set local .env file
|
||||||
|
HF_TOKEN =
|
||||||
|
DATA_FOLDER =
|
||||||
|
GWEN_FOLDER =
|
||||||
|
ARGOS_DEVICE_TYPE = auto
|
||||||
```
|
```
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|||||||
@@ -3,4 +3,4 @@
|
|||||||
# SPDX-FileCopyrightText: 2026-present Wolfang Torres <wolfang.torres@gmail.com>
|
# SPDX-FileCopyrightText: 2026-present Wolfang Torres <wolfang.torres@gmail.com>
|
||||||
#
|
#
|
||||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||||
__version__ = "0.2.1"
|
__version__ = "0.2.4"
|
||||||
|
|||||||
@@ -9,13 +9,21 @@ from .api import (
|
|||||||
is_file,
|
is_file,
|
||||||
list_input_files,
|
list_input_files,
|
||||||
pre_process_a_dictionary_file,
|
pre_process_a_dictionary_file,
|
||||||
|
proccess_a_completed_file,
|
||||||
process_a_dictation_file,
|
process_a_dictation_file,
|
||||||
process_a_dictionary_file,
|
process_a_dictionary_file,
|
||||||
process_a_phrases_file,
|
process_a_phrases_file,
|
||||||
select_file,
|
select_file,
|
||||||
select_folder,
|
select_folder,
|
||||||
)
|
)
|
||||||
from .constants import DICTATION_TYPE, DICT_TYPE, LANGUAGES, PHRASES_TYPE
|
from .constants import (
|
||||||
|
COMPLETED_TYPE,
|
||||||
|
DICTATION_TYPE,
|
||||||
|
DICT_TYPE,
|
||||||
|
INPUT,
|
||||||
|
LANGUAGES,
|
||||||
|
PHRASES_TYPE,
|
||||||
|
)
|
||||||
from .utility import ProcessFile, ProcessFolder
|
from .utility import ProcessFile, ProcessFolder
|
||||||
|
|
||||||
|
|
||||||
@@ -50,7 +58,7 @@ def cli_select_files() -> ProcessFile:
|
|||||||
return input_file
|
return input_file
|
||||||
|
|
||||||
|
|
||||||
def cli_select_folder() -> ProcessFile:
|
def cli_select_folder() -> ProcessFolder:
|
||||||
"""Loops until it finds a valid input_folder"""
|
"""Loops until it finds a valid input_folder"""
|
||||||
print("Select data folder:")
|
print("Select data folder:")
|
||||||
in_folder = None
|
in_folder = None
|
||||||
@@ -66,7 +74,7 @@ def cli_select_folder() -> ProcessFile:
|
|||||||
print(f"Selected {selected}")
|
print(f"Selected {selected}")
|
||||||
s = None
|
s = None
|
||||||
while s not in ("yes", "y", "no", "n"):
|
while s not in ("yes", "y", "no", "n"):
|
||||||
s = input("if this the folder? (yes/no, go inside)")
|
s = input("if this the folder? (yes / no, go inside) : ")
|
||||||
if s in ("yes", "y"):
|
if s in ("yes", "y"):
|
||||||
in_folder = selected
|
in_folder = selected
|
||||||
else:
|
else:
|
||||||
@@ -100,9 +108,10 @@ def cli_select_language(languages: list = None) -> str:
|
|||||||
for language_id, language in avaliable_languages:
|
for language_id, language in avaliable_languages:
|
||||||
if languages and language_id in languages:
|
if languages and language_id in languages:
|
||||||
print(f"{language_id} - {language}")
|
print(f"{language_id} - {language}")
|
||||||
|
print("all - All languages")
|
||||||
s = None
|
s = None
|
||||||
while not s or s not in LANGUAGES.AvailableLanguages:
|
while not s or s not in LANGUAGES.AvailableLanguages + ("all",):
|
||||||
lan_codes = [lan_id for lan_id, lan in avaliable_languages]
|
lan_codes = [lan_id for lan_id, lan in avaliable_languages] + ["all"]
|
||||||
s = input(f"Please select the language {', '.join(lan_codes)}: ")
|
s = input(f"Please select the language {', '.join(lan_codes)}: ")
|
||||||
return s
|
return s
|
||||||
|
|
||||||
@@ -115,33 +124,56 @@ def main():
|
|||||||
break
|
break
|
||||||
elif option == "s":
|
elif option == "s":
|
||||||
input_file = cli_select_files()
|
input_file = cli_select_files()
|
||||||
if DICT_TYPE in input_file.input_file.suffixes:
|
if DICT_TYPE is input_file.file_type:
|
||||||
dict_selected = cli_select_dictionay_tsv()
|
dict_selected = cli_select_dictionay_tsv()
|
||||||
if dict_selected:
|
if dict_selected:
|
||||||
language_id = cli_select_language()
|
language_id = cli_select_language()
|
||||||
print(
|
print(
|
||||||
f"pre-processing file {input_file} with language {language_id}"
|
f"pre-processing {DICT_TYPE} file {input_file} with language {language_id}"
|
||||||
)
|
)
|
||||||
pre_process_a_dictionary_file(input_file, language_id)
|
pre_process_a_dictionary_file(input_file, language_id)
|
||||||
else:
|
else:
|
||||||
print(f"Processing file {input_file} with language {language_id}")
|
print(
|
||||||
|
f"Processing {DICT_TYPE} file {input_file} with language {language_id}"
|
||||||
|
)
|
||||||
language_id = cli_select_language(
|
language_id = cli_select_language(
|
||||||
input_file.available_dictionary_languages
|
input_file.available_dictionary_languages
|
||||||
)
|
)
|
||||||
process_a_dictionary_file(input_file, language_id)
|
process_a_dictionary_file(input_file, language_id)
|
||||||
elif PHRASES_TYPE in input_file.input_file.suffixes:
|
elif PHRASES_TYPE is input_file.file_type:
|
||||||
language_id = cli_select_language()
|
language_id = cli_select_language()
|
||||||
print(f"processing file {input_file} with language {language_id}")
|
print(
|
||||||
|
f"processing {PHRASES_TYPE} file {input_file} with language {language_id}"
|
||||||
|
)
|
||||||
process_a_phrases_file(input_file, language_id)
|
process_a_phrases_file(input_file, language_id)
|
||||||
elif DICTATION_TYPE in input_file.input_file.suffixes:
|
elif DICTATION_TYPE is input_file.file_type:
|
||||||
language_id = cli_select_language()
|
language_id = cli_select_language()
|
||||||
print(f"processing file {input_file} with language {language_id}")
|
print(
|
||||||
|
f"processing {DICTATION_TYPE} file {input_file} with language {language_id}"
|
||||||
|
)
|
||||||
process_a_dictation_file(input_file, language_id)
|
process_a_dictation_file(input_file, language_id)
|
||||||
|
elif COMPLETED_TYPE is input_file.file_type:
|
||||||
|
language_id = input_file.input_file.suffixes[1][1:]
|
||||||
|
print(
|
||||||
|
f"processing {COMPLETED_TYPE} file {input_file} with language {language_id}"
|
||||||
|
)
|
||||||
|
proccess_a_completed_file(input_file, language_id)
|
||||||
|
else:
|
||||||
|
print(f"File {input_file} is not recognised")
|
||||||
elif option == "f":
|
elif option == "f":
|
||||||
in_folder = cli_select_folder()
|
|
||||||
language_id = cli_select_language()
|
language_id = cli_select_language()
|
||||||
|
langs = (
|
||||||
|
LANGUAGES.AvailableLanguages if language_id == "all" else [language_id]
|
||||||
|
)
|
||||||
|
in_folder = cli_select_folder()
|
||||||
|
for language_id in langs:
|
||||||
print(f"Selected: {in_folder} with language {language_id}")
|
print(f"Selected: {in_folder} with language {language_id}")
|
||||||
folder_proccess(in_folder, language_id=language_id)
|
for dirpath, dirnames, filenames in tuple(
|
||||||
|
in_folder.absolute_input_folder.walk()
|
||||||
|
)[1:]:
|
||||||
|
if not dirnames and filenames:
|
||||||
|
folder = select_folder(dirpath.relative_to(INPUT))
|
||||||
|
folder_proccess(folder, language_id=language_id)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -12,8 +12,14 @@ from genanki import Deck, Model, Note, Package
|
|||||||
from pinyin_tone_converter.pinyin_tone_converter import PinyinToneConverter
|
from pinyin_tone_converter.pinyin_tone_converter import PinyinToneConverter
|
||||||
|
|
||||||
# Local
|
# Local
|
||||||
from .utility import DictionaryResult, ProcessFile, ProcessFolder, TranslationResult
|
from .constants import COMPLETED_TYPE, DICTATION_TYPE, DICT_TYPE, PHRASES_TYPE
|
||||||
from .constants import DICT_TYPE, DICTATION_TYPE, PHRASES_TYPE
|
from .utility import (
|
||||||
|
DictionaryResult,
|
||||||
|
ProcessFile,
|
||||||
|
ProcessFolder,
|
||||||
|
SimpleResult,
|
||||||
|
TranslationResult,
|
||||||
|
)
|
||||||
|
|
||||||
# Constants
|
# Constants
|
||||||
|
|
||||||
@@ -137,9 +143,86 @@ HSK_MODEL = Model(
|
|||||||
css=CSS,
|
css=CSS,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
SIMPLE_HSK_MODEL = Model(
|
||||||
|
2819647620,
|
||||||
|
"Simple HSK Model",
|
||||||
|
fields=[
|
||||||
|
{"name": "Meaning"},
|
||||||
|
{"name": "Pinyin"},
|
||||||
|
{"name": "Simplified"},
|
||||||
|
{"name": "Audio"},
|
||||||
|
],
|
||||||
|
templates=[
|
||||||
|
{
|
||||||
|
"name": "Card 1",
|
||||||
|
"qfmt": (
|
||||||
|
"<strong>{{Pinyin}}</strong>"
|
||||||
|
"<br>{{Meaning}}"
|
||||||
|
"<br>{{Audio}}"
|
||||||
|
"<br>Simplified: {{type:Simplified}}"
|
||||||
|
),
|
||||||
|
"afmt": (
|
||||||
|
"{{FrontSide}}<hr id='answer''><div class='simple'>{{Simplified}}</div>"
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Card 2",
|
||||||
|
"qfmt": (
|
||||||
|
"<div class='simple'>{{Simplified}}</div>" "<br>Pinyin: {{type:Pinyin}}"
|
||||||
|
),
|
||||||
|
"afmt": (
|
||||||
|
"{{FrontSide}}<hr id='answer'><strong>{{Pinyin}}</strong>"
|
||||||
|
"<br>{{Meaning}}<br>{{Audio}}"
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Card 3",
|
||||||
|
"qfmt": ("{{Audio}}" "<br>Simplified: {{type:Simplified}}"),
|
||||||
|
"afmt": (
|
||||||
|
"{{FrontSide}}<hr id='answer'><strong>{{Pinyin}}</strong>"
|
||||||
|
"<br><div class='simple'>{{Simplified}}</div>"
|
||||||
|
),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
css=CSS,
|
||||||
|
)
|
||||||
|
|
||||||
# Proccess
|
# Proccess
|
||||||
|
|
||||||
|
|
||||||
|
def output_anki_completed(
|
||||||
|
process_file: ProcessFile, results: list[SimpleResult]
|
||||||
|
) -> Path:
|
||||||
|
"""Creates an anki file for a completed file"""
|
||||||
|
final_file = (
|
||||||
|
process_file.output_name.parent / f"{process_file.input_file.stem}.apkg"
|
||||||
|
)
|
||||||
|
deck_name = "::".join(process_file.input_file.parts[:-1] + (final_file.stem,))
|
||||||
|
deck = Deck(
|
||||||
|
random.randrange(1 << 30, 1 << 31),
|
||||||
|
deck_name,
|
||||||
|
f"Deck for {process_file.input_file.stem}, "
|
||||||
|
"created in https://www.wolfang.info.ve/hskankicreator/",
|
||||||
|
)
|
||||||
|
audios = []
|
||||||
|
for result in results:
|
||||||
|
note = Note(
|
||||||
|
model=SIMPLE_HSK_MODEL,
|
||||||
|
fields=[
|
||||||
|
result.meaning,
|
||||||
|
PinyinToneConverter().convert_text(result.pinyin),
|
||||||
|
result.character,
|
||||||
|
f"[sound:{result.audio_path.name}]",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
deck.add_note(note)
|
||||||
|
audios.append(result.audio_path)
|
||||||
|
package = Package(deck)
|
||||||
|
package.media_files = audios
|
||||||
|
package.write_to_file(final_file)
|
||||||
|
return final_file
|
||||||
|
|
||||||
|
|
||||||
def output_anki_dictation(
|
def output_anki_dictation(
|
||||||
process_file: ProcessFile, results: list[DictionaryResult]
|
process_file: ProcessFile, results: list[DictionaryResult]
|
||||||
) -> Path:
|
) -> Path:
|
||||||
@@ -262,7 +345,7 @@ def output_anki_package(
|
|||||||
fields=[
|
fields=[
|
||||||
result.translated,
|
result.translated,
|
||||||
result.line,
|
result.line,
|
||||||
f"[sound:{result.audio_path}]",
|
f"[sound:{result.audio_path.name}]",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
deck.add_note(note)
|
deck.add_note(note)
|
||||||
@@ -277,7 +360,7 @@ def output_anki_package(
|
|||||||
PinyinToneConverter().convert_text(result.pinyin),
|
PinyinToneConverter().convert_text(result.pinyin),
|
||||||
result.simplified,
|
result.simplified,
|
||||||
result.traditional,
|
result.traditional,
|
||||||
f"[sound:{result.audio_path}]",
|
f"[sound:{result.audio_path.name}]",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
deck.add_note(note)
|
deck.add_note(note)
|
||||||
@@ -289,12 +372,26 @@ def output_anki_package(
|
|||||||
fields=[
|
fields=[
|
||||||
result.translated,
|
result.translated,
|
||||||
result.line,
|
result.line,
|
||||||
f"[sound:{result.audio_path}]",
|
f"[sound:{result.audio_path.name}]",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
deck.add_note(note)
|
||||||
|
audios.append(result.audio_path)
|
||||||
|
elif process_file.file_type is COMPLETED_TYPE:
|
||||||
|
for result in results:
|
||||||
|
note = Note(
|
||||||
|
model=SIMPLE_HSK_MODEL,
|
||||||
|
fields=[
|
||||||
|
result.meaning,
|
||||||
|
PinyinToneConverter().convert_text(result.pinyin),
|
||||||
|
result.character,
|
||||||
|
f"[sound:{result.audio_path.name}]",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
deck.add_note(note)
|
deck.add_note(note)
|
||||||
audios.append(result.audio_path)
|
audios.append(result.audio_path)
|
||||||
decks.append(deck)
|
decks.append(deck)
|
||||||
|
if decks:
|
||||||
package = Package(decks)
|
package = Package(decks)
|
||||||
package.media_files = audios
|
package.media_files = audios
|
||||||
package.write_to_file(final_file)
|
package.write_to_file(final_file)
|
||||||
|
|||||||
@@ -4,17 +4,20 @@ Interface for managuing and procesing files
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
# Standard Library
|
# Standard Library
|
||||||
|
import re
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
# Local
|
# Local
|
||||||
from . import DATA_FOLDER
|
from . import DATA_FOLDER
|
||||||
from .anki_generation import (
|
from .anki_generation import (
|
||||||
|
output_anki_completed,
|
||||||
output_anki_dictation,
|
output_anki_dictation,
|
||||||
output_anki_dictionary,
|
output_anki_dictionary,
|
||||||
output_anki_phrase,
|
|
||||||
output_anki_package,
|
output_anki_package,
|
||||||
|
output_anki_phrase,
|
||||||
)
|
)
|
||||||
from .constants import (
|
from .constants import (
|
||||||
|
COMPLETED_TYPE,
|
||||||
DICTATION_TYPE,
|
DICTATION_TYPE,
|
||||||
DICT_TYPE,
|
DICT_TYPE,
|
||||||
INPUT,
|
INPUT,
|
||||||
@@ -24,6 +27,7 @@ from .constants import (
|
|||||||
RESOURCES,
|
RESOURCES,
|
||||||
)
|
)
|
||||||
from .proccessor import (
|
from .proccessor import (
|
||||||
|
completed_process,
|
||||||
dictation_process,
|
dictation_process,
|
||||||
dictionary_bulk_process,
|
dictionary_bulk_process,
|
||||||
dictionary_pre_process,
|
dictionary_pre_process,
|
||||||
@@ -231,7 +235,9 @@ def process_a_dictation_file(process_file: ProcessFile, language_id: str) -> Pat
|
|||||||
with process_file.absolute_input_file.open(
|
with process_file.absolute_input_file.open(
|
||||||
"r", encoding="utf8", newline="\n"
|
"r", encoding="utf8", newline="\n"
|
||||||
) as file:
|
) as file:
|
||||||
text_lines = [line.strip() for line in file.read().split("。") if line.strip()]
|
text = file.read().strip()
|
||||||
|
result = re.split(r"[!?。;]*", text)
|
||||||
|
text_lines = [line.strip() for line in result if line.strip()]
|
||||||
results = dictation_process(text_lines, process_file)
|
results = dictation_process(text_lines, process_file)
|
||||||
return output_anki_dictation(process_file, results)
|
return output_anki_dictation(process_file, results)
|
||||||
|
|
||||||
@@ -249,14 +255,32 @@ def process_a_phrases_file(process_file: ProcessFile, language_id: str) -> Path:
|
|||||||
return output_anki_phrase(process_file, results)
|
return output_anki_phrase(process_file, results)
|
||||||
|
|
||||||
|
|
||||||
|
def proccess_a_completed_file(process_file: ProcessFile, language_id: str) -> Path:
|
||||||
|
"""Process a completed file"""
|
||||||
|
process_file.language_id = language_id
|
||||||
|
TTS.create_tts()
|
||||||
|
with process_file.absolute_input_file.open(
|
||||||
|
"r", encoding="utf8", newline="\n"
|
||||||
|
) as file:
|
||||||
|
text_lines = [line.strip() for line in file.readlines() if line.strip()]
|
||||||
|
results = completed_process(text_lines, process_file)
|
||||||
|
return output_anki_completed(process_file, results)
|
||||||
|
|
||||||
|
|
||||||
def folder_proccess(process_folder: ProcessFolder, language_id: str):
|
def folder_proccess(process_folder: ProcessFolder, language_id: str):
|
||||||
process_folder.language_id = language_id
|
process_folder.language_id = language_id
|
||||||
|
print(f"Proccesing folder {process_folder} with language {language_id}")
|
||||||
|
final_file = process_folder.output_name.with_suffix(".apkg")
|
||||||
|
if final_file.is_file():
|
||||||
|
print("File already proccessed")
|
||||||
|
return final_file
|
||||||
TTS.create_tts()
|
TTS.create_tts()
|
||||||
TRANS.create_translator(LANGUAGES.CN, language_id)
|
TRANS.create_translator(LANGUAGES.CN, language_id)
|
||||||
CCCEDICT.create_cedict(language_id)
|
CCCEDICT.create_cedict(language_id)
|
||||||
results = {}
|
results = {}
|
||||||
for process_file in process_folder.input_files:
|
for process_file in process_folder.input_files:
|
||||||
print(f"Proccessing {process_file}")
|
print(f"Proccessing {process_file}")
|
||||||
|
try:
|
||||||
if process_file.file_type is DICT_TYPE:
|
if process_file.file_type is DICT_TYPE:
|
||||||
with process_file.absolute_input_file.open(
|
with process_file.absolute_input_file.open(
|
||||||
"r", encoding="utf8", newline="\n"
|
"r", encoding="utf8", newline="\n"
|
||||||
@@ -276,6 +300,23 @@ def folder_proccess(process_folder: ProcessFolder, language_id: str):
|
|||||||
with process_file.absolute_input_file.open(
|
with process_file.absolute_input_file.open(
|
||||||
"r", encoding="utf8", newline="\n"
|
"r", encoding="utf8", newline="\n"
|
||||||
) as file:
|
) as file:
|
||||||
text_lines = [line.strip() for line in file.readlines() if line.strip()]
|
text_lines = [
|
||||||
|
line.strip() for line in file.readlines() if line.strip()
|
||||||
|
]
|
||||||
results[process_file] = translator_process(text_lines, process_file)
|
results[process_file] = translator_process(text_lines, process_file)
|
||||||
|
elif process_file.file_type is COMPLETED_TYPE:
|
||||||
|
completed_language_id = process_file.input_file.suffixes[1][1:]
|
||||||
|
if completed_language_id != language_id:
|
||||||
|
continue
|
||||||
|
process_file.language_id = completed_language_id
|
||||||
|
with process_file.absolute_input_file.open(
|
||||||
|
"r", encoding="utf8", newline="\n"
|
||||||
|
) as file:
|
||||||
|
text_lines = [
|
||||||
|
line.strip() for line in file.readlines() if line.strip()
|
||||||
|
]
|
||||||
|
results[process_file] = completed_process(text_lines, process_file)
|
||||||
|
except (AttributeError, AssertionError):
|
||||||
|
print(f"Error procesing {process_file} with language {language_id}")
|
||||||
|
continue
|
||||||
return output_anki_package(process_folder, results)
|
return output_anki_package(process_folder, results)
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ GWEN_TTS = GWEN_FOLDER / "Qwen3-TTS-12Hz-0.6B-CustomVoice"
|
|||||||
PHRASES_TYPE = ".phrases"
|
PHRASES_TYPE = ".phrases"
|
||||||
DICT_TYPE = ".dictionary"
|
DICT_TYPE = ".dictionary"
|
||||||
DICTATION_TYPE = ".dictation"
|
DICTATION_TYPE = ".dictation"
|
||||||
|
COMPLETED_TYPE = ".completed"
|
||||||
|
|
||||||
|
|
||||||
class LANGUAGES:
|
class LANGUAGES:
|
||||||
@@ -34,8 +35,11 @@ class LANGUAGES:
|
|||||||
RU = "ru"
|
RU = "ru"
|
||||||
TR = "tr"
|
TR = "tr"
|
||||||
TH = "th"
|
TH = "th"
|
||||||
JP = "jp"
|
JA = "ja"
|
||||||
AvailableLanguages = (EN, ES, FR, RU, TR, TH, JP)
|
KO = "ko"
|
||||||
|
PB = "pb"
|
||||||
|
VI = "vi"
|
||||||
|
AvailableLanguages = (ES, EN, FR, RU, TR, TH, JA, KO, PB, VI)
|
||||||
LanguageNames = {
|
LanguageNames = {
|
||||||
EN: "English",
|
EN: "English",
|
||||||
ES: "Spanish",
|
ES: "Spanish",
|
||||||
@@ -43,5 +47,8 @@ class LANGUAGES:
|
|||||||
RU: "Russian",
|
RU: "Russian",
|
||||||
TR: "Turkish",
|
TR: "Turkish",
|
||||||
TH: "Thai",
|
TH: "Thai",
|
||||||
JP: "Japanese",
|
JA: "Japanese",
|
||||||
|
KO: "Korean",
|
||||||
|
PB: "Portuguese (Brazil)",
|
||||||
|
VI: "Vietnamese",
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,7 +10,14 @@ import soundfile as sf
|
|||||||
|
|
||||||
# Local
|
# Local
|
||||||
from .constants import LANGUAGES
|
from .constants import LANGUAGES
|
||||||
from .utility import CCCEDICT, TTS, DictionaryResult, ProcessFile, TranslationResult
|
from .utility import (
|
||||||
|
CCCEDICT,
|
||||||
|
TTS,
|
||||||
|
DictionaryResult,
|
||||||
|
ProcessFile,
|
||||||
|
SimpleResult,
|
||||||
|
TranslationResult,
|
||||||
|
)
|
||||||
|
|
||||||
# Constants
|
# Constants
|
||||||
|
|
||||||
@@ -27,7 +34,11 @@ def dictation_process(
|
|||||||
results = []
|
results = []
|
||||||
for n, line in enumerate(text_lines):
|
for n, line in enumerate(text_lines):
|
||||||
audio_line = line.strip()
|
audio_line = line.strip()
|
||||||
audio_path = process_file.resources / f"N{n:03n}.wav"
|
posible_name = tuple(process_file.resources.glob(f"N{n:03n}-*.wav"))
|
||||||
|
if posible_name:
|
||||||
|
audio_path = posible_name[0]
|
||||||
|
else:
|
||||||
|
audio_path = process_file.resources / f"N{n:03n}-{uuid.uuid1()}.wav"
|
||||||
if not audio_path.exists():
|
if not audio_path.exists():
|
||||||
wavs, sr = TTS.generate(f"{audio_line}。")
|
wavs, sr = TTS.generate(f"{audio_line}。")
|
||||||
sf.write(audio_path, wavs[0], sr)
|
sf.write(audio_path, wavs[0], sr)
|
||||||
@@ -47,7 +58,11 @@ def translator_process(
|
|||||||
results = []
|
results = []
|
||||||
for n, line in enumerate(text_lines):
|
for n, line in enumerate(text_lines):
|
||||||
audio_line = line.strip()
|
audio_line = line.strip()
|
||||||
audio_path = process_file.resources / f"N{n:03n}.wav"
|
posible_name = tuple(process_file.resources.glob(f"N{n:03n}-*.wav"))
|
||||||
|
if posible_name:
|
||||||
|
audio_path = posible_name[0]
|
||||||
|
else:
|
||||||
|
audio_path = process_file.resources / f"N{n:03n}-{uuid.uuid1()}.wav"
|
||||||
if not audio_path.exists():
|
if not audio_path.exists():
|
||||||
wavs, sr = TTS.generate(f"{audio_line}。")
|
wavs, sr = TTS.generate(f"{audio_line}。")
|
||||||
sf.write(audio_path, wavs[0], sr)
|
sf.write(audio_path, wavs[0], sr)
|
||||||
@@ -60,7 +75,26 @@ def translator_process(
|
|||||||
return results
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
def completed_process(
|
||||||
|
text_lines: list[str], process_file: ProcessFile
|
||||||
|
) -> list[TranslationResult]:
|
||||||
|
"""Process a complete file, not translation necesary"""
|
||||||
|
results = []
|
||||||
|
dictionary = CCCEDICT.create_cedict()
|
||||||
|
for n, line in enumerate(text_lines):
|
||||||
|
char, meaning = line.split(maxsplit=1)
|
||||||
|
entries = dictionary.get(char)
|
||||||
|
pinying = entries[0].pinyin
|
||||||
|
audio_path = process_file.resources / f"{char}.wav"
|
||||||
|
if not audio_path.is_file():
|
||||||
|
wavs, sr = TTS.generate(f"{char}。")
|
||||||
|
sf.write(audio_path, wavs[0], sr)
|
||||||
|
results.append(SimpleResult(char, pinying, meaning, audio_path))
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
def dictionary_bulk_process(words_list: list[str], process_file: ProcessFile):
|
def dictionary_bulk_process(words_list: list[str], process_file: ProcessFile):
|
||||||
|
dictionary_en = CCCEDICT.create_cedict(LANGUAGES.EN)
|
||||||
dictionary = CCCEDICT.create_cedict(process_file.language_id)
|
dictionary = CCCEDICT.create_cedict(process_file.language_id)
|
||||||
with process_file.dictionary_resource_file.open(
|
with process_file.dictionary_resource_file.open(
|
||||||
"w", encoding="utf8", newline=""
|
"w", encoding="utf8", newline=""
|
||||||
@@ -72,31 +106,33 @@ def dictionary_bulk_process(words_list: list[str], process_file: ProcessFile):
|
|||||||
number = 1
|
number = 1
|
||||||
for words in words_list:
|
for words in words_list:
|
||||||
word = words.split()[0]
|
word = words.split()[0]
|
||||||
hint = " ".join(words.split()[1:]) if len(words.split()) > 1 else None
|
hint = words.split(maxsplit=1)[1] if len(words.split()) > 1 else None
|
||||||
if entries := dictionary.get(word):
|
entries_en = dictionary_en.get(word)
|
||||||
if len(entries) > 1:
|
entries = dictionary.get(word)
|
||||||
print(f"\nWARNING: {word} has multiple meanings:")
|
if entries:
|
||||||
pos_meanings = (
|
all_meanings = [
|
||||||
[
|
meaning for entry in entries for meaning in entry.meanings
|
||||||
entry
|
|
||||||
for entry in entries
|
|
||||||
for meaning in entry.meanings
|
|
||||||
if hint in meaning
|
|
||||||
]
|
]
|
||||||
if hint
|
pos_meanings = [
|
||||||
else []
|
meaning
|
||||||
|
for entry, entry_en in zip(entries, entries_en)
|
||||||
|
for meaning, meaning_en in zip(entry.meanings, entry_en.meanings)
|
||||||
|
if hint and (hint in meaning_en or hint in meaning)
|
||||||
|
]
|
||||||
|
pos_meanings = pos_meanings or all_meanings
|
||||||
|
if not pos_meanings:
|
||||||
|
raise ValueError(
|
||||||
|
f"Not menaing found for hint {hint}, {all_meanings}"
|
||||||
)
|
)
|
||||||
pos_entries = pos_meanings or entries
|
meanings_text = "\n".join(
|
||||||
meanings = []
|
f"{n+1}: {meaning}" for n, meaning in enumerate(pos_meanings)
|
||||||
for entry in pos_entries:
|
)
|
||||||
for meaning in entry.meanings:
|
pos_entries = []
|
||||||
if hint and hint in meaning:
|
for meaning in pos_meanings:
|
||||||
meanings.append(meaning)
|
for ent in entries:
|
||||||
if not meanings:
|
if meaning in ent.meanings:
|
||||||
meanings = [
|
pos_entries.append(ent)
|
||||||
meaning for entry in pos_entries for meaning in entry.meanings
|
entry = pos_entries[0]
|
||||||
]
|
|
||||||
meanings_text = meanings[0]
|
|
||||||
tsv_writer.writerow(
|
tsv_writer.writerow(
|
||||||
{
|
{
|
||||||
"n": number,
|
"n": number,
|
||||||
@@ -183,7 +219,6 @@ def dictionary_process(process_file: ProcessFile) -> list[DictionaryResult]:
|
|||||||
if not audio_path.exists():
|
if not audio_path.exists():
|
||||||
wavs, sr = TTS.generate(f"{line['simplified']}。")
|
wavs, sr = TTS.generate(f"{line['simplified']}。")
|
||||||
sf.write(audio_path, wavs[0], sr)
|
sf.write(audio_path, wavs[0], sr)
|
||||||
print(line)
|
|
||||||
result = DictionaryResult(
|
result = DictionaryResult(
|
||||||
**line, audio_path=audio_path, language_id=process_file.language_id
|
**line, audio_path=audio_path, language_id=process_file.language_id
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ from qwen_tts import Qwen3TTSModel
|
|||||||
# Local
|
# Local
|
||||||
from .constants import (
|
from .constants import (
|
||||||
CCCEDICT_PATH,
|
CCCEDICT_PATH,
|
||||||
|
COMPLETED_TYPE,
|
||||||
DICTATION_TYPE,
|
DICTATION_TYPE,
|
||||||
DICT_TYPE,
|
DICT_TYPE,
|
||||||
GWEN_TTS,
|
GWEN_TTS,
|
||||||
@@ -53,8 +54,8 @@ class TRANS:
|
|||||||
TRANS.PACKAGES,
|
TRANS.PACKAGES,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
print(f"available packages {packages[:5]}")
|
|
||||||
packages_to_install = []
|
packages_to_install = []
|
||||||
|
ready = False
|
||||||
for in_package in packages:
|
for in_package in packages:
|
||||||
if in_package.from_code == from_code:
|
if in_package.from_code == from_code:
|
||||||
if in_package.to_code == to_code:
|
if in_package.to_code == to_code:
|
||||||
@@ -64,7 +65,12 @@ class TRANS:
|
|||||||
f"->{in_package.to_code}"
|
f"->{in_package.to_code}"
|
||||||
)
|
)
|
||||||
packages_to_install.append(in_package)
|
packages_to_install.append(in_package)
|
||||||
|
ready = True
|
||||||
break
|
break
|
||||||
|
for in_package in packages:
|
||||||
|
if ready:
|
||||||
|
break
|
||||||
|
if in_package.from_code == from_code:
|
||||||
for out_package in packages:
|
for out_package in packages:
|
||||||
if out_package.to_code == to_code:
|
if out_package.to_code == to_code:
|
||||||
if in_package.to_code == out_package.from_code:
|
if in_package.to_code == out_package.from_code:
|
||||||
@@ -78,6 +84,7 @@ class TRANS:
|
|||||||
)
|
)
|
||||||
packages_to_install.append(in_package)
|
packages_to_install.append(in_package)
|
||||||
packages_to_install.append(out_package)
|
packages_to_install.append(out_package)
|
||||||
|
ready = True
|
||||||
break
|
break
|
||||||
for package in packages_to_install:
|
for package in packages_to_install:
|
||||||
print(f"instaling package {package}")
|
print(f"instaling package {package}")
|
||||||
@@ -110,13 +117,13 @@ class TranslatedEntry:
|
|||||||
@property
|
@property
|
||||||
def meanings(self):
|
def meanings(self):
|
||||||
"""Entry translated meaning list"""
|
"""Entry translated meaning list"""
|
||||||
|
if not self._translated_meanings:
|
||||||
for meaning in self.entry.meanings:
|
for meaning in self.entry.meanings:
|
||||||
if self.language_id != LANGUAGES.EN:
|
if self.language_id != LANGUAGES.EN:
|
||||||
print(f"translating from {LANGUAGES.EN} to {self.language_id}")
|
|
||||||
print(f"-> {meaning}")
|
|
||||||
trans_meaning = argostranslate.translate.translate(
|
trans_meaning = argostranslate.translate.translate(
|
||||||
meaning, LANGUAGES.EN, self.language_id
|
meaning, LANGUAGES.EN, self.language_id
|
||||||
)
|
)
|
||||||
|
print(f"{meaning}-> {trans_meaning}")
|
||||||
else:
|
else:
|
||||||
trans_meaning = meaning
|
trans_meaning = meaning
|
||||||
self._translated_meanings.append(trans_meaning)
|
self._translated_meanings.append(trans_meaning)
|
||||||
@@ -196,6 +203,7 @@ class TTS:
|
|||||||
print(f"finish to generate {text}")
|
print(f"finish to generate {text}")
|
||||||
return audio
|
return audio
|
||||||
|
|
||||||
|
|
||||||
# Clases
|
# Clases
|
||||||
|
|
||||||
|
|
||||||
@@ -226,7 +234,7 @@ class ProcessFolder:
|
|||||||
def input_files(self):
|
def input_files(self):
|
||||||
input_files = []
|
input_files = []
|
||||||
for file in self.absolute_input_folder.glob(f"*.txt"):
|
for file in self.absolute_input_folder.glob(f"*.txt"):
|
||||||
for file_type in (DICT_TYPE, PHRASES_TYPE, DICTATION_TYPE):
|
for file_type in (DICT_TYPE, PHRASES_TYPE, DICTATION_TYPE, COMPLETED_TYPE):
|
||||||
if file_type in file.suffixes:
|
if file_type in file.suffixes:
|
||||||
input_files.append(
|
input_files.append(
|
||||||
ProcessFile(
|
ProcessFile(
|
||||||
@@ -263,9 +271,18 @@ class ProcessFile:
|
|||||||
# process file type
|
# process file type
|
||||||
self.out_folder = OUTPUT / input_file.parent
|
self.out_folder = OUTPUT / input_file.parent
|
||||||
self.out_folder.mkdir(parents=True, exist_ok=True)
|
self.out_folder.mkdir(parents=True, exist_ok=True)
|
||||||
resources = RESOURCES / input_file
|
|
||||||
self.resources = resources.parent / resources.stem
|
@property
|
||||||
self.resources.mkdir(parents=True, exist_ok=True)
|
def resources(self):
|
||||||
|
resources = RESOURCES / self.input_file
|
||||||
|
if self.file_type is COMPLETED_TYPE:
|
||||||
|
resources = (
|
||||||
|
resources.parent / f"{self.input_file.name.split(".")[0]}.dictionary"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
resources = resources.parent / resources.stem
|
||||||
|
resources.mkdir(parents=True, exist_ok=True)
|
||||||
|
return resources
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def file_type(self):
|
def file_type(self):
|
||||||
@@ -275,6 +292,8 @@ class ProcessFile:
|
|||||||
return DICT_TYPE
|
return DICT_TYPE
|
||||||
elif PHRASES_TYPE in self.input_file.suffixes:
|
elif PHRASES_TYPE in self.input_file.suffixes:
|
||||||
return PHRASES_TYPE
|
return PHRASES_TYPE
|
||||||
|
elif COMPLETED_TYPE in self.input_file.suffixes:
|
||||||
|
return COMPLETED_TYPE
|
||||||
else:
|
else:
|
||||||
raise ValueError("File type not recognized")
|
raise ValueError("File type not recognized")
|
||||||
|
|
||||||
@@ -297,6 +316,8 @@ class ProcessFile:
|
|||||||
"""Posible name for the output file, still missing the filetype"""
|
"""Posible name for the output file, still missing the filetype"""
|
||||||
if self.language_id is None:
|
if self.language_id is None:
|
||||||
raise ValueError("Not a valid language selected")
|
raise ValueError("Not a valid language selected")
|
||||||
|
if self.file_type is COMPLETED_TYPE:
|
||||||
|
return self.out_folder / f"{self.input_file.stem}.temp"
|
||||||
return self.out_folder / f"{self.input_file.stem}.{self.language_id}.temp"
|
return self.out_folder / f"{self.input_file.stem}.{self.language_id}.temp"
|
||||||
|
|
||||||
@property
|
@property
|
||||||
@@ -319,6 +340,16 @@ class ProcessFile:
|
|||||||
return f"Proccess:{self.input_file}"
|
return f"Proccess:{self.input_file}"
|
||||||
|
|
||||||
|
|
||||||
|
class SimpleResult:
|
||||||
|
"""Simplied result class"""
|
||||||
|
|
||||||
|
def __init__(self, character: str, pinyin: str, meaning: str, audio_path: Path):
|
||||||
|
self.character = character
|
||||||
|
self.pinyin = pinyin
|
||||||
|
self.meaning = meaning
|
||||||
|
self.audio_path = audio_path
|
||||||
|
|
||||||
|
|
||||||
class TranslationResult:
|
class TranslationResult:
|
||||||
"""Result of a translated process"""
|
"""Result of a translated process"""
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user