grok3 vibe coded this translation algo!
I'm hacking with a new crew & we are working with unsloth fine tuning locally. Im learning a lot as i go. Also i am in the process of working out an ETL job for translating my blog into the top 20 languages and using pandoc to convert rst to md for agents.This comes out of the conversation that every piece is static data should be intelligently translated from now on. Calling hermes sequentially is more than enough to peg my GPU which is a local 4090. Backfilling is painfully slow with only one card. Grok3 vibed coded an algorithm that keeps track of strings that should not be translated. This is used for code blocks, URLs & images using regex replacements that are then passed to the LLM which is prompted to _NOT_ translate the substitutions. Once we get the completion from the model inference we can easily restore the substituted strings with the original values! CC @Lisa_MegaWatts & Robert & @ajaxdavis new file: translate_content.py
This commit is contained in:
parent
e5038b4c67
commit
3eb23977a0
1 changed files with 228 additions and 0 deletions
228
translate_content.py
Normal file
228
translate_content.py
Normal file
|
|
@ -0,0 +1,228 @@
|
|||
#!/usr/bin/env python3
|
||||
import os
|
||||
import hashlib
|
||||
import json
|
||||
from openai import OpenAI
|
||||
import pandoc
|
||||
import re
|
||||
import time
|
||||
|
||||
# AI client setup
|
||||
client = OpenAI(base_url="https://hermes.ai.unturf.com/v1", api_key="choose-any-value")
|
||||
MODEL = "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic"
|
||||
|
||||
TOP_20_LANGS = {
|
||||
"en": "English",
|
||||
"zh": "Chinese (Simplified)",
|
||||
"hi": "Hindi",
|
||||
"es": "Spanish",
|
||||
"fr": "French",
|
||||
"ar": "Arabic",
|
||||
"bn": "Bengali",
|
||||
"ru": "Russian",
|
||||
"pt": "Portuguese",
|
||||
"ur": "Urdu",
|
||||
"id": "Indonesian",
|
||||
"de": "German",
|
||||
"ja": "Japanese",
|
||||
"sw": "Swahili",
|
||||
"mr": "Marathi",
|
||||
"te": "Telugu",
|
||||
"tr": "Turkish",
|
||||
"ta": "Tamil",
|
||||
"zh-tw": "Chinese (Traditional)",
|
||||
"ko": "Korean",
|
||||
}
|
||||
|
||||
CONTENT_DIR = "content"
|
||||
HASH_FILE = "content/translation_hashes.json"
|
||||
|
||||
|
||||
def replace_special_content(content):
|
||||
"""Replace code blocks, URIs, and images with unique placeholders."""
|
||||
patterns = [
|
||||
(re.compile(r"(\.\.\s+code::\s+[^\n]*(?:\n\s+[^\n]+)*)"), "__CODE_{}__"),
|
||||
(re.compile(r"`([^`]+?)`(?![`_])"), "__CODE_{}__"),
|
||||
(re.compile(r"`([^`]+?)\s+<(https?://[^\s>]+)>`_"), "__URI_{}__"),
|
||||
(re.compile(r"(`[^`]+?`_\s+)?<(https?://[^\s>]+)>"), "__URI_{}__"),
|
||||
(re.compile(r"(\.\.\s+_.*?:(?:\s+https?://[^\s]+)?)"), "__URI_{}__"),
|
||||
(re.compile(r"(\.\.\s+image::\s+[^\s]+(?:\s*\n\s+:.*?)*)"), "__IMG_{}__"),
|
||||
]
|
||||
|
||||
placeholders = {}
|
||||
counter = {"code": 0, "uri": 0, "img": 0}
|
||||
|
||||
def replacer(match, placeholder_template, key):
|
||||
nonlocal counter
|
||||
full_match = match.group(0)
|
||||
placeholder = placeholder_template.format(counter[key])
|
||||
placeholders[placeholder] = full_match
|
||||
counter[key] += 1
|
||||
return placeholder
|
||||
|
||||
replaced_content = content
|
||||
for pattern, template in patterns:
|
||||
if "CODE" in template:
|
||||
replaced_content = pattern.sub(lambda m: replacer(m, template, "code"), replaced_content)
|
||||
elif "URI" in template:
|
||||
replaced_content = pattern.sub(lambda m: replacer(m, template, "uri"), replaced_content)
|
||||
elif "IMG" in template:
|
||||
replaced_content = pattern.sub(lambda m: replacer(m, template, "img"), replaced_content)
|
||||
|
||||
return replaced_content, placeholders
|
||||
|
||||
|
||||
def restore_special_content(content, placeholders):
|
||||
"""Restore original code, URIs, and images from placeholders."""
|
||||
for placeholder, original in placeholders.items():
|
||||
content = content.replace(placeholder, original)
|
||||
return content
|
||||
|
||||
|
||||
def translate_text(text, target_lang_full, retries=3, delay=5):
|
||||
prompt = (
|
||||
f"Translate the following reStructuredText (.rst) content into {target_lang_full}. "
|
||||
"The input is in English and formatted as valid reStructuredText, which includes metadata "
|
||||
"like titles (underlined with # or =), directives, and body text. "
|
||||
"Ensure the output remains valid reStructuredText, preserving the structure, syntax, and formatting "
|
||||
"of the original, including proper handling of titles, directives, and text. "
|
||||
"Do not translate or modify placeholders like __CODE_0__, __URI_0__, __IMG_0__, etc., as they represent "
|
||||
"code blocks, URLs, or image references that should remain unchanged. Only translate the surrounding text. "
|
||||
"Here is the text to translate:\n\n"
|
||||
f"{text}"
|
||||
)
|
||||
messages = [{"role": "user", "content": prompt}]
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
response = client.chat.completions.create(
|
||||
model=MODEL, messages=messages, temperature=0.5, max_tokens=60000
|
||||
)
|
||||
return response.choices[0].message.content
|
||||
except Exception as e:
|
||||
print(f"Attempt {attempt + 1}/{retries} failed: {e}")
|
||||
if attempt < retries - 1:
|
||||
time.sleep(delay)
|
||||
else:
|
||||
raise
|
||||
|
||||
|
||||
def load_hashes():
|
||||
if os.path.exists(HASH_FILE):
|
||||
with open(HASH_FILE, "r") as f:
|
||||
return json.load(f)
|
||||
return {}
|
||||
|
||||
|
||||
def save_hashes(hashes):
|
||||
with open(HASH_FILE, "w") as f:
|
||||
json.dump(hashes, f, indent=2)
|
||||
|
||||
|
||||
def get_content_hash(content):
|
||||
return hashlib.sha256(content.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def process_file(file_path):
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
lines = f.readlines()
|
||||
header_end = next(
|
||||
(i for i, line in enumerate(lines) if not line.strip() or i > 4), len(lines)
|
||||
)
|
||||
header = lines[:header_end]
|
||||
content = "".join(lines[header_end:])
|
||||
|
||||
base_name = os.path.splitext(os.path.basename(file_path))[0]
|
||||
slug = base_name
|
||||
source_hash = get_content_hash(content)
|
||||
|
||||
hashes = load_hashes()
|
||||
file_hashes = hashes.get(file_path, {})
|
||||
needs_update = file_hashes.get("source") != source_hash
|
||||
|
||||
# Replace code, URIs, and images with placeholders
|
||||
replaced_content, placeholders = replace_special_content(content)
|
||||
|
||||
for lang_code, lang_full in TOP_20_LANGS.items():
|
||||
lang_dir = f"{CONTENT_DIR}/{lang_code}"
|
||||
os.makedirs(lang_dir, exist_ok=True)
|
||||
|
||||
rst_file = f"{lang_dir}/{slug}.rst"
|
||||
md_file = f"{lang_dir}/{slug}.md"
|
||||
|
||||
# Update header with lang_code
|
||||
new_header = []
|
||||
lang_set = False
|
||||
for line in header:
|
||||
if line.startswith(":lang:"):
|
||||
new_header.append(f":lang: {lang_code}\n")
|
||||
lang_set = True
|
||||
else:
|
||||
new_header.append(line)
|
||||
if not lang_set:
|
||||
new_header.append(f":lang: {lang_code}\n")
|
||||
|
||||
# Check existing hashes for this language
|
||||
lang_hashes = file_hashes.get(lang_code, {})
|
||||
rst_hash = lang_hashes.get("rst")
|
||||
md_hash = lang_hashes.get("md")
|
||||
|
||||
# Check if files exist and match hashes
|
||||
rst_exists = os.path.exists(rst_file)
|
||||
md_exists = os.path.exists(md_file)
|
||||
if rst_exists and md_exists and not needs_update:
|
||||
with open(rst_file, "r", encoding="utf-8") as f:
|
||||
current_rst_hash = get_content_hash(f.read())
|
||||
with open(md_file, "r", encoding="utf-8") as f:
|
||||
current_md_hash = get_content_hash(f.read())
|
||||
if current_rst_hash == rst_hash and current_md_hash == md_hash:
|
||||
print(f"Skipping {slug} in {lang_full} ({lang_code}) - up to date")
|
||||
continue
|
||||
|
||||
# Translate or use original content
|
||||
if lang_code == "en":
|
||||
translated_replaced_content = replaced_content
|
||||
else:
|
||||
print(f"Translating {slug} to {lang_full} ({lang_code})")
|
||||
translated_replaced_content = translate_text(replaced_content, lang_full)
|
||||
|
||||
# Restore special content
|
||||
translated_content = restore_special_content(translated_replaced_content, placeholders)
|
||||
full_rst_content = "".join(new_header) + translated_content
|
||||
|
||||
# Save .rst and calculate hash
|
||||
with open(rst_file, "w", encoding="utf-8") as f:
|
||||
f.write(full_rst_content)
|
||||
rst_hash = get_content_hash(full_rst_content)
|
||||
|
||||
# Convert to .md and calculate hash
|
||||
replaced_full_rst = "".join(new_header) + translated_replaced_content
|
||||
md_content = pandoc.read(replaced_full_rst, format="rst")
|
||||
md_replaced = pandoc.write(md_content, format="markdown")
|
||||
md_final = restore_special_content(md_replaced, placeholders)
|
||||
|
||||
with open(md_file, "w", encoding="utf-8") as f:
|
||||
f.write(md_final)
|
||||
md_hash = get_content_hash(md_final)
|
||||
|
||||
# Update hashes for this language
|
||||
if file_path not in hashes:
|
||||
hashes[file_path] = {}
|
||||
hashes[file_path]["source"] = source_hash
|
||||
if lang_code not in hashes[file_path]:
|
||||
hashes[file_path][lang_code] = {}
|
||||
hashes[file_path][lang_code]["rst"] = rst_hash
|
||||
hashes[file_path][lang_code]["md"] = md_hash
|
||||
|
||||
# Save hashes after each language
|
||||
save_hashes(hashes)
|
||||
|
||||
|
||||
def main():
|
||||
for root, _, files in os.walk(CONTENT_DIR):
|
||||
for file in files:
|
||||
if file.endswith(".rst") and not any(f"/{lang}/" in root for lang in TOP_20_LANGS.keys()):
|
||||
process_file(os.path.join(root, file))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue