logs and remove ta it hangs my llm...

modified:   translate_content.py

translate_content.py

@@ -2,10 +2,16 @@
 import os
 import hashlib
 import json
-from openai import OpenAI
+from openai import OpenAI, OpenAIError
 import pandoc
 import re
 import time
+import logging
+
+# Set up logging
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
+)
 
 # AI client setup
 client = OpenAI(base_url="https://hermes.ai.unturf.com/v1", api_key="choose-any-value")
@@ -29,7 +35,7 @@ TOP_20_LANGS = {
     "mr": "Marathi",
     "te": "Telugu",
     "tr": "Turkish",
-    "ta": "Tamil",
+    # "ta": "Tamil",
     "zh-tw": "Chinese (Traditional)",
     "ko": "Korean",
 }
@@ -85,7 +91,8 @@ def restore_special_content(content, placeholders):
     return content
 
 
-def translate_text(text, target_lang_full, retries=3, delay=5):
+def translate_text(text, target_lang_full, retries=3, delay=5, timeout=100):
+    """Translate text with logging, a long timeout, and error handling."""
     prompt = (
         f"Translate the following reStructuredText (.rst) content into {target_lang_full}. "
         "The input is in English and formatted as valid reStructuredText, which includes metadata "
@@ -98,18 +105,47 @@ def translate_text(text, target_lang_full, retries=3, delay=5):
         f"{text}"
     )
     messages = [{"role": "user", "content": prompt}]
+
+    logging.info(
+        f"Attempting translation to {target_lang_full} (length of input: {len(text)} characters)"
+    )
+
     for attempt in range(retries):
         try:
             response = client.chat.completions.create(
-                model=MODEL, messages=messages, temperature=0.5, max_tokens=60000
+                model=MODEL,
+                messages=messages,
+                temperature=0.5,
+                max_tokens=60000,
+                timeout=timeout,  # 100 seconds timeout
+            )
+            result = response.choices[0].message.content
+            logging.info(
+                f"Successfully translated to {target_lang_full} on attempt {attempt + 1}"
+            )
+            return result
+        except OpenAIError as e:
+            logging.error(
+                f"OpenAI API error on attempt {attempt + 1}/{retries}: {str(e)}"
             )
-            return response.choices[0].message.content
-        except Exception as e:
-            print(f"Attempt {attempt + 1}/{retries} failed: {e}")
             if attempt < retries - 1:
+                logging.info(f"Retrying in {delay} seconds...")
                 time.sleep(delay)
             else:
-                raise
+                raise Exception(
+                    f"Failed to translate to {target_lang_full} after {retries} attempts: {str(e)}"
+                )
+        except Exception as e:
+            logging.error(
+                f"Unexpected error on attempt {attempt + 1}/{retries}: {str(e)}"
+            )
+            if attempt < retries - 1:
+                logging.info(f"Retrying in {delay} seconds...")
+                time.sleep(delay)
+            else:
+                raise Exception(
+                    f"Translation to {target_lang_full} failed after {retries} attempts: {str(e)}"
+                )
 
 
 def load_hashes():
@@ -181,14 +217,16 @@ def process_file(file_path):
             with open(md_file, "r", encoding="utf-8") as f:
                 current_md_hash = get_content_hash(f.read())
             if current_rst_hash == rst_hash and current_md_hash == md_hash:
-                print(f"Skipping {slug} in {lang_full} ({lang_code}) - up to date")
+                logging.info(
+                    f"Skipping {slug} in {lang_full} ({lang_code}) - up to date"
+                )
                 continue
 
         # Translate or use original content
         if lang_code == "en":
             translated_replaced_content = replaced_content
         else:
-            print(f"Translating {slug} to {lang_full} ({lang_code})")
+            logging.info(f"Translating {slug} to {lang_full} ({lang_code})")
             translated_replaced_content = translate_text(replaced_content, lang_full)
 
         # Restore special content
@@ -231,6 +269,7 @@ def main():
             if file.endswith(".rst") and not any(
                 f"/{lang}/" in root for lang in TOP_20_LANGS.keys()
             ):
+                logging.info(f"Processing file: {os.path.join(root, file)}")
                 process_file(os.path.join(root, file))