Update app.py

app.py

@@ -9,6 +9,8 @@ import datetime
 import tempfile
 import soundfile as sf
 import os
+import re
+
 
 
 # --- Initiation ---
@@ -108,6 +110,40 @@ def call_microsoft_translate(text, src_lang, tgt_lang):
     # TODO: implement Microsoft API call
     return "Microsoft translated text"
 
+def safe_tokenize_sanskrit(text):
+    """
+    Return a list of tokens for Sanskrit (Devanagari). Try IndicTrans2 first,
+    then MBART-50, then XLM-R, finally a simple regex fallback.
+    """
+    # 1) Try IndicTrans2 tokenizer (no model needed)
+    try:
+        # IndicTrans2 tokenizer usually supports .tokenize(text)
+        return indictrans_tokenizer.tokenize(text)
+    except Exception:
+        pass
+
+    # 2) Try MBART-50 tokenizer (ensure it’s initialized)
+    try:
+        # MBART uses SentencePiece; .tokenize works and yields subwords
+        return tokenizer.tokenize(text)  # reuse your mbart tokenizer var
+    except Exception:
+        pass
+
+    # 3) Try XLM-R tokenizer (initialized at the top)
+    try:
+        return xlm_tokenizer.tokenize(text)
+    except Exception:
+        pass
+
+    # 4) Regex fallback: split on Sanskrit word boundaries and punctuation
+    # This keeps Devanagari characters together and separates punctuation/whitespace
+    return [tok for tok in re.split(r"(\s+|[—–\-॥।,.;:!?()
+
+\[\]
+
+{}\"'])", text) if tok.strip()]
+
+
     
 def run_task(text, language, task):
     # Always return: [audio_numpy, audio_filepath, text_output]
@@ -199,7 +235,11 @@ def run_task(text, language, task):
             return None, None, segmented_text
             #return None, None, xlm_tokenizer.tokenize(text)
         elif language == "Sanskrit":
-            return None, None, indictrans_tokenizer.tokenize(text)
+            raw_tokens = safe_tokenize_sanskrit(text)
+            # Return a human-readable string; if you prefer list, wrap with str(tokens)
+            tokens = normalize_sp_tokens(raw_tokens)
+            return None, None, " ".join(tokens)
+            #return None, None, indictrans_tokenizer.tokenize(text)
         else:
             return None, None, "Unsupported language"
 
@@ -207,6 +247,12 @@ def run_task(text, language, task):
         return None, None, "Unsupported task"
 
 
+def normalize_sp_tokens(tokens):
+    # Remove SentencePiece underscores and collapse spaces
+    return [t.replace("▁", "") for t in tokens]
+
+    
+
     
 # --- Gradio Interface ---
 iface = gr.Interface(