How to Build a Multilingual AI Voice Assistant That Understands Hindi and English

multilingual-voice-assistant/
├── app/
│   ├── __init__.py
│   ├── hinglish_normalizer.py
│   └── wer_evaluator.py
├── tests/
│   └── test_multilingual_assistant.py
└── requirements.txt

import re
import unicodedata
from enum import Enum
from pydantic import BaseModel

DEVANAGARI_RANGE = re.compile(r"[\u0900-\u097F]")
LATIN_WORD_RANGE = re.compile(r"[a-zA-Z]+")
COMMON_ROMAN_HINDI_TOKENS = {
    "mera", "meri", "kya", "hai", "nahi", "karo", "karna", "kab",
    "aayega", "paisa", "bhaiya", "kripya", "batao", "mujhe", "chahiye",
}

class UtteranceLanguageMode(str, Enum):
    HINDI_DEVANAGARI = "HINDI_DEVANAGARI"
    HINGLISH_CODE_SWITCH = "HINGLISH_CODE_SWITCH"
    ENGLISH_INDIAN = "ENGLISH_INDIAN"

class LanguageAnalysis(BaseModel):
    mode: UtteranceLanguageMode
    devanagari_char_count: int
    latin_word_count: int
    normalized_tts_text: str

def detect_and_normalize_utterance(text: str) -> LanguageAnalysis:
    """Detects Hindi/English/Hinglish mode and normalizes symbols for clean TTS pronunciation."""
    norm = unicodedata.normalize("NFKC", text).strip()
    dev_chars = len(DEVANAGARI_RANGE.findall(norm))
    latin_words = [w.lower() for w in LATIN_WORD_RANGE.findall(norm)]
    roman_hindi_hits = sum(1 for w in latin_words if w in COMMON_ROMAN_HINDI_TOKENS)

    if dev_chars > 0 and len(latin_words) > 0:
        mode = UtteranceLanguageMode.HINGLISH_CODE_SWITCH
    elif dev_chars > 0:
        mode = UtteranceLanguageMode.HINDI_DEVANAGARI
    elif roman_hindi_hits >= 1:
        mode = UtteranceLanguageMode.HINGLISH_CODE_SWITCH
    else:
        mode = UtteranceLanguageMode.ENGLISH_INDIAN

    # Expand ₹ currency symbol so TTS reads it naturally in the target register
    if mode in (UtteranceLanguageMode.HINDI_DEVANAGARI, UtteranceLanguageMode.HINGLISH_CODE_SWITCH):
        tts_text = re.sub(r"₹\s*([0-9,]+)", r"\1 रुपये", norm)
    else:
        tts_text = re.sub(r"₹\s*([0-9,]+)", r"\1 rupees", norm)

    return LanguageAnalysis(
        mode=mode,
        devanagari_char_count=dev_chars,
        latin_word_count=len(latin_words),
        normalized_tts_text=tts_text,
    )

import re
import unicodedata
from typing import List

def normalize_for_asr_eval(text: str) -> str:
    """Normalizes Unicode, lowercases Latin characters, and strips punctuation for fair WER/CER."""
    norm = unicodedata.normalize("NFKC", text).lower()
    norm = re.sub(r"[^\w\s\u0900-\u097F]", "", norm)
    return " ".join(norm.split())

def _edit_distance(ref_seq: List[str], hyp_seq: List[str]) -> int:
    n, m = len(ref_seq), len(hyp_seq)
    dp = [[0] * (m + 1) for _ in range(n + 1)]
    for i in range(n + 1):
        dp[i][0] = i
    for j in range(m + 1):
        dp[0][j] = j
    for i in range(1, n + 1):
        for j in range(1, m + 1):
            cost = 0 if ref_seq[i - 1] == hyp_seq[j - 1] else 1
            dp[i][j] = min(
                dp[i - 1][j] + 1,      # deletion
                dp[i][j - 1] + 1,      # insertion
                dp[i - 1][j - 1] + cost # substitution
            )
    return dp[n][m]

def compute_wer_and_cer(reference: str, hypothesis: str) -> dict:
    """Computes normalized Word Error Rate (WER) and Character Error Rate (CER)."""
    clean_ref = normalize_for_asr_eval(reference)
    clean_hyp = normalize_for_asr_eval(hypothesis)

    ref_words = clean_ref.split()
    hyp_words = clean_hyp.split()
    word_edits = _edit_distance(ref_words, hyp_words)
    wer = round(word_edits / max(1, len(ref_words)), 4)

    ref_chars = list(clean_ref.replace(" ", ""))
    hyp_chars = list(clean_hyp.replace(" ", ""))
    char_edits = _edit_distance(ref_chars, hyp_chars)
    cer = round(char_edits / max(1, len(ref_chars)), 4)

    return {"wer": wer, "cer": cer, "word_edits": word_edits, "ref_word_count": len(ref_words)}

from app.hinglish_normalizer import detect_and_normalize_utterance, UtteranceLanguageMode
from app.wer_evaluator import compute_wer_and_cer

def test_hinglish_detection_and_rupee_expansion():
    mixed = detect_and_normalize_utterance("मेरा refund ₹1,500 कब आएगा?")
    assert mixed.mode == UtteranceLanguageMode.HINGLISH_CODE_SWITCH
    assert "1,500 रुपये" in mixed.normalized_tts_text

    roman_hinglish = detect_and_normalize_utterance("Bhaiya mera order status kya hai?")
    assert roman_hinglish.mode == UtteranceLanguageMode.HINGLISH_CODE_SWITCH

    english = detect_and_normalize_utterance("Please send the invoice for ₹2,000.")
    assert english.mode == UtteranceLanguageMode.ENGLISH_INDIAN
    assert "2,000 rupees" in english.normalized_tts_text

def test_wer_and_cer_calculation_on_bilingual_transcript():
    ref = "नमस्ते मेरा order status बताइए"
    hyp = "नमस्ते मेरा order स्टेटस बताइए"
    metrics = compute_wer_and_cer(ref, hyp)
    assert metrics["wer"] == 0.2  # 1 word script difference out of 5 words
    assert metrics["cer"] < 0.35