"""Detect the language of a customer message, so the model is told explicitly how to reply. A general rule in the system prompt ("answer in the customer's language") is not enough on its own: models drift, for example by ending an English reply with a Darija sentence. A short per-message instruction based on this detection keeps every reply in the right language. """ from __future__ import annotations import re _ARABIC_LETTER = re.compile(r"[ء-ي]") _LATIN_LETTER = re.compile(r"[A-Za-zÀ-ÿ]") # Words that only appear in Algerian Darija, written in Arabic letters. _DARIJA_ARABIC = re.compile(r"(قداش|قداه|واش|كاين|ماكانش|راني|راهو|راهي|نحب|تحب|بزاف|دروك|درك|تاع|باش|شحال|وين|علاش|كيفاش|بصح)") # Darija written in Latin letters: common words, or digits used as letters (3, 7, 9) inside a word. _DARIJA_LATIN_WORDS = re.compile( r"\b(ta3|wach|wesh|kayen|kayn|makach|rani|rak|nheb|t7eb|ch7al|chhal|bezzaf|chwiya|dork|drk|win|wech|saha|sahit|" r"salam|khoya|kifach|3lach|bach|mlih|ghir|rah|nbadel|nbeddel|3andkom|3andek|mta3)\b", re.IGNORECASE, ) _DIGIT_AS_LETTER = re.compile(r"[a-z][379][a-z]|\b[379][a-z]{2,}|[a-z]{2,}[379]\b", re.IGNORECASE) _FRENCH = re.compile( r"\b(je|vous|votre|vos|nous|est|les|des|une|pour|avec|merci|bonjour|bonsoir|commande|où|combien|livraison|" r"chiffres|derniers|ma|mon|mes|le|la|du|se|par|pas|quand|comment)\b", re.IGNORECASE, ) _ENGLISH = re.compile( r"\b(the|you|your|is|are|and|can|could|order|please|with|what|where|when|how|my|do|does|i|me|to|of|all|list|who)\b", re.IGNORECASE, ) DARIJA_LATIN = "Algerian Darija in Latin letters (Algerian words only, never Moroccan forms)" DARIJA_ARABIC = "Algerian Darija in Arabic letters" MSA = "Modern Standard Arabic (الفصحى), with no Darija words, including in the closing sentence" FRENCH = "French only, with no Darija words, including in the closing sentence" ENGLISH = "English only, with no Darija words, including in the closing sentence" def detect_reply_language(text: str) -> str | None: """Return a short description of the language to reply in, or None when it is unclear.""" arabic = len(_ARABIC_LETTER.findall(text)) latin = len(_LATIN_LETTER.findall(text)) if arabic == 0 and latin == 0: return None if arabic > latin: return DARIJA_ARABIC if _DARIJA_ARABIC.search(text) else MSA if _DARIJA_LATIN_WORDS.search(text) or _DIGIT_AS_LETTER.search(text): return DARIJA_LATIN french = len(_FRENCH.findall(text)) english = len(_ENGLISH.findall(text)) if french > english: return FRENCH if english > french: return ENGLISH return None def language_instruction(text: str) -> str | None: language = detect_reply_language(text) if language is None: return None return f"Reply to the next customer message in {language}."