Store Support Bot in Python: Source Code

1"""Detect the language of a customer message, so the model is told explicitly how to reply.23A general rule in the system prompt ("answer in the customer's language") is not enough on its own:4models drift, for example by ending an English reply with a Darija sentence. A short per-message5instruction based on this detection keeps every reply in the right language.6"""78from __future__ import annotations910import re1112_ARABIC_LETTER = re.compile(r"[ء-ي]")13_LATIN_LETTER = re.compile(r"[A-Za-zÀ-ÿ]")1415# Words that only appear in Algerian Darija, written in Arabic letters.16_DARIJA_ARABIC = re.compile(r"(قداش|قداه|واش|كاين|ماكانش|راني|راهو|راهي|نحب|تحب|بزاف|دروك|درك|تاع|باش|شحال|وين|علاش|كيفاش|بصح)")17# Darija written in Latin letters: common words, or digits used as letters (3, 7, 9) inside a word.18_DARIJA_LATIN_WORDS = re.compile(19    r"\b(ta3|wach|wesh|kayen|kayn|makach|rani|rak|nheb|t7eb|ch7al|chhal|bezzaf|chwiya|dork|drk|win|wech|saha|sahit|"20    r"salam|khoya|kifach|3lach|bach|mlih|ghir|rah|nbadel|nbeddel|3andkom|3andek|mta3)\b",21    re.IGNORECASE,22)23_DIGIT_AS_LETTER = re.compile(r"[a-z][379][a-z]|\b[379][a-z]{2,}|[a-z]{2,}[379]\b", re.IGNORECASE)24_FRENCH = re.compile(25    r"\b(je|vous|votre|vos|nous|est|les|des|une|pour|avec|merci|bonjour|bonsoir|commande|où|combien|livraison|"26    r"chiffres|derniers|ma|mon|mes|le|la|du|se|par|pas|quand|comment)\b",27    re.IGNORECASE,28)29_ENGLISH = re.compile(30    r"\b(the|you|your|is|are|and|can|could|order|please|with|what|where|when|how|my|do|does|i|me|to|of|all|list|who)\b",31    re.IGNORECASE,32)3334DARIJA_LATIN = "Algerian Darija in Latin letters (Algerian words only, never Moroccan forms)"35DARIJA_ARABIC = "Algerian Darija in Arabic letters"36MSA = "Modern Standard Arabic (الفصحى), with no Darija words, including in the closing sentence"37FRENCH = "French only, with no Darija words, including in the closing sentence"38ENGLISH = "English only, with no Darija words, including in the closing sentence"394041def detect_reply_language(text: str) -> str | None:42    """Return a short description of the language to reply in, or None when it is unclear."""43    arabic = len(_ARABIC_LETTER.findall(text))44    latin = len(_LATIN_LETTER.findall(text))45    if arabic == 0 and latin == 0:46        return None47    if arabic > latin:48        return DARIJA_ARABIC if _DARIJA_ARABIC.search(text) else MSA4950    if _DARIJA_LATIN_WORDS.search(text) or _DIGIT_AS_LETTER.search(text):51        return DARIJA_LATIN52    french = len(_FRENCH.findall(text))53    english = len(_ENGLISH.findall(text))54    if french > english:55        return FRENCH56    if english > french:57        return ENGLISH58    return None596061def language_instruction(text: str) -> str | None:62    language = detect_reply_language(text)63    if language is None:64        return None65    return f"Reply to the next customer message in {language}."