"""Reproducible Persian localization using HPLT Marian; reviewed overrides take precedence.
Run with PYTHONPATH pointing at optional ctranslate2/sentencepiece/pyyaml dependencies.
Model weights are external and are not distributed with the website.
"""
from pathlib import Path
import re,json,yaml,time,sys
ROOT=Path(__file__).resolve().parents[1]
cache_file=ROOT/'scripts/fa-translation-cache.json'
cache=json.loads(cache_file.read_text()) if cache_file.exists() else {}
over=json.loads((ROOT/'scripts/fa-overrides.json').read_text());phrases=over['phrases'];ui=over['ui']
needed=set()
glossary=dict(line.split('\t',1) for line in (ROOT/'scripts/fa-glossary.tsv').read_text().splitlines() if line.strip())
glossary={k.lower():v for k,v in glossary.items()}

# Preserve complete MDX tags (including multiline attributes), destinations, citations and UI substitutions.
protected=re.compile(r'(<(?:[^>"\']|"[^"]*"|\'[^\']*\')+>|\$\{[^}]*\}|\{[a-zA-Z][^{}]*\}|\[(?:COUNT|SEARCH_TERM|DIFFERENT_TERM)\]|https?://[^\s<)]+|`[^`]*`|\]\([^)]*\)|&(?:#\d+|#x[0-9a-fA-F]+|[a-zA-Z]+);|\*\*|\[|\n|\s*^#{1,6}\s+)',re.M)
def tr(s):
 if not isinstance(s,str) or not re.search('[A-Za-z]{2}',s):return s
 a=s.strip()
 if a in phrases:return s.replace(a,phrases[a])
 if a.lower() in glossary:return s.replace(a,glossary[a.lower()])
 if a in cache:return s.replace(a,cache[a])
 needed.add(a);return s

def prose(s):
 if s.strip() in phrases or s.strip().lower() in glossary:return tr(s)
 out=[]
 for i,p in enumerate(protected.split(s)):
  if i%2:out.append(p);continue
  # Sentence boundaries keep individual segments short and cacheable.
  parts=re.split(r'(?<=[.!?])(?=\s+[A-Z])',p)
  out.append(''.join(tr(x) for x in parts))
 return ''.join(out)

def yaml_translate(d):
 if isinstance(d,list):return [yaml_translate(x) for x in d]
 if isinstance(d,dict):return {k:prose(v) if k in ['title','description','prompt','explanation','text'] and isinstance(v,str) else yaml_translate(v) if isinstance(v,(dict,list)) else v for k,v in d.items()}
 return d

def process(write=False):
 en=json.loads((ROOT/'apps/web/src/i18n/messages/en.json').read_text());fa={}
 for k,v in en.items():fa[k]=ui[k] if k in ui else prose(v)
 if write:(ROOT/'apps/web/src/i18n/messages/fa.json').write_text(json.dumps(dict(sorted(fa.items())),ensure_ascii=False,indent=2)+'\n')
 for p in (ROOT/'content/en').rglob('*.mdx'):
  _,fm,body=p.read_text().split('---',2);d=yaml.safe_load(fm)
  for k in ['title','description']:d[k]=prose(d[k])
  aliases=[prose(x) for x in d.get('aliases',[])]+d.get('aliases',[]);seen={d['title'].strip().lower()};clean=[]
  for a in aliases:
   n=a.strip().lower()
   if n not in seen:clean.append(a);seen.add(n)
  if aliases:d['aliases']=clean
  result='---\n'+yaml.safe_dump(d,allow_unicode=True,sort_keys=False,width=100000)+'---'+prose(body)
  if write:
   q=ROOT/'content/fa'/p.relative_to(ROOT/'content/en');q.parent.mkdir(parents=True,exist_ok=True);q.write_text(result)
 for p in [*(ROOT/'content/en').rglob('*.quiz.yaml'),*(ROOT/'content/tracks/en').glob('*.yaml')]:
  d=yaml_translate(yaml.safe_load(p.read_text()));q=ROOT/str(p.relative_to(ROOT)).replace('/en/','/fa/')
  if write:q.parent.mkdir(parents=True,exist_ok=True);q.write_text(yaml.safe_dump(d,allow_unicode=True,sort_keys=False,width=100000))
 p=ROOT/'scripts/cardio-tokens.json'
 if p.exists():
  for t in json.loads(p.read_text()):prose(t['text'])
 # Static simulator text and accessibility attributes.
 from bs4 import BeautifulSoup,Comment
 raw=(ROOT/'vendor/cardiomechanics/index.html').read_text();soup=BeautifulSoup(raw,'html.parser')
 for el in soup.find_all(string=True):
  if not isinstance(el,Comment) and el.parent.name not in ['script','style']:prose(str(el))
 for el in soup.find_all(True):
  for a in ['title','aria-label','placeholder']:
   if el.get(a):prose(el[a])

if __name__=='__main__':
 process()
 todo=sorted(needed-set(cache)-set(phrases))
 print('UNIQUE_SEGMENTS',len(todo),flush=True)
 if '--render' not in sys.argv and todo:
  import ctranslate2,sentencepiece as sp
  model=ROOT.parent/'hplt-model';tokenizer=sp.SentencePieceProcessor(model_file=str(model/'model.en-fa.spm'))
  translator=ctranslate2.Translator(str(model/'ct2'),device='cpu',compute_type='int8',inter_threads=1,intra_threads=6)
  # Numerals and abbreviated measurement labels are kept exactly as in the source.
  numeric=re.compile('|'.join(re.escape(x) for x in sorted(glossary,key=len,reverse=True))+r'|(?-i:\b[A-Z]{2,10}\b)|\d+(?:[.,]\d+)*(?:[–−-]\d+(?:\.\d+)*)?',re.I)
  numbers_only=re.compile(r'\d+(?:[.,]\d+)*')
  began=time.time()
  def prepare(s):
   nums=[]
   def repl(m):nums.append(glossary.get(m[0].lower(),m[0]));return ' X'+str(len(nums)-1).zfill(3)+'X '
   return numeric.sub(repl,s),nums
  def mt(texts):
   if not texts:return []
   results=translator.translate_batch([tokenizer.encode(s,out_type=str) for s in texts],beam_size=3,max_decoding_length=384,max_input_length=512,max_batch_size=64)
   return [tokenizer.decode(x.hypotheses[0]) for x in results]
  def space_like(src,out):
   return re.match(r'^\s*',src)[0]+out.strip()+re.search(r'\s*$',src)[0]
  chunk_cache={}
  for begin in range(0,len(todo),128):
   batch=todo[begin:begin+128];prepared=[prepare(s) for s in batch]
   outputs=mt([s for s,_ in prepared]);invalid={};chunks=set()
   for i,(src,(_,nums),out) in enumerate(zip(batch,prepared,outputs)):
    if not all(('X'+str(j).zfill(3)+'X') in out for j in range(len(nums))):
     parts=numeric.split(src);invalid[i]=parts
     chunks.update(x.strip() for x in parts if re.search('[A-Za-z]',x) and x.strip() not in chunk_cache)
   ordered=sorted(chunks);chunk_cache.update(zip(ordered,mt(ordered)))
   for i,(src,(_,nums)) in enumerate(zip(batch,prepared)):
    if i in invalid:
     out=''
     for j,x in enumerate(invalid[i]):
      out+=space_like(x,chunk_cache[x.strip()]) if re.search('[A-Za-z]',x) else x
      if j<len(nums):out+=nums[j]
    else:
     out=outputs[i]
     for j,n in enumerate(nums):out=out.replace('X'+str(j).zfill(3)+'X',n)
    out=out.replace('⁇','–').replace('{','&#123;').replace('}','&#125;').replace('<','&lt;').replace('>','&gt;')
    if len(out)>max(140,len(src)*4) or re.search(r'X\d+X',out):out=src
    normal=out.translate(str.maketrans('۰۱۲۳۴۵۶۷۸۹٠١٢٣٤٥٦٧٨٩','01234567890123456789'))
    if sorted(numbers_only.findall(normal))!=sorted(numbers_only.findall(src)):out=src
    cache[src]=out
   tmp=cache_file.with_suffix('.tmp');tmp.write_text(json.dumps(cache,ensure_ascii=False));tmp.replace(cache_file)
   print('TRANSLATED',begin+len(batch),'/',len(todo),'elapsed',round(time.time()-began),flush=True)
 process(write=True)
 print('RENDERED',flush=True)