#!/usr/bin/env python3.11
# -*- coding: utf-8 -*-
"""
analyze_endings_latin.py

Сравнивает частоту окончаний слов в туркменском (латиница) образце против
европейских образцов - после OCR через модель 'eng' (которая теряет
диакритику Ää/Ňň/Çç/Üü/Öö/Ýý), чтобы найти окончания, которые остаются
характерными для туркменского даже без диакритических знаков.
"""

import os
import re
from collections import Counter

import fitz
import pytesseract
from PIL import Image

INPUT_DIR = "/home/dayhanbiz/public_html/3-damja.science/ТЕСТ_образцы_языков/Тексты на 11 языках"

TURKMEN_FILE = "Görogly Şadessany.pdf"
EUROPEAN_FILES = [
    "Dobson R. Russia’s railway advance into Central Asia.pdf",
    "Tcharykow N. Un voyage dans l’Ouzbekistan en 1671.pdf",
    "Umland Andreas (ed.). Stalinismus und Stalin-Kult in Zentralasien - Turkmenistan 1924-1953 2009.pdf",
]

WORD_RE = re.compile(r"\w+", re.UNICODE)


def render_pages(doc, start=4, end=9, dpi=200):
    indices = [i - 1 for i in range(start, end + 1) if 0 <= i - 1 < doc.page_count]
    if not indices:
        indices = list(range(min(doc.page_count, 6)))
    images = []
    zoom = dpi / 72
    mat = fitz.Matrix(zoom, zoom)
    for idx in indices:
        page = doc.load_page(idx)
        pix = page.get_pixmap(matrix=mat)
        images.append(Image.frombytes("RGB", [pix.width, pix.height], pix.samples))
    return images


def ocr_words(images, lang="eng"):
    words = []
    for img in images:
        data = pytesseract.image_to_data(img, lang=lang, output_type=pytesseract.Output.DICT)
        for t in data.get("text", []):
            t = t.strip()
            if t:
                words.append(t)
    return words


def get_words(filename):
    path = os.path.join(INPUT_DIR, filename)
    doc = fitz.open(path)
    images = render_pages(doc)
    words = ocr_words(images, lang="eng")
    doc.close()
    return [w for w in words if WORD_RE.fullmatch(w) and len(w) >= 4]


def ending_freq(words, length):
    c = Counter(w[-length:].lower() for w in words if len(w) > length)
    total = len(words) if words else 1
    return c, total


def main():
    print("Распознаю туркменский (латиница) образец...")
    tk_words = get_words(TURKMEN_FILE)
    print(f"  слов: {len(tk_words)}")

    print("Распознаю европейские образцы...")
    eu_words = []
    for f in EUROPEAN_FILES:
        eu_words.extend(get_words(f))
    print(f"  слов: {len(eu_words)}")

    for length in (2, 3, 4):
        print(f"\n===== Окончания длиной {length} буквы =====")
        tk_c, tk_total = ending_freq(tk_words, length)
        eu_c, eu_total = ending_freq(eu_words, length)

        candidates = []
        for ending, count in tk_c.most_common(40):
            tk_ratio = count / tk_total
            eu_ratio = eu_c.get(ending, 0) / eu_total
            if tk_ratio >= 0.008 and eu_ratio < tk_ratio / 3:
                candidates.append((ending, tk_ratio, eu_ratio))

        candidates.sort(key=lambda x: -x[1])
        for ending, tk_r, eu_r in candidates[:15]:
            print(f"  -{ending:<6} туркм.={tk_r:.3f} ({int(tk_r*tk_total)} слов)   европ.={eu_r:.3f} ({int(eu_r*eu_total)} слов)")


if __name__ == "__main__":
    main()