Fix Kurdish letter forms, test on Python 3.15, bump to 3.1.0

.github/workflows/ci.yml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"]
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/release.yml

@@ -10,7 +10,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"]
     steps:
       - uses: actions/checkout@v4
       - uses: astral-sh/setup-uv@v5

arabic_reshaper/__init__.py

@@ -8,4 +8,4 @@
                               ENABLE_ALL_LIGATURES)
 
 
-__version__ = '3.0.0'
+from .__version__ import __version__

arabic_reshaper/__version__.py

@@ -1 +1 @@
-__version__ = '3.0.0'
+__version__ = '3.1.0'

arabic_reshaper/letters.py

@@ -369,7 +369,7 @@
     # ARABIC LETTER JEEM
     '\u062C': ('\u062C', '\uFE9F', '\uFEA0', '\uFE9E'),
     # ARABIC LETTER HAH
-    '\u062D': ('\uFEA1', '\uFEA3', '\uFEA4', '\uFEA2'),
+    '\u062D': ('\u062D', '\uFEA3', '\uFEA4', '\uFEA2'),
     # ARABIC LETTER KHAH
     '\u062E': ('\u062E', '\uFEA7', '\uFEA8', '\uFEA6'),
     # ARABIC LETTER DAL
@@ -411,7 +411,7 @@
     # ARABIC LETTER NOON
     '\u0646': ('\u0646', '\uFEE7', '\uFEE8', '\uFEE6'),
     # ARABIC LETTER HEH
-    '\u0647': ('\uFBAB', '\uFBAB', '\uFBAB', '\uFBAB'),
+    '\u0647': ('\u0647', '\u06BE', '\uFBAB', '\uFEEA'),
     # ARABIC LETTER WAW
     '\u0648': ('\u0648', '', '', '\uFEEE'),
     # ARABIC LETTER (UIGHUR KAZAKH KIRGHIZ)? ALEF MAKSURA
@@ -452,6 +452,8 @@
     '\u068E': ('\u068E', '', '', '\uFB87'),
     # ARABIC LETTER RREH
     '\u0691': ('\u0691', '', '', '\uFB8D'),
+    # KURDISH LETTER REH WITH SMALL V BELOW
+    '\u0695': ('\u0695', '', '', '\uE001'),
     # ARABIC LETTER JEH
     '\u0698': ('\u0698', '', '', '\uFB8B'),
     # ARABIC LETTER VEH
@@ -501,7 +503,7 @@
     # ARABIC LETTER YEH BARREE WITH HAMZA ABOVE
     '\u06D3': ('\u06D3', '', '', '\uFBB1'),
     # Kurdish letter YEAH
-    '\u06ce': ('\uE004', '\uE005', '\uE006', '\uE004'),
+    '\u06ce': ('\u06ce', '\uE005', '\uE006', '\uE004'),
     # Kurdish letter Hamza same as arabic Teh without the point
     '\u06d5': ('\u06d5', '', '', '\uE000'),
     # ZWJ

arabic_reshaper/tests/test_005_kurdish.py

@@ -42,5 +42,49 @@ def test_words(self):
                 self.assertEqual(self.reshaper.reshape(text), expected)
 
 
+class TestKurdishFontForms(unittest.TestCase):
+    """Forms checked against the Sarchia and Unikurd Web fonts, which draw
+    them from these code points. See test_006_kurdish_font_glyphs.py."""
+
+    def setUp(self):
+        self.reshaper = arabic_reshaper.ArabicReshaper(
+            configuration={'language': 'Kurdish'}
+        )
+
+    def check(self, cases):
+        for text, expected in cases:
+            with self.subTest(text=text):
+                self.assertEqual(self.reshaper.reshape(text), expected)
+
+    def test_heh(self):
+        self.check((
+            ('\u0647', '\u0647'),
+            ('\u0647\u0645', '\u06BE\uFEE2'),
+            ('\u0645\u0647\u0645', '\uFEE3\uFBAB\uFEE2'),
+            ('\u0645\u0647', '\uFEE3\uFEEA'),
+            # هەولێر (Erbil): initial heh before ae
+            ('\u0647\u06D5\u0648\u0644\u06CE\u0631',
+             '\u06BE\uE000\u0648\uFEDF\uE006\uFEAE'),
+        ))
+
+    def test_isolated_hah_and_yeah(self):
+        # Kurdish fonts have no glyph at U+FEA1, and U+E004 is the final
+        # form of yeah, so the isolated forms use the base letters.
+        self.check((
+            ('\u062D', '\u062D'),
+            ('\u06CE', '\u06CE'),
+        ))
+
+    def test_reh_with_small_v_below(self):
+        self.check((
+            ('\u0695', '\u0695'),
+            ('\u0645\u0695', '\uFEE3\uE001'),
+            # does not join the following letter
+            ('\u0645\u0695\u0645', '\uFEE3\uE001\u0645'),
+            # کوڕ (boy): isolated, after waw
+            ('\u06A9\u0648\u0695', '\uFB90\uFEEE\u0695'),
+        ))
+
+
 if __name__ == '__main__':
     unittest.main()

arabic_reshaper/tests/test_006_kurdish_font_glyphs.py

@@ -0,0 +1,167 @@
+"""Check the Kurdish letter table against real Kurdish fonts.
+
+Kurdish fonts such as Sarchia and Unikurd Web draw some contextual forms
+from private-use code points (U+E000-U+E009) or from presentation-form code
+points they use in their own way, and they leave out many standard
+presentation forms. ``LETTERS_KURDISH`` has to match those fonts, and a test
+on strings alone cannot show that the right glyph appears on screen.
+
+This test uses the font itself. These fonts also have OpenType (GSUB) rules
+that pick the right contextual glyph for the plain Unicode letter. For every
+letter and every position (isolated, initial, medial, final) it checks that:
+
+* HarfBuzz shaping the *unshaped* text with the font's own rules, and
+* the font's cmap glyph for the code point arabic_reshaper outputs
+
+give the same glyph (same glyph, or an identical outline and advance width).
+
+Put Kurdish fonts (for example ``85_Sarchia_Soran.ttf`` or Unikurd Web's
+``Unikuweb.ttf``) anywhere under ``fonts/`` at the repository root. That
+directory is git-ignored. The test runs only on fonts that use the Kurdish
+private-use convention, and skips letters a font doesn't have.
+"""
+from pathlib import Path
+
+import pytest
+
+pytest.importorskip('fontTools', reason='fonttools not installed')
+hb = pytest.importorskip('uharfbuzz', reason='uharfbuzz not installed')
+
+from fontTools.pens.recordingPen import RecordingPen
+from fontTools.ttLib import TTFont
+
+from arabic_reshaper import ArabicReshaper
+from arabic_reshaper.letters import FINAL, INITIAL, ISOLATED, LETTERS_KURDISH, MEDIAL
+
+_FONTS_DIR = Path(__file__).parent.parent.parent / 'fonts'
+_ALL_FONTS = sorted(
+    p for ext in ('*.ttf', '*.otf') for p in _FONTS_DIR.rglob(ext)
+) if _FONTS_DIR.is_dir() else []
+
+# The Sorani Kurdish alphabet. Tested even when a letter is missing from
+# LETTERS_KURDISH, so a missing entry fails instead of going unnoticed.
+SORANI_ALPHABET = (
+    'ئابپتجچحخدرڕزژسشعغفڤقکگلڵمنوۆهەیێ'
+)
+
+MEEM = 'م'  # dual-joining neighbour present in every Arabic-script font
+FORM_NAMES = {ISOLATED: 'isolated', INITIAL: 'initial',
+              MEDIAL: 'medial', FINAL: 'final'}
+
+
+def _contexts(letter):
+    """(form, text, index of the letter in the text) for every position."""
+    return [
+        (ISOLATED, letter, 0),
+        (INITIAL, letter + MEEM, 0),
+        (MEDIAL, MEEM + letter + MEEM, 1),
+        (FINAL, MEEM + letter, 1),
+    ]
+
+
+def _is_kurdish_pua_font(cmap):
+    # Kurdish fonts put the final form of AE (U+06D5) at U+E000.
+    return 0x06D5 in cmap and 0xE000 in cmap
+
+
+def _collect_cases():
+    cases = []
+    letters = sorted(set(LETTERS_KURDISH) | set(SORANI_ALPHABET))
+    for path in _ALL_FONTS:
+        cmap = TTFont(str(path), lazy=True).getBestCmap() or {}
+        if not _is_kurdish_pua_font(cmap):
+            continue
+        for letter in letters:
+            if ord(letter) not in cmap:
+                continue
+            for form, text, index in _contexts(letter):
+                cases.append(pytest.param(
+                    path, letter, form, text, index,
+                    id=f'{path.stem}-U+{ord(letter):04X}-{FORM_NAMES[form]}',
+                ))
+    return cases
+
+
+_CASES = _collect_cases()
+
+
+class _Font:
+    _cache = {}
+
+    @classmethod
+    def load(cls, path):
+        if path not in cls._cache:
+            cls._cache[path] = cls(path)
+        return cls._cache[path]
+
+    def __init__(self, path):
+        self.name = path.name
+        self.tt = TTFont(str(path))
+        self.cmap = self.tt.getBestCmap()
+        self.order = self.tt.getGlyphOrder()
+        self.glyph_set = self.tt.getGlyphSet()
+        self.hb_font = hb.Font(hb.Face(hb.Blob.from_file_path(str(path))))
+
+    def shaped_glyph_at(self, text, index):
+        """Glyph name HarfBuzz picks for the character at ``index``."""
+        buf = hb.Buffer()
+        buf.add_str(text)
+        buf.guess_segment_properties()
+        # Turn off optional ligatures so every character keeps its own glyph.
+        hb.shape(self.hb_font, buf, {'liga': False, 'dlig': False})
+        names = [self.order[info.codepoint]
+                 for info in buf.glyph_infos if info.cluster == index]
+        assert len(names) == 1, (
+            f'{self.name}: expected one glyph for {text!r}[{index}], got {names}'
+        )
+        return names[0]
+
+    def signature(self, glyph_name):
+        pen = RecordingPen()
+        self.glyph_set[glyph_name].draw(pen)
+        return self.tt['hmtx'][glyph_name][0], pen.value
+
+    def same_glyph(self, a, b):
+        return a == b or self.signature(a) == self.signature(b)
+
+
+@pytest.fixture(scope='module')
+def reshaper():
+    return ArabicReshaper(configuration={
+        'language': 'Kurdish',
+        'support_ligatures': False,
+    })
+
+
+@pytest.mark.skipif(
+    not _CASES,
+    reason='No Kurdish font (with U+06D5 and U+E000) found under fonts/',
+)
+@pytest.mark.parametrize('font_path, letter, form, text, index', _CASES)
+def test_letter_form_matches_font(font_path, reshaper, letter, form, text,
+                                  index):
+    font = _Font.load(font_path)
+    expected_glyph = font.shaped_glyph_at(text, index)
+
+    reshaped = reshaper.reshape(text)
+    assert len(reshaped) == len(text), repr(reshaped)
+    out_char = reshaped[index]
+    mapped_glyph = font.cmap.get(ord(out_char))
+
+    if mapped_glyph is not None and font.same_glyph(mapped_glyph, expected_glyph):
+        return
+
+    if form != ISOLATED and expected_glyph == font.cmap[ord(letter)]:
+        # The font keeps the plain glyph here, so it defines no contextual
+        # form for this letter and says nothing about what the right one is.
+        pytest.skip(f'{font.name} has no {FORM_NAMES[form]} form '
+                    f'for U+{ord(letter):04X}')
+
+    assert mapped_glyph is not None, (
+        f'{font.name}: reshaper outputs U+{ord(out_char):04X} for {text!r}, '
+        f'which the font has no glyph for; the font uses {expected_glyph!r}'
+    )
+    pytest.fail(
+        f'{font.name}: reshaper outputs U+{ord(out_char):04X} '
+        f'({mapped_glyph!r}) for {text!r}, but the font uses {expected_glyph!r}'
+    )

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "arabic-reshaper"
-version = "3.0.1"
+version = "3.1.0"
 description = "Reconstruct Arabic sentences to be used in applications that do not support Arabic"
 readme = "README.md"
 license = { text = "MIT" }
@@ -24,6 +24,7 @@ classifiers = [
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
     "Programming Language :: Python :: 3.14",
+    "Programming Language :: Python :: 3.15",
     "Topic :: Software Development :: Libraries :: Python Modules",
 ]
 requires-python = ">=3.10"