From 2237ea09767d7b99c49584562531ec6f37a81e8a Mon Sep 17 00:00:00 2001 From: Abdullah Diab Date: Mon, 2 Nov 2026 15:50:36 +0400 Subject: [PATCH 1/2] Fix Kurdish letter forms to match Kurdish fonts, test on Python 3.15 Check LETTERS_KURDISH against the Sarchia and Unikurd Web fonts and fix the entries where they disagree with the glyphs the fonts pick through their own OpenType rules: - HEH (U+0647) used U+FBAB, the fonts' medial glyph, for every form, so an initial or final heh got a stray connecting stroke. It now uses U+0647, U+06BE, U+FBAB and U+FEEA. - Isolated HAH (U+062D) used U+FEA1, which these fonts lack, so nothing was drawn. It now uses U+062D. - Isolated YEAH (U+06CE) used U+E004, the fonts' final form. It now uses U+06CE. - REH WITH SMALL V BELOW (U+0695) was missing. It is right-joining, with its final form at U+E001. Add test_006_kurdish_font_glyphs.py, which shapes each Kurdish letter in every position with HarfBuzz using a font's own rules and compares the glyph with the one arabic_reshaper's output maps to. It runs on Kurdish fonts placed under fonts/ and is skipped otherwise. Add string tests for the fixed entries so CI covers them without the fonts. Add Python 3.15 to the CI and release test matrices and the classifiers. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 2 +- .github/workflows/release.yml | 2 +- arabic_reshaper/letters.py | 8 +- arabic_reshaper/tests/test_005_kurdish.py | 44 +++++ .../tests/test_006_kurdish_font_glyphs.py | 167 ++++++++++++++++++ pyproject.toml | 1 + 6 files changed, 219 insertions(+), 5 deletions(-) create mode 100644 arabic_reshaper/tests/test_006_kurdish_font_glyphs.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4bbf916..3cc3f63 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 950afc1..7df40ce 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,7 +10,7 @@ jobs: strategy: fail-fast: true matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"] steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v5 diff --git a/arabic_reshaper/letters.py b/arabic_reshaper/letters.py index 7ffe780..1fb8186 100644 --- a/arabic_reshaper/letters.py +++ b/arabic_reshaper/letters.py @@ -369,7 +369,7 @@ # ARABIC LETTER JEEM '\u062C': ('\u062C', '\uFE9F', '\uFEA0', '\uFE9E'), # ARABIC LETTER HAH - '\u062D': ('\uFEA1', '\uFEA3', '\uFEA4', '\uFEA2'), + '\u062D': ('\u062D', '\uFEA3', '\uFEA4', '\uFEA2'), # ARABIC LETTER KHAH '\u062E': ('\u062E', '\uFEA7', '\uFEA8', '\uFEA6'), # ARABIC LETTER DAL @@ -411,7 +411,7 @@ # ARABIC LETTER NOON '\u0646': ('\u0646', '\uFEE7', '\uFEE8', '\uFEE6'), # ARABIC LETTER HEH - '\u0647': ('\uFBAB', '\uFBAB', '\uFBAB', '\uFBAB'), + '\u0647': ('\u0647', '\u06BE', '\uFBAB', '\uFEEA'), # ARABIC LETTER WAW '\u0648': ('\u0648', '', '', '\uFEEE'), # ARABIC LETTER (UIGHUR KAZAKH KIRGHIZ)? ALEF MAKSURA @@ -452,6 +452,8 @@ '\u068E': ('\u068E', '', '', '\uFB87'), # ARABIC LETTER RREH '\u0691': ('\u0691', '', '', '\uFB8D'), + # KURDISH LETTER REH WITH SMALL V BELOW + '\u0695': ('\u0695', '', '', '\uE001'), # ARABIC LETTER JEH '\u0698': ('\u0698', '', '', '\uFB8B'), # ARABIC LETTER VEH @@ -501,7 +503,7 @@ # ARABIC LETTER YEH BARREE WITH HAMZA ABOVE '\u06D3': ('\u06D3', '', '', '\uFBB1'), # Kurdish letter YEAH - '\u06ce': ('\uE004', '\uE005', '\uE006', '\uE004'), + '\u06ce': ('\u06ce', '\uE005', '\uE006', '\uE004'), # Kurdish letter Hamza same as arabic Teh without the point '\u06d5': ('\u06d5', '', '', '\uE000'), # ZWJ diff --git a/arabic_reshaper/tests/test_005_kurdish.py b/arabic_reshaper/tests/test_005_kurdish.py index d1ab76d..caa39ac 100644 --- a/arabic_reshaper/tests/test_005_kurdish.py +++ b/arabic_reshaper/tests/test_005_kurdish.py @@ -42,5 +42,49 @@ def test_words(self): self.assertEqual(self.reshaper.reshape(text), expected) +class TestKurdishFontForms(unittest.TestCase): + """Forms checked against the Sarchia and Unikurd Web fonts, which draw + them from these code points. See test_006_kurdish_font_glyphs.py.""" + + def setUp(self): + self.reshaper = arabic_reshaper.ArabicReshaper( + configuration={'language': 'Kurdish'} + ) + + def check(self, cases): + for text, expected in cases: + with self.subTest(text=text): + self.assertEqual(self.reshaper.reshape(text), expected) + + def test_heh(self): + self.check(( + ('\u0647', '\u0647'), + ('\u0647\u0645', '\u06BE\uFEE2'), + ('\u0645\u0647\u0645', '\uFEE3\uFBAB\uFEE2'), + ('\u0645\u0647', '\uFEE3\uFEEA'), + # هەولێر (Erbil): initial heh before ae + ('\u0647\u06D5\u0648\u0644\u06CE\u0631', + '\u06BE\uE000\u0648\uFEDF\uE006\uFEAE'), + )) + + def test_isolated_hah_and_yeah(self): + # Kurdish fonts have no glyph at U+FEA1, and U+E004 is the final + # form of yeah, so the isolated forms use the base letters. + self.check(( + ('\u062D', '\u062D'), + ('\u06CE', '\u06CE'), + )) + + def test_reh_with_small_v_below(self): + self.check(( + ('\u0695', '\u0695'), + ('\u0645\u0695', '\uFEE3\uE001'), + # does not join the following letter + ('\u0645\u0695\u0645', '\uFEE3\uE001\u0645'), + # کوڕ (boy): isolated, after waw + ('\u06A9\u0648\u0695', '\uFB90\uFEEE\u0695'), + )) + + if __name__ == '__main__': unittest.main() diff --git a/arabic_reshaper/tests/test_006_kurdish_font_glyphs.py b/arabic_reshaper/tests/test_006_kurdish_font_glyphs.py new file mode 100644 index 0000000..6e58ca4 --- /dev/null +++ b/arabic_reshaper/tests/test_006_kurdish_font_glyphs.py @@ -0,0 +1,167 @@ +"""Check the Kurdish letter table against real Kurdish fonts. + +Kurdish fonts such as Sarchia and Unikurd Web draw some contextual forms +from private-use code points (U+E000-U+E009) or from presentation-form code +points they use in their own way, and they leave out many standard +presentation forms. ``LETTERS_KURDISH`` has to match those fonts, and a test +on strings alone cannot show that the right glyph appears on screen. + +This test uses the font itself. These fonts also have OpenType (GSUB) rules +that pick the right contextual glyph for the plain Unicode letter. For every +letter and every position (isolated, initial, medial, final) it checks that: + +* HarfBuzz shaping the *unshaped* text with the font's own rules, and +* the font's cmap glyph for the code point arabic_reshaper outputs + +give the same glyph (same glyph, or an identical outline and advance width). + +Put Kurdish fonts (for example ``85_Sarchia_Soran.ttf`` or Unikurd Web's +``Unikuweb.ttf``) anywhere under ``fonts/`` at the repository root. That +directory is git-ignored. The test runs only on fonts that use the Kurdish +private-use convention, and skips letters a font doesn't have. +""" +from pathlib import Path + +import pytest + +pytest.importorskip('fontTools', reason='fonttools not installed') +hb = pytest.importorskip('uharfbuzz', reason='uharfbuzz not installed') + +from fontTools.pens.recordingPen import RecordingPen +from fontTools.ttLib import TTFont + +from arabic_reshaper import ArabicReshaper +from arabic_reshaper.letters import FINAL, INITIAL, ISOLATED, LETTERS_KURDISH, MEDIAL + +_FONTS_DIR = Path(__file__).parent.parent.parent / 'fonts' +_ALL_FONTS = sorted( + p for ext in ('*.ttf', '*.otf') for p in _FONTS_DIR.rglob(ext) +) if _FONTS_DIR.is_dir() else [] + +# The Sorani Kurdish alphabet. Tested even when a letter is missing from +# LETTERS_KURDISH, so a missing entry fails instead of going unnoticed. +SORANI_ALPHABET = ( + 'ئابپتجچحخدرڕزژسشعغفڤقکگلڵمنوۆهەیێ' +) + +MEEM = 'م' # dual-joining neighbour present in every Arabic-script font +FORM_NAMES = {ISOLATED: 'isolated', INITIAL: 'initial', + MEDIAL: 'medial', FINAL: 'final'} + + +def _contexts(letter): + """(form, text, index of the letter in the text) for every position.""" + return [ + (ISOLATED, letter, 0), + (INITIAL, letter + MEEM, 0), + (MEDIAL, MEEM + letter + MEEM, 1), + (FINAL, MEEM + letter, 1), + ] + + +def _is_kurdish_pua_font(cmap): + # Kurdish fonts put the final form of AE (U+06D5) at U+E000. + return 0x06D5 in cmap and 0xE000 in cmap + + +def _collect_cases(): + cases = [] + letters = sorted(set(LETTERS_KURDISH) | set(SORANI_ALPHABET)) + for path in _ALL_FONTS: + cmap = TTFont(str(path), lazy=True).getBestCmap() or {} + if not _is_kurdish_pua_font(cmap): + continue + for letter in letters: + if ord(letter) not in cmap: + continue + for form, text, index in _contexts(letter): + cases.append(pytest.param( + path, letter, form, text, index, + id=f'{path.stem}-U+{ord(letter):04X}-{FORM_NAMES[form]}', + )) + return cases + + +_CASES = _collect_cases() + + +class _Font: + _cache = {} + + @classmethod + def load(cls, path): + if path not in cls._cache: + cls._cache[path] = cls(path) + return cls._cache[path] + + def __init__(self, path): + self.name = path.name + self.tt = TTFont(str(path)) + self.cmap = self.tt.getBestCmap() + self.order = self.tt.getGlyphOrder() + self.glyph_set = self.tt.getGlyphSet() + self.hb_font = hb.Font(hb.Face(hb.Blob.from_file_path(str(path)))) + + def shaped_glyph_at(self, text, index): + """Glyph name HarfBuzz picks for the character at ``index``.""" + buf = hb.Buffer() + buf.add_str(text) + buf.guess_segment_properties() + # Turn off optional ligatures so every character keeps its own glyph. + hb.shape(self.hb_font, buf, {'liga': False, 'dlig': False}) + names = [self.order[info.codepoint] + for info in buf.glyph_infos if info.cluster == index] + assert len(names) == 1, ( + f'{self.name}: expected one glyph for {text!r}[{index}], got {names}' + ) + return names[0] + + def signature(self, glyph_name): + pen = RecordingPen() + self.glyph_set[glyph_name].draw(pen) + return self.tt['hmtx'][glyph_name][0], pen.value + + def same_glyph(self, a, b): + return a == b or self.signature(a) == self.signature(b) + + +@pytest.fixture(scope='module') +def reshaper(): + return ArabicReshaper(configuration={ + 'language': 'Kurdish', + 'support_ligatures': False, + }) + + +@pytest.mark.skipif( + not _CASES, + reason='No Kurdish font (with U+06D5 and U+E000) found under fonts/', +) +@pytest.mark.parametrize('font_path, letter, form, text, index', _CASES) +def test_letter_form_matches_font(font_path, reshaper, letter, form, text, + index): + font = _Font.load(font_path) + expected_glyph = font.shaped_glyph_at(text, index) + + reshaped = reshaper.reshape(text) + assert len(reshaped) == len(text), repr(reshaped) + out_char = reshaped[index] + mapped_glyph = font.cmap.get(ord(out_char)) + + if mapped_glyph is not None and font.same_glyph(mapped_glyph, expected_glyph): + return + + if form != ISOLATED and expected_glyph == font.cmap[ord(letter)]: + # The font keeps the plain glyph here, so it defines no contextual + # form for this letter and says nothing about what the right one is. + pytest.skip(f'{font.name} has no {FORM_NAMES[form]} form ' + f'for U+{ord(letter):04X}') + + assert mapped_glyph is not None, ( + f'{font.name}: reshaper outputs U+{ord(out_char):04X} for {text!r}, ' + f'which the font has no glyph for; the font uses {expected_glyph!r}' + ) + pytest.fail( + f'{font.name}: reshaper outputs U+{ord(out_char):04X} ' + f'({mapped_glyph!r}) for {text!r}, but the font uses {expected_glyph!r}' + ) diff --git a/pyproject.toml b/pyproject.toml index 5850dbe..996416d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ classifiers = [ "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", "Topic :: Software Development :: Libraries :: Python Modules", ] requires-python = ">=3.10" From 4268ddbae10f93238ecfb754aba4927de81b17ac Mon Sep 17 00:00:00 2001 From: Abdullah Diab Date: Mon, 2 Nov 2026 15:57:06 +0400 Subject: [PATCH 2/2] Bump version to 3.1.0 3.0.1 is released. Since then the Kurdish table gained lam with small v and reh with small v below, several Kurdish forms changed, and Python 3.15 is supported, so this is a minor release. - pyproject.toml: 3.0.1 -> 3.1.0 - uv.lock: was still at 3.0.0, now 3.1.0 - __version__ was still 3.0.0 and defined twice; __init__ now imports it from __version__.py so there is one copy to bump. Co-Authored-By: Claude Opus 5.5 --- arabic_reshaper/__init__.py | 2 +- arabic_reshaper/__version__.py | 2 +- pyproject.toml | 2 +- uv.lock | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/arabic_reshaper/__init__.py b/arabic_reshaper/__init__.py index 9c49a4d..d646904 100644 --- a/arabic_reshaper/__init__.py +++ b/arabic_reshaper/__init__.py @@ -8,4 +8,4 @@ ENABLE_ALL_LIGATURES) -__version__ = '3.0.0' +from .__version__ import __version__ diff --git a/arabic_reshaper/__version__.py b/arabic_reshaper/__version__.py index 4eb28e3..7f5601d 100644 --- a/arabic_reshaper/__version__.py +++ b/arabic_reshaper/__version__.py @@ -1 +1 @@ -__version__ = '3.0.0' +__version__ = '3.1.0' diff --git a/pyproject.toml b/pyproject.toml index 996416d..c2b9a0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "arabic-reshaper" -version = "3.0.1" +version = "3.1.0" description = "Reconstruct Arabic sentences to be used in applications that do not support Arabic" readme = "README.md" license = { text = "MIT" } diff --git a/uv.lock b/uv.lock index 8adf8fe..2b10946 100644 --- a/uv.lock +++ b/uv.lock @@ -4,7 +4,7 @@ requires-python = ">=3.10" [[package]] name = "arabic-reshaper" -version = "3.0.0" +version = "3.1.0" source = { editable = "." } [package.optional-dependencies]