diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4bbf916..3cc3f63 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 950afc1..7df40ce 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,7 +10,7 @@ jobs: strategy: fail-fast: true matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"] steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v5 diff --git a/arabic_reshaper/__init__.py b/arabic_reshaper/__init__.py index 9c49a4d..d646904 100644 --- a/arabic_reshaper/__init__.py +++ b/arabic_reshaper/__init__.py @@ -8,4 +8,4 @@ ENABLE_ALL_LIGATURES) -__version__ = '3.0.0' +from .__version__ import __version__ diff --git a/arabic_reshaper/__version__.py b/arabic_reshaper/__version__.py index 4eb28e3..7f5601d 100644 --- a/arabic_reshaper/__version__.py +++ b/arabic_reshaper/__version__.py @@ -1 +1 @@ -__version__ = '3.0.0' +__version__ = '3.1.0' diff --git a/arabic_reshaper/letters.py b/arabic_reshaper/letters.py index 7ffe780..1fb8186 100644 --- a/arabic_reshaper/letters.py +++ b/arabic_reshaper/letters.py @@ -369,7 +369,7 @@ # ARABIC LETTER JEEM '\u062C': ('\u062C', '\uFE9F', '\uFEA0', '\uFE9E'), # ARABIC LETTER HAH - '\u062D': ('\uFEA1', '\uFEA3', '\uFEA4', '\uFEA2'), + '\u062D': ('\u062D', '\uFEA3', '\uFEA4', '\uFEA2'), # ARABIC LETTER KHAH '\u062E': ('\u062E', '\uFEA7', '\uFEA8', '\uFEA6'), # ARABIC LETTER DAL @@ -411,7 +411,7 @@ # ARABIC LETTER NOON '\u0646': ('\u0646', '\uFEE7', '\uFEE8', '\uFEE6'), # ARABIC LETTER HEH - '\u0647': ('\uFBAB', '\uFBAB', '\uFBAB', '\uFBAB'), + '\u0647': ('\u0647', '\u06BE', '\uFBAB', '\uFEEA'), # ARABIC LETTER WAW '\u0648': ('\u0648', '', '', '\uFEEE'), # ARABIC LETTER (UIGHUR KAZAKH KIRGHIZ)? ALEF MAKSURA @@ -452,6 +452,8 @@ '\u068E': ('\u068E', '', '', '\uFB87'), # ARABIC LETTER RREH '\u0691': ('\u0691', '', '', '\uFB8D'), + # KURDISH LETTER REH WITH SMALL V BELOW + '\u0695': ('\u0695', '', '', '\uE001'), # ARABIC LETTER JEH '\u0698': ('\u0698', '', '', '\uFB8B'), # ARABIC LETTER VEH @@ -501,7 +503,7 @@ # ARABIC LETTER YEH BARREE WITH HAMZA ABOVE '\u06D3': ('\u06D3', '', '', '\uFBB1'), # Kurdish letter YEAH - '\u06ce': ('\uE004', '\uE005', '\uE006', '\uE004'), + '\u06ce': ('\u06ce', '\uE005', '\uE006', '\uE004'), # Kurdish letter Hamza same as arabic Teh without the point '\u06d5': ('\u06d5', '', '', '\uE000'), # ZWJ diff --git a/arabic_reshaper/tests/test_005_kurdish.py b/arabic_reshaper/tests/test_005_kurdish.py index d1ab76d..caa39ac 100644 --- a/arabic_reshaper/tests/test_005_kurdish.py +++ b/arabic_reshaper/tests/test_005_kurdish.py @@ -42,5 +42,49 @@ def test_words(self): self.assertEqual(self.reshaper.reshape(text), expected) +class TestKurdishFontForms(unittest.TestCase): + """Forms checked against the Sarchia and Unikurd Web fonts, which draw + them from these code points. See test_006_kurdish_font_glyphs.py.""" + + def setUp(self): + self.reshaper = arabic_reshaper.ArabicReshaper( + configuration={'language': 'Kurdish'} + ) + + def check(self, cases): + for text, expected in cases: + with self.subTest(text=text): + self.assertEqual(self.reshaper.reshape(text), expected) + + def test_heh(self): + self.check(( + ('\u0647', '\u0647'), + ('\u0647\u0645', '\u06BE\uFEE2'), + ('\u0645\u0647\u0645', '\uFEE3\uFBAB\uFEE2'), + ('\u0645\u0647', '\uFEE3\uFEEA'), + # هەولێر (Erbil): initial heh before ae + ('\u0647\u06D5\u0648\u0644\u06CE\u0631', + '\u06BE\uE000\u0648\uFEDF\uE006\uFEAE'), + )) + + def test_isolated_hah_and_yeah(self): + # Kurdish fonts have no glyph at U+FEA1, and U+E004 is the final + # form of yeah, so the isolated forms use the base letters. + self.check(( + ('\u062D', '\u062D'), + ('\u06CE', '\u06CE'), + )) + + def test_reh_with_small_v_below(self): + self.check(( + ('\u0695', '\u0695'), + ('\u0645\u0695', '\uFEE3\uE001'), + # does not join the following letter + ('\u0645\u0695\u0645', '\uFEE3\uE001\u0645'), + # کوڕ (boy): isolated, after waw + ('\u06A9\u0648\u0695', '\uFB90\uFEEE\u0695'), + )) + + if __name__ == '__main__': unittest.main() diff --git a/arabic_reshaper/tests/test_006_kurdish_font_glyphs.py b/arabic_reshaper/tests/test_006_kurdish_font_glyphs.py new file mode 100644 index 0000000..6e58ca4 --- /dev/null +++ b/arabic_reshaper/tests/test_006_kurdish_font_glyphs.py @@ -0,0 +1,167 @@ +"""Check the Kurdish letter table against real Kurdish fonts. + +Kurdish fonts such as Sarchia and Unikurd Web draw some contextual forms +from private-use code points (U+E000-U+E009) or from presentation-form code +points they use in their own way, and they leave out many standard +presentation forms. ``LETTERS_KURDISH`` has to match those fonts, and a test +on strings alone cannot show that the right glyph appears on screen. + +This test uses the font itself. These fonts also have OpenType (GSUB) rules +that pick the right contextual glyph for the plain Unicode letter. For every +letter and every position (isolated, initial, medial, final) it checks that: + +* HarfBuzz shaping the *unshaped* text with the font's own rules, and +* the font's cmap glyph for the code point arabic_reshaper outputs + +give the same glyph (same glyph, or an identical outline and advance width). + +Put Kurdish fonts (for example ``85_Sarchia_Soran.ttf`` or Unikurd Web's +``Unikuweb.ttf``) anywhere under ``fonts/`` at the repository root. That +directory is git-ignored. The test runs only on fonts that use the Kurdish +private-use convention, and skips letters a font doesn't have. +""" +from pathlib import Path + +import pytest + +pytest.importorskip('fontTools', reason='fonttools not installed') +hb = pytest.importorskip('uharfbuzz', reason='uharfbuzz not installed') + +from fontTools.pens.recordingPen import RecordingPen +from fontTools.ttLib import TTFont + +from arabic_reshaper import ArabicReshaper +from arabic_reshaper.letters import FINAL, INITIAL, ISOLATED, LETTERS_KURDISH, MEDIAL + +_FONTS_DIR = Path(__file__).parent.parent.parent / 'fonts' +_ALL_FONTS = sorted( + p for ext in ('*.ttf', '*.otf') for p in _FONTS_DIR.rglob(ext) +) if _FONTS_DIR.is_dir() else [] + +# The Sorani Kurdish alphabet. Tested even when a letter is missing from +# LETTERS_KURDISH, so a missing entry fails instead of going unnoticed. +SORANI_ALPHABET = ( + 'ئابپتجچحخدرڕزژسشعغفڤقکگلڵمنوۆهەیێ' +) + +MEEM = 'م' # dual-joining neighbour present in every Arabic-script font +FORM_NAMES = {ISOLATED: 'isolated', INITIAL: 'initial', + MEDIAL: 'medial', FINAL: 'final'} + + +def _contexts(letter): + """(form, text, index of the letter in the text) for every position.""" + return [ + (ISOLATED, letter, 0), + (INITIAL, letter + MEEM, 0), + (MEDIAL, MEEM + letter + MEEM, 1), + (FINAL, MEEM + letter, 1), + ] + + +def _is_kurdish_pua_font(cmap): + # Kurdish fonts put the final form of AE (U+06D5) at U+E000. + return 0x06D5 in cmap and 0xE000 in cmap + + +def _collect_cases(): + cases = [] + letters = sorted(set(LETTERS_KURDISH) | set(SORANI_ALPHABET)) + for path in _ALL_FONTS: + cmap = TTFont(str(path), lazy=True).getBestCmap() or {} + if not _is_kurdish_pua_font(cmap): + continue + for letter in letters: + if ord(letter) not in cmap: + continue + for form, text, index in _contexts(letter): + cases.append(pytest.param( + path, letter, form, text, index, + id=f'{path.stem}-U+{ord(letter):04X}-{FORM_NAMES[form]}', + )) + return cases + + +_CASES = _collect_cases() + + +class _Font: + _cache = {} + + @classmethod + def load(cls, path): + if path not in cls._cache: + cls._cache[path] = cls(path) + return cls._cache[path] + + def __init__(self, path): + self.name = path.name + self.tt = TTFont(str(path)) + self.cmap = self.tt.getBestCmap() + self.order = self.tt.getGlyphOrder() + self.glyph_set = self.tt.getGlyphSet() + self.hb_font = hb.Font(hb.Face(hb.Blob.from_file_path(str(path)))) + + def shaped_glyph_at(self, text, index): + """Glyph name HarfBuzz picks for the character at ``index``.""" + buf = hb.Buffer() + buf.add_str(text) + buf.guess_segment_properties() + # Turn off optional ligatures so every character keeps its own glyph. + hb.shape(self.hb_font, buf, {'liga': False, 'dlig': False}) + names = [self.order[info.codepoint] + for info in buf.glyph_infos if info.cluster == index] + assert len(names) == 1, ( + f'{self.name}: expected one glyph for {text!r}[{index}], got {names}' + ) + return names[0] + + def signature(self, glyph_name): + pen = RecordingPen() + self.glyph_set[glyph_name].draw(pen) + return self.tt['hmtx'][glyph_name][0], pen.value + + def same_glyph(self, a, b): + return a == b or self.signature(a) == self.signature(b) + + +@pytest.fixture(scope='module') +def reshaper(): + return ArabicReshaper(configuration={ + 'language': 'Kurdish', + 'support_ligatures': False, + }) + + +@pytest.mark.skipif( + not _CASES, + reason='No Kurdish font (with U+06D5 and U+E000) found under fonts/', +) +@pytest.mark.parametrize('font_path, letter, form, text, index', _CASES) +def test_letter_form_matches_font(font_path, reshaper, letter, form, text, + index): + font = _Font.load(font_path) + expected_glyph = font.shaped_glyph_at(text, index) + + reshaped = reshaper.reshape(text) + assert len(reshaped) == len(text), repr(reshaped) + out_char = reshaped[index] + mapped_glyph = font.cmap.get(ord(out_char)) + + if mapped_glyph is not None and font.same_glyph(mapped_glyph, expected_glyph): + return + + if form != ISOLATED and expected_glyph == font.cmap[ord(letter)]: + # The font keeps the plain glyph here, so it defines no contextual + # form for this letter and says nothing about what the right one is. + pytest.skip(f'{font.name} has no {FORM_NAMES[form]} form ' + f'for U+{ord(letter):04X}') + + assert mapped_glyph is not None, ( + f'{font.name}: reshaper outputs U+{ord(out_char):04X} for {text!r}, ' + f'which the font has no glyph for; the font uses {expected_glyph!r}' + ) + pytest.fail( + f'{font.name}: reshaper outputs U+{ord(out_char):04X} ' + f'({mapped_glyph!r}) for {text!r}, but the font uses {expected_glyph!r}' + ) diff --git a/pyproject.toml b/pyproject.toml index 5850dbe..c2b9a0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "arabic-reshaper" -version = "3.0.1" +version = "3.1.0" description = "Reconstruct Arabic sentences to be used in applications that do not support Arabic" readme = "README.md" license = { text = "MIT" } @@ -24,6 +24,7 @@ classifiers = [ "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", "Topic :: Software Development :: Libraries :: Python Modules", ] requires-python = ">=3.10" diff --git a/uv.lock b/uv.lock index 8adf8fe..2b10946 100644 --- a/uv.lock +++ b/uv.lock @@ -4,7 +4,7 @@ requires-python = ">=3.10" [[package]] name = "arabic-reshaper" -version = "3.0.0" +version = "3.1.0" source = { editable = "." } [package.optional-dependencies]