From f9648ba5e1b77d7d15e98ce6998c2c33c3c8af56 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 7 Sep 2026 14:47:54 +0200 Subject: [PATCH 1/4] gh-155292: Skip updating unicodedata with mismatched interpreter mkstringprep uses things like str.lower(), so it generates the wrong result if run in an interpreter with a different Unicode data version than the target. This means that updating the Unicode version is a two-step process: run makeunicodedata.py, then compile, then run mkstringprep.py. The two steps can (and should) be combined when re-running regen-unicodedata to verify that the data is up to date. The GH-155292 fix only considered that case. Change makeunicodedata.py to only run mkstringprep.py when the current interpreter is up to it. Otherwise, show a reminder. As an extra complication, download the input (RFC 3454) in the "first step", since an out-of-date stringprep.py's freshness assertion may prevent downloads. --- Tools/unicode/makeunicodedata.py | 32 +++++++++++++++++++++++++++++--- Tools/unicode/mkstringprep.py | 13 +------------ 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/Tools/unicode/makeunicodedata.py b/Tools/unicode/makeunicodedata.py index ab453686d4dfb1a..43f9f41038f09f1 100644 --- a/Tools/unicode/makeunicodedata.py +++ b/Tools/unicode/makeunicodedata.py @@ -816,10 +816,31 @@ def makeunicodename(unicode, trace): def makestringprep(): FILE = "Lib/stringprep.py" + RFC_LOCAL = os.path.join(DATA_DIR, "rfc3454.txt") + RFC_URL = "https://www.rfc-editor.org/rfc/rfc3454.txt" + print("--- Preparing", FILE, "...") + # mkstringprep expects a local copy of RFC 3454. Download it now. + # (stringprep is used for URL handling, and if it's not matched + # with the compiled unicodedata, downloads would fail.) + if not os.path.exists(RFC_LOCAL): + download_data(RFC_LOCAL, RFC_URL) + MKSTRINGPREP = "Tools/unicode/mkstringprep.py" + # mkstringprep needs to be run with a Python version that has "its" + # unicode data, since it uses str.lower() and similar. + import unicodedata + if unicodedata.unidata_version != UNIDATA_VERSION: + print() + print("!! Skipping mkstringprep -- mimatched Unicode version !!") + print() + print("Please compile CPython with the updated Unicode database,") + print("then use that interpreter to run:") + print(f" python {MKSTRINGPREP} > {FILE}") + return + with open(FILE, "w") as f: f.truncate() subprocess.check_call([sys.executable, MKSTRINGPREP], stdout=f) @@ -934,14 +955,12 @@ class Difference(Exception):pass def open_data(template, version): local = os.path.join(DATA_DIR, template % ('-'+version,)) if not os.path.exists(local): - import urllib.request if version == '3.2.0': # irregular url structure url = ('https://www.unicode.org/Public/3.2-Update/'+template) % ('-'+version,) else: url = ('https://www.unicode.org/Public/%s/ucd/'+template) % (version, '') - os.makedirs(os.path.dirname(local), exist_ok=True) - urllib.request.urlretrieve(url, filename=local) + download_data(local, url) if local.endswith('.txt'): return open(local, encoding='utf-8') else: @@ -949,6 +968,13 @@ def open_data(template, version): return open(local, 'rb') +def download_data(local, url): + import urllib.request + os.makedirs(os.path.dirname(local), exist_ok=True) + print(f'Downloading {url} to {local}') + urllib.request.urlretrieve(url, filename=local) + + def expand_range(char_range: str) -> Iterator[int]: ''' Parses ranges of code points, as described in UAX #44: diff --git a/Tools/unicode/mkstringprep.py b/Tools/unicode/mkstringprep.py index 9740338fef9d3d4..9e3222a32c08d8c 100644 --- a/Tools/unicode/mkstringprep.py +++ b/Tools/unicode/mkstringprep.py @@ -3,9 +3,6 @@ import unicodedata as unicodedata_current from unicodedata import ucd_3_2_0 as unicodedata_320 -FILENAME = "Tools/unicode/data/rfc3454.txt" -URL = "https://www.rfc-editor.org/rfc/rfc3454.txt" - def gen_category(cats): for i in range(0, 0x110000): if unicodedata_320.category(chr(i)) in cats: @@ -52,15 +49,7 @@ def compact_set(l): ############## Read the tables in the RFC ####################### -try: - data_file = open(FILENAME, encoding='utf-8') -except FileNotFoundError: - import urllib.request - os.makedirs(os.path.dirname(FILENAME), exist_ok=True) - urllib.request.urlretrieve(URL, filename=FILENAME) - data_file = open(FILENAME, encoding='utf-8') - -with data_file: +with open("Tools/unicode/data/rfc3454.txt", encoding='utf-8') as data_file: data = data_file.readlines() tables = [] From 3f0b50f879d5aa40369ed1143b714269739eb7db Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 7 Sep 2026 15:06:07 +0200 Subject: [PATCH 2/4] Move the now-shared DATA_DIR up --- Tools/unicode/makeunicodedata.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Tools/unicode/makeunicodedata.py b/Tools/unicode/makeunicodedata.py index 43f9f41038f09f1..32019b35c9f62bf 100644 --- a/Tools/unicode/makeunicodedata.py +++ b/Tools/unicode/makeunicodedata.py @@ -40,6 +40,9 @@ SCRIPT = os.path.normpath(sys.argv[0]) VERSION = "3.3" +# Local cache location +DATA_DIR = os.path.join('Tools', 'unicode', 'data') + # The Unicode Database # -------------------- # When changing UCD version please update @@ -950,8 +953,6 @@ class Difference(Exception):pass normalization_changes)) -DATA_DIR = os.path.join('Tools', 'unicode', 'data') - def open_data(template, version): local = os.path.join(DATA_DIR, template % ('-'+version,)) if not os.path.exists(local): From 4d5d35a91cdf6929b4c01e61c58dc4e5448490f4 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 7 Sep 2026 15:13:23 +0200 Subject: [PATCH 3/4] Add blurb --- .../next/Build/2026-09-07-15-13-19.gh-issue-155292._WNxL7.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Build/2026-09-07-15-13-19.gh-issue-155292._WNxL7.rst diff --git a/Misc/NEWS.d/next/Build/2026-09-07-15-13-19.gh-issue-155292._WNxL7.rst b/Misc/NEWS.d/next/Build/2026-09-07-15-13-19.gh-issue-155292._WNxL7.rst new file mode 100644 index 000000000000000..daa9746938d3b6b --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-09-07-15-13-19.gh-issue-155292._WNxL7.rst @@ -0,0 +1,3 @@ +In ``makeunicodedata.py``, the script for updating Unicode data, skip +generating ``stringprep.py`` if the current interpreter's Unicode data +version does not match the target version. From 7d844dfa8ca3c9520c2abe059ffcbf5d8f1cadfd Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 7 Sep 2026 15:18:39 +0200 Subject: [PATCH 4/4] Remove unused import --- Tools/unicode/mkstringprep.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Tools/unicode/mkstringprep.py b/Tools/unicode/mkstringprep.py index 9e3222a32c08d8c..60052aae3b62983 100644 --- a/Tools/unicode/mkstringprep.py +++ b/Tools/unicode/mkstringprep.py @@ -1,5 +1,4 @@ import re -import os import unicodedata as unicodedata_current from unicodedata import ucd_3_2_0 as unicodedata_320