From d8e251386b04f382fafdf25ac9cfed87ec19a326 Mon Sep 17 00:00:00 2001 From: Don Santos Date: Mon, 24 Aug 2026 16:39:08 +0200 Subject: [PATCH 1/2] Avoid quadratic canonical combining-class ordering --- test/misc.c | 40 +++++++++++++ utf8proc.c | 161 +++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 174 insertions(+), 27 deletions(-) diff --git a/test/misc.c b/test/misc.c index 7ea2ebc..c07421a 100644 --- a/test/misc.c +++ b/test/misc.c @@ -44,11 +44,51 @@ static void issue317(void) /* #317 */ } } +static void canonical_order_long(void) +{ + enum { groups = 64 }; + utf8proc_uint8_t input[groups * 8 + 2]; + utf8proc_uint8_t *output; + size_t i, pos = 0; + + check(utf8proc_get_property(0x0301)->combining_class == 230, "unexpected combining class for U+0301"); + check(utf8proc_get_property(0x0307)->combining_class == 230, "unexpected combining class for U+0307"); + check(utf8proc_get_property(0x0327)->combining_class == 202, "unexpected combining class for U+0327"); + + input[pos++] = 'a'; + for (i = 0; i < groups; ++i) { + input[pos++] = 0xcc; input[pos++] = 0x81; /* U+0301, CCC 230 */ + input[pos++] = 0xcc; input[pos++] = 0xa7; /* U+0327, CCC 202 */ + input[pos++] = 0xcc; input[pos++] = 0x87; /* U+0307, CCC 230 */ + input[pos++] = 0xcc; input[pos++] = 0xa7; /* U+0327, CCC 202 */ + } + input[pos] = 0; + + output = utf8proc_NFD(input); + check(output != NULL, "NFD allocation failed"); + + pos = 0; + check(output[pos++] == 'a', "starter changed during canonical ordering"); + for (i = 0; i < groups * 2; ++i) { + check(output[pos++] == 0xcc && output[pos++] == 0xa7, + "CCC 202 mark not ordered before CCC 230 marks"); + } + for (i = 0; i < groups; ++i) { + check(output[pos++] == 0xcc && output[pos++] == 0x81, + "equal-CCC order changed for U+0301"); + check(output[pos++] == 0xcc && output[pos++] == 0x87, + "equal-CCC order changed for U+0307"); + } + check(output[pos] == 0, "unexpected bytes after reordered sequence"); + utf8proc_free(output); +} + int main(void) { issue128(); issue102(); issue317(); + canonical_order_long(); #ifdef UNICODE_VERSION printf("Unicode version: Makefile has %s, has API %s\n", UNICODE_VERSION, utf8proc_unicode_version()); check(!strcmp(UNICODE_VERSION, utf8proc_unicode_version()), "utf8proc_unicode_version mismatch"); diff --git a/utf8proc.c b/utf8proc.c index 8afb11a..21a8de0 100644 --- a/utf8proc.c +++ b/utf8proc.c @@ -537,6 +537,139 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_char(utf8proc_int32_t uc, return 1; } +static utf8proc_propval_t canonical_combining_class(utf8proc_int32_t uc) { + return unsafe_get_property(uc)->combining_class; +} + +static void canonical_order_reverse(utf8proc_int32_t *buffer, utf8proc_ssize_t first, utf8proc_ssize_t last) { + while (first < --last) { + utf8proc_int32_t temp = buffer[first]; + buffer[first++] = buffer[last]; + buffer[last] = temp; + } +} + +static utf8proc_ssize_t canonical_order_rotate(utf8proc_int32_t *buffer, utf8proc_ssize_t first, utf8proc_ssize_t middle, utf8proc_ssize_t last) { + utf8proc_ssize_t result = first + (last - middle); + + if (first == middle || middle == last) return result; + + canonical_order_reverse(buffer, first, middle); + canonical_order_reverse(buffer, middle, last); + canonical_order_reverse(buffer, first, last); + + return result; +} + +static utf8proc_ssize_t canonical_order_lower_bound(const utf8proc_int32_t *buffer, utf8proc_ssize_t first, utf8proc_ssize_t last, utf8proc_propval_t value) { + while (first < last) { + utf8proc_ssize_t middle = first + (last - first) / 2; + + if (canonical_combining_class(buffer[middle]) < value) + first = middle + 1; + else + last = middle; + } + + return first; +} + +static utf8proc_ssize_t canonical_order_upper_bound(const utf8proc_int32_t *buffer, utf8proc_ssize_t first, utf8proc_ssize_t last, utf8proc_propval_t value) { + while (first < last) { + utf8proc_ssize_t middle = first + (last - first) / 2; + + if (value < canonical_combining_class(buffer[middle])) + last = middle; + else + first = middle + 1; + } + + return first; +} + +static void canonical_order_merge(utf8proc_int32_t *buffer, utf8proc_ssize_t first, utf8proc_ssize_t middle, utf8proc_ssize_t last) { + utf8proc_ssize_t first_cut, second_cut, new_middle; + utf8proc_ssize_t first_length = middle - first; + utf8proc_ssize_t second_length = last - middle; + + if (first_length == 0 || second_length == 0) return; + if (canonical_combining_class(buffer[middle - 1]) <= canonical_combining_class(buffer[middle])) return; + + if (last - first == 2) { + if (canonical_combining_class(buffer[middle]) < canonical_combining_class(buffer[first])) { + utf8proc_int32_t temp = buffer[first]; + buffer[first] = buffer[middle]; + buffer[middle] = temp; + } + return; + } + + if (first_length > second_length) { + first_cut = first + first_length / 2; + second_cut = canonical_order_lower_bound(buffer, middle, last, canonical_combining_class(buffer[first_cut])); + } + else { + second_cut = middle + second_length / 2; + first_cut = canonical_order_upper_bound(buffer, first, middle, canonical_combining_class(buffer[second_cut])); + } + + new_middle = canonical_order_rotate(buffer, first_cut, middle, second_cut); + canonical_order_merge(buffer, first, first_cut, new_middle); + canonical_order_merge(buffer, new_middle, second_cut, last); +} + +/* Keep short common-case runs cheap; use a stable in-place merge for long runs + * so canonical ordering stays allocation-free without quadratic behavior. */ +static void canonical_order_sort(utf8proc_int32_t *buffer, utf8proc_ssize_t first, utf8proc_ssize_t last) { + utf8proc_ssize_t length = last - first; + + if (length < 2) return; + + if (length <= 16) { + utf8proc_ssize_t i; + + for (i = first + 1; i < last; ++i) { + utf8proc_int32_t current = buffer[i]; + utf8proc_propval_t current_class = canonical_combining_class(current); + utf8proc_ssize_t j = i; + + while (j > first && canonical_combining_class(buffer[j - 1]) > current_class) { + buffer[j] = buffer[j - 1]; + --j; + } + + buffer[j] = current; + } + } + else { + utf8proc_ssize_t middle = first + length / 2; + + canonical_order_sort(buffer, first, middle); + canonical_order_sort(buffer, middle, last); + canonical_order_merge(buffer, first, middle, last); + } +} + +static void canonical_order(utf8proc_int32_t *buffer, utf8proc_ssize_t length) { + utf8proc_ssize_t pos = 0; + + while (pos < length) { + utf8proc_ssize_t first; + + while (pos < length && + (buffer[pos] < 0 || canonical_combining_class(buffer[pos]) == 0)) + ++pos; + + first = pos; + + while (pos < length && buffer[pos] >= 0 && + canonical_combining_class(buffer[pos]) > 0) + ++pos; + + canonical_order_sort(buffer, first, pos); + } +} + UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose( const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options @@ -590,33 +723,7 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_custom( } } if ((options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) && bufsize >= wpos) { - utf8proc_ssize_t pos = 0; - while (pos < wpos-1) { - utf8proc_int32_t uc1, uc2; - const utf8proc_property_t *property1, *property2; - uc1 = buffer[pos]; - if (uc1 < 0) { - /* skip grapheme break */ - pos++; - continue; - } - uc2 = buffer[pos+1]; - if (uc2 < 0) { - /* cannot recombine; skip grapheme break */ - pos+=2; - continue; - } - property1 = unsafe_get_property(uc1); - property2 = unsafe_get_property(uc2); - if (property1->combining_class > property2->combining_class && - property2->combining_class > 0) { - buffer[pos] = uc2; - buffer[pos+1] = uc1; - if (pos > 0) pos--; else pos++; - } else { - pos++; - } - } + canonical_order(buffer, wpos); } return wpos; } From 6cafc399c334249fa843b37a2b744a60ffd40b5c Mon Sep 17 00:00:00 2001 From: Don Santos Date: Mon, 24 Aug 2026 17:49:40 +0200 Subject: [PATCH 2/2] Use stable radix ordering for long combining runs --- utf8proc.c | 82 +++++++++++++----------------------------------------- 1 file changed, 20 insertions(+), 62 deletions(-) diff --git a/utf8proc.c b/utf8proc.c index 21a8de0..03128d2 100644 --- a/utf8proc.c +++ b/utf8proc.c @@ -549,77 +549,36 @@ static void canonical_order_reverse(utf8proc_int32_t *buffer, utf8proc_ssize_t f } } -static utf8proc_ssize_t canonical_order_rotate(utf8proc_int32_t *buffer, utf8proc_ssize_t first, utf8proc_ssize_t middle, utf8proc_ssize_t last) { - utf8proc_ssize_t result = first + (last - middle); +static void canonical_order_rotate(utf8proc_int32_t *buffer, utf8proc_ssize_t first, utf8proc_ssize_t middle, utf8proc_ssize_t last) { - if (first == middle || middle == last) return result; + if (first == middle || middle == last) return; canonical_order_reverse(buffer, first, middle); canonical_order_reverse(buffer, middle, last); canonical_order_reverse(buffer, first, last); - - return result; -} - -static utf8proc_ssize_t canonical_order_lower_bound(const utf8proc_int32_t *buffer, utf8proc_ssize_t first, utf8proc_ssize_t last, utf8proc_propval_t value) { - while (first < last) { - utf8proc_ssize_t middle = first + (last - first) / 2; - - if (canonical_combining_class(buffer[middle]) < value) - first = middle + 1; - else - last = middle; - } - - return first; } -static utf8proc_ssize_t canonical_order_upper_bound(const utf8proc_int32_t *buffer, utf8proc_ssize_t first, utf8proc_ssize_t last, utf8proc_propval_t value) { - while (first < last) { - utf8proc_ssize_t middle = first + (last - first) / 2; - - if (value < canonical_combining_class(buffer[middle])) - last = middle; - else - first = middle + 1; - } - - return first; -} - -static void canonical_order_merge(utf8proc_int32_t *buffer, utf8proc_ssize_t first, utf8proc_ssize_t middle, utf8proc_ssize_t last) { - utf8proc_ssize_t first_cut, second_cut, new_middle; - utf8proc_ssize_t first_length = middle - first; - utf8proc_ssize_t second_length = last - middle; +/* Stable in-place partition by one combining-class bit. */ +static utf8proc_ssize_t canonical_order_partition(utf8proc_int32_t *buffer, utf8proc_ssize_t first, utf8proc_ssize_t last, unsigned int mask) { + utf8proc_ssize_t length = last - first; - if (first_length == 0 || second_length == 0) return; - if (canonical_combining_class(buffer[middle - 1]) <= canonical_combining_class(buffer[middle])) return; + if (length == 0) return first; + if (length == 1) + return first + ((canonical_combining_class(buffer[first]) & mask) == 0); - if (last - first == 2) { - if (canonical_combining_class(buffer[middle]) < canonical_combining_class(buffer[first])) { - utf8proc_int32_t temp = buffer[first]; - buffer[first] = buffer[middle]; - buffer[middle] = temp; - } - return; - } + { + utf8proc_ssize_t middle = first + length / 2; + utf8proc_ssize_t left = canonical_order_partition(buffer, first, middle, mask); + utf8proc_ssize_t right = canonical_order_partition(buffer, middle, last, mask); + utf8proc_ssize_t zeroes_right = right - middle; - if (first_length > second_length) { - first_cut = first + first_length / 2; - second_cut = canonical_order_lower_bound(buffer, middle, last, canonical_combining_class(buffer[first_cut])); - } - else { - second_cut = middle + second_length / 2; - first_cut = canonical_order_upper_bound(buffer, first, middle, canonical_combining_class(buffer[second_cut])); + canonical_order_rotate(buffer, left, middle, right); + return left + zeroes_right; } - - new_middle = canonical_order_rotate(buffer, first_cut, middle, second_cut); - canonical_order_merge(buffer, first, first_cut, new_middle); - canonical_order_merge(buffer, new_middle, second_cut, last); } -/* Keep short common-case runs cheap; use a stable in-place merge for long runs - * so canonical ordering stays allocation-free without quadratic behavior. */ +/* Canonical combining classes fit in 8 bits. Stable LSD radix passes keep + * equal-class marks in input order without heap allocation. */ static void canonical_order_sort(utf8proc_int32_t *buffer, utf8proc_ssize_t first, utf8proc_ssize_t last) { utf8proc_ssize_t length = last - first; @@ -642,11 +601,10 @@ static void canonical_order_sort(utf8proc_int32_t *buffer, utf8proc_ssize_t firs } } else { - utf8proc_ssize_t middle = first + length / 2; + unsigned int mask; - canonical_order_sort(buffer, first, middle); - canonical_order_sort(buffer, middle, last); - canonical_order_merge(buffer, first, middle, last); + for (mask = 1; mask <= 0x80; mask <<= 1) + canonical_order_partition(buffer, first, last, mask); } }