From 6a29bd5f6c93ae763b800b45ffb674cea0b85bad Mon Sep 17 00:00:00 2001 From: Da Shen Date: Thu, 20 Aug 2026 14:29:28 +0800 Subject: [PATCH 1/2] =?UTF-8?q?[1229]=20=E4=BB=BB=E5=8A=A1=E6=96=87?= =?UTF-8?q?=E6=A1=A3=EF=BC=9Apr=5F4399=20=E8=BF=81=E7=A7=BB=E8=AE=A1?= =?UTF-8?q?=E5=88=92=E4=B8=8E=E8=B6=8A=E7=95=8C=E8=AF=BB=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E8=AE=B0=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude --- devel/1229.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 devel/1229.md diff --git a/devel/1229.md b/devel/1229.md new file mode 100644 index 0000000000..6ae8c8ac6a --- /dev/null +++ b/devel/1229.md @@ -0,0 +1,42 @@ +# 1229 从 pr_4399 迁移 moebius 优化 + +从 `pr_4399` 分支挑选对用户可感知的优化迁移到主干侧分支。选择标准： +绝对收益够大（微秒级分配器压力优化不单独迁移）、或正确性修复。 +分支复用自早前性能调研，任务文档重建于此。 + +## 1 scheme 解析器三处越界读修复（迁移自 687b0d3d8） + +### What + +`moebius/moebius/data/scheme_der.cpp` 三处「先读后判界」的越界读改为 +循环顶先判界再读字符： + +1. `string_to_scheme_tree` 词元扫描分支：`ch= s[end_index]` 在缓冲区 + 末尾越界读一字节； +2. 引号串扫描分支：同款越界； +3. `unslash` 尾部 `ch= s[i]` 越界——同样改为循环顶读取。 + +另有两处顺带修正： + +- `string_to_scheme_tree` 的 `replace(s,"\015","")` 整串拷贝改为先探测 + 含 CR 才替换（启动加载全部 .scm 的必经路径，绝大多数文件无 CR）； +- `moebius/bench/Data/Scheme/block_bench.cpp` 资源路径失效修复 + （`bench/` → `moebius/bench/`，bench 目录迁移后路径未跟上）。 + +### Why + +越界读是未定义行为（ASAN 会报）；scheme 解析是启动加载全部 .scm +的必经路径。性能持平（block_bench 5.69→5.77 ns/char，噪声内）。 + +### 验证 + +- 新建 `moebius/tests/moebius/data/scheme_der_test.cpp`（9 用例：词元在 + 缓冲区末尾、末尾反斜杠、引号转义、未闭合引号、注释、quote 糖、 + CR 剔除、block 多表达式），随 commit 一并迁移 +- `xmake test "moebius_tests/*"` 全过 + +### 涉及文件 + +- `moebius/moebius/data/scheme_der.cpp` +- `moebius/bench/Data/Scheme/block_bench.cpp` +- `moebius/tests/moebius/data/scheme_der_test.cpp`（新建） From 914277498f18401dcc83fb7dd85915c4196852c6 Mon Sep 17 00:00:00 2001 From: Da Shen Date: Thu, 20 Aug 2026 14:29:29 +0800 Subject: [PATCH 2/2] =?UTF-8?q?[1229]=20[moebius]=20scheme=20=E8=A7=A3?= =?UTF-8?q?=E6=9E=90=E5=99=A8=E4=BF=AE=E5=A4=8D=E8=AF=8D=E5=85=83/?= =?UTF-8?q?=E5=BC=95=E5=8F=B7/unslash=20=E4=B8=89=E5=A4=84=E8=B6=8A?= =?UTF-8?q?=E7=95=8C=E8=AF=BB=EF=BC=8CCR=20=E6=9B=BF=E6=8D=A2=E6=94=B9?= =?UTF-8?q?=E5=85=88=E6=8E=A2=E6=B5=8B=EF=BC=88=E8=BF=81=E7=A7=BB=E8=87=AA?= =?UTF-8?q?=20pr=5F4399=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude --- moebius/bench/Data/Scheme/block_bench.cpp | 4 +- moebius/moebius/data/scheme_der.cpp | 34 ++++--- .../tests/moebius/data/scheme_der_test.cpp | 89 +++++++++++++++++++ 3 files changed, 113 insertions(+), 14 deletions(-) create mode 100644 moebius/tests/moebius/data/scheme_der_test.cpp diff --git a/moebius/bench/Data/Scheme/block_bench.cpp b/moebius/bench/Data/Scheme/block_bench.cpp index 3d19123d2d..1bd60b8ec7 100644 --- a/moebius/bench/Data/Scheme/block_bench.cpp +++ b/moebius/bench/Data/Scheme/block_bench.cpp @@ -19,7 +19,7 @@ int main () { lolly::init_tbox (); string buffer; - url u= url_pwd () * url ("bench/Data/Scheme/dictionary.scm"); + url u= url_pwd () * url ("moebius/bench/Data/Scheme/dictionary.scm"); load_string (u, buffer, false); bench.minEpochIterations (10) .batch (N (buffer)) @@ -30,7 +30,7 @@ main () { bench.run ("serializing large group of simple element", [&] { scheme_tree_to_block (parsed_tree); }); - u= url_pwd () * url ("bench/Data/Scheme/virtual-font.scm"); + u= url_pwd () * url ("moebius/bench/Data/Scheme/virtual-font.scm"); load_string (u, buffer, false); bench.batch (N (buffer)) .run ("parsing group of complex tree", diff --git a/moebius/moebius/data/scheme_der.cpp b/moebius/moebius/data/scheme_der.cpp index a4253fe2fc..d3a3d89902 100644 --- a/moebius/moebius/data/scheme_der.cpp +++ b/moebius/moebius/data/scheme_der.cpp @@ -40,8 +40,9 @@ scm_unquote (string s) { ******************************************************************************/ void unslash (string& s, int i, int end_index, string& r, int& r_index) { - char ch= s[i]; + // 循环顶先判界再读字符:原实现末尾的 ch= s[i] 会越界读一字节 while (i < end_index) { + char ch= s[i]; if ((ch == '\\') && ((i + 1) < end_index)) { i++; ch= s[i]; @@ -68,7 +69,6 @@ unslash (string& s, int i, int end_index, string& r, int& r_index) { r_index++; } i++; - ch= s[i]; } } @@ -142,14 +142,16 @@ string_to_scheme_tree (string& s, int& i, const int length) { i++; int end_index = i; const int start_index= i; - char ch = s[end_index]; - unsigned char* types = char_type; - while (!(ch == '\"') && end_index < length) { + char ch; + unsigned char* types= char_type; + // 先判界再读字符:原实现的 ch= s[end_index] 会越界读末尾一字节 + while (end_index < length) { + ch= s[end_index]; + if (ch == '\"') break; if (types[(unsigned char) ch] & CT_ESC) { if (end_index < length - 1) end_index++; } end_index++; - ch= s[end_index]; } const int r_size = 1; // N ("\""); int quoted_index= r_size; @@ -171,15 +173,16 @@ string_to_scheme_tree (string& s, int& i, const int length) { default: { int end_index = i; const int start_index= i; - char ch = s[end_index]; - unsigned char* types = char_type; - while (end_index < length && - !(types[(unsigned char) ch] & (CT_SPC | CT_PAREN))) { + char ch; + unsigned char* types= char_type; + // 先判界再读字符,避免词元结尾在缓冲区末尾时越界读 + while (end_index < length) { + ch= s[end_index]; + if (types[(unsigned char) ch] & (CT_SPC | CT_PAREN)) break; if (types[(unsigned char) ch] & CT_ESC) { if (end_index < length - 1) end_index++; } end_index++; - ch= s[end_index]; } const int r_size = 0; // empty string int token_index= r_size; @@ -197,7 +200,14 @@ string_to_scheme_tree (string& s, int& i, const int length) { scheme_tree string_to_scheme_tree (string s) { if (!char_type_init) init_char_type (); - s = replace (s, "\015", ""); + // 含 CR 时才做整串替换拷贝 + bool has_cr= false; + for (int k= 0; k < N (s); k++) + if (s[k] == '\015') { + has_cr= true; + break; + } + if (has_cr) s= replace (s, "\015", ""); int i = 0; const int length= N (s); return string_to_scheme_tree (s, i, length); diff --git a/moebius/tests/moebius/data/scheme_der_test.cpp b/moebius/tests/moebius/data/scheme_der_test.cpp new file mode 100644 index 0000000000..0fae916dd7 --- /dev/null +++ b/moebius/tests/moebius/data/scheme_der_test.cpp @@ -0,0 +1,89 @@ +/** \file scheme_der_test.cpp + * \copyright GPLv3 + * \details Unit tests for scheme source parsing (string_to_scheme_tree) + * \date 2026 + */ + +#include "moe_doctests.hpp" + +#include "tree_helper.hpp" + +#include +#include + +using namespace moebius; + +using namespace moebius::data; + +static string +atom_of (scheme_tree t) { + return t->label; +} + +TEST_SUITE ("scheme_der") { + + TEST_CASE ("parse simple list") { + scheme_tree t= string_to_scheme_tree ("(a b c)"); + CHECK (is_tuple (t)); + CHECK_EQ (N (t), 3); + CHECK (atom_of (t[0]) == "a"); + CHECK (atom_of (t[2]) == "c"); + } + + TEST_CASE ("parse token at buffer end") { + // 词元恰在缓冲区末尾结束(原实现越界读末尾一字节) + scheme_tree t= string_to_scheme_tree ("(abc)"); + CHECK_EQ (N (t), 1); + CHECK (atom_of (t[0]) == "abc"); + } + + TEST_CASE ("parse trailing backslash at end") { + // 转义字符位于缓冲区末尾(原实现 unslash 越界读) + scheme_tree t= string_to_scheme_tree ("abc\\"); + CHECK (atom_of (t) == "abc\\"); + } + + TEST_CASE ("parse quoted string with escapes") { + scheme_tree t= string_to_scheme_tree ("\"a\\nb\\tc\\\\d\""); + string s= atom_of (t); + CHECK (s == "\"a\nb\tc\\d\""); + } + + TEST_CASE ("parse unclosed quote at end") { + // 未闭合引号在缓冲区末尾截断,补上收尾引号 + scheme_tree t= string_to_scheme_tree ("\"abc"); + CHECK (atom_of (t) == "\"abc\""); + } + + TEST_CASE ("parse skips comments") { + scheme_tree t= string_to_scheme_tree ("; comment\n(foo)"); + CHECK (is_tuple (t)); + CHECK_EQ (N (t), 1); + CHECK (atom_of (t[0]) == "foo"); + } + + TEST_CASE ("parse quote sugar") { + scheme_tree t= string_to_scheme_tree ("'x"); + CHECK (is_tuple (t)); + CHECK_EQ (N (t), 2); + CHECK (atom_of (t[0]) == "'"); + CHECK (atom_of (t[1]) == "x"); + } + + TEST_CASE ("parse with carriage returns") { + // CR 字符被整串剔除,词元内 CR 直接拼接 + scheme_tree t= string_to_scheme_tree ("(a\015b c\015)"); + CHECK_EQ (N (t), 2); + CHECK (atom_of (t[0]) == "ab"); + CHECK (atom_of (t[1]) == "c"); + } + + TEST_CASE ("block parse concatenated expressions") { + scheme_tree t= block_to_scheme_tree ("(a) (b) (c)"); + CHECK (is_tuple (t)); + CHECK_EQ (N (t), 3); + CHECK_EQ (N (t[1]), 1); + CHECK (atom_of (t[1][0]) == "b"); + } + +} // TEST_SUITE