From 332b17c99c7cdcbd2f8243a2cf73fd5753dcda38 Mon Sep 17 00:00:00 2001 From: Jace Xia Date: Sun, 15 Nov 2026 11:17:10 +0800 Subject: [PATCH] fix(memory): pass the configured MySQL charset to the driver MysqlConfig.charset (DATABASE_MYSQL_CHARSET, --short-term-memory-charset) was documented but never used: MysqlSTMBackend built the database URL without it, so the driver always fell back to its own default. Append ?charset= to both the pymysql and aiomysql URLs. Change the default from utf8 to utf8mb4, which is what both drivers already used when no charset was passed, so deployments that never set it keep the same behavior. An empty value still leaves the choice to the driver. Update the docs and sample configs to match. --- config.yaml.full | 2 +- .../framework/memory/short-term/mysql.en.mdx | 4 +- .../framework/memory/short-term/mysql.mdx | 4 +- .../environment-variables.en.mdx | 2 +- .../configuration/environment-variables.mdx | 2 +- tests/memory/test_mysql_charset.py | 69 +++++++++++++++++++ veadk/cli/cli_harness.py | 2 +- veadk/configs/database_configs.py | 5 +- .../mysql_backend.py | 10 ++- 9 files changed, 89 insertions(+), 11 deletions(-) create mode 100644 tests/memory/test_mysql_charset.py diff --git a/config.yaml.full b/config.yaml.full index 196cc143e..b41b5a2da 100644 --- a/config.yaml.full +++ b/config.yaml.full @@ -143,7 +143,7 @@ database: user: password: database: - charset: utf8 + charset: utf8mb4 # [optional] postgresql: host: # host or ip diff --git a/docs/content/docs/framework/memory/short-term/mysql.en.mdx b/docs/content/docs/framework/memory/short-term/mysql.en.mdx index d7f0d3bc4..ec387fa26 100644 --- a/docs/content/docs/framework/memory/short-term/mysql.en.mdx +++ b/docs/content/docs/framework/memory/short-term/mysql.en.mdx @@ -20,7 +20,7 @@ database: user: password: database: - charset: utf8 + charset: utf8mb4 ``` | Field | Env var | Default | Description | @@ -29,7 +29,7 @@ database: | `user` | `DATABASE_MYSQL_USER` | `""` | Username. | | `password` | `DATABASE_MYSQL_PASSWORD` | `""` | Password. | | `database` | `DATABASE_MYSQL_DATABASE` | `""` | Database name. | -| `charset` | `DATABASE_MYSQL_CHARSET` | `utf8` | Character set. | +| `charset` | `DATABASE_MYSQL_CHARSET` | `utf8mb4` | Character set, passed to the driver as `?charset=`. An empty string leaves it to the driver. | ## Usage diff --git a/docs/content/docs/framework/memory/short-term/mysql.mdx b/docs/content/docs/framework/memory/short-term/mysql.mdx index 3ebb492cf..a60e7c681 100644 --- a/docs/content/docs/framework/memory/short-term/mysql.mdx +++ b/docs/content/docs/framework/memory/short-term/mysql.mdx @@ -20,7 +20,7 @@ database: user: password: database: - charset: utf8 + charset: utf8mb4 ``` | 配置项 | 环境变量 | 默认值 | 说明 | @@ -29,7 +29,7 @@ database: | `user` | `DATABASE_MYSQL_USER` | `""` | 用户名。 | | `password` | `DATABASE_MYSQL_PASSWORD` | `""` | 密码。 | | `database` | `DATABASE_MYSQL_DATABASE` | `""` | 数据库名。 | -| `charset` | `DATABASE_MYSQL_CHARSET` | `utf8` | 字符集。 | +| `charset` | `DATABASE_MYSQL_CHARSET` | `utf8mb4` | 字符集,以 `?charset=` 传给驱动;设为空字符串则由驱动决定。 | ## 用法 diff --git a/docs/content/docs/references/configuration/environment-variables.en.mdx b/docs/content/docs/references/configuration/environment-variables.en.mdx index 0207053df..523c108c1 100644 --- a/docs/content/docs/references/configuration/environment-variables.en.mdx +++ b/docs/content/docs/references/configuration/environment-variables.en.mdx @@ -95,7 +95,7 @@ Prefix `DATABASE_`, grouped by storage type. Memory and the knowledge base read | | `DATABASE_MYSQL_USER` | Username | | | `DATABASE_MYSQL_PASSWORD` | Password | | | `DATABASE_MYSQL_DATABASE` | Database name | -| | `DATABASE_MYSQL_CHARSET` | Charset, default utf8 | +| | `DATABASE_MYSQL_CHARSET` | Charset, default utf8mb4 | | Redis | `DATABASE_REDIS_HOST` | Host | | | `DATABASE_REDIS_PORT` | Port, default 6379 | | | `DATABASE_REDIS_PASSWORD` | Password | diff --git a/docs/content/docs/references/configuration/environment-variables.mdx b/docs/content/docs/references/configuration/environment-variables.mdx index d1209350b..c45fc4a69 100644 --- a/docs/content/docs/references/configuration/environment-variables.mdx +++ b/docs/content/docs/references/configuration/environment-variables.mdx @@ -95,7 +95,7 @@ volcengine: | | `DATABASE_MYSQL_USER` | 用户名 | | | `DATABASE_MYSQL_PASSWORD` | 密码 | | | `DATABASE_MYSQL_DATABASE` | 数据库名 | -| | `DATABASE_MYSQL_CHARSET` | 字符集,默认 utf8 | +| | `DATABASE_MYSQL_CHARSET` | 字符集,默认 utf8mb4 | | Redis | `DATABASE_REDIS_HOST` | 主机地址 | | | `DATABASE_REDIS_PORT` | 端口,默认 6379 | | | `DATABASE_REDIS_PASSWORD` | 密码 | diff --git a/tests/memory/test_mysql_charset.py b/tests/memory/test_mysql_charset.py new file mode 100644 index 000000000..3311d70c6 --- /dev/null +++ b/tests/memory/test_mysql_charset.py @@ -0,0 +1,69 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pytest +from sqlalchemy.engine import make_url + +from veadk.configs.database_configs import MysqlConfig +from veadk.memory.short_term_memory_backends.mysql_backend import MysqlSTMBackend + +BACKEND = "veadk.memory.short_term_memory_backends.mysql_backend" + + +@pytest.fixture(autouse=True) +def isolated_database_environment(monkeypatch): + for key in tuple(os.environ): + if key.startswith("DATABASE_MYSQL_"): + monkeypatch.delenv(key) + + +def _backend_url(monkeypatch, async_driver, **config): + monkeypatch.setattr(f"{BACKEND}.should_use_async_db_drivers", lambda: async_driver) + mysql_config = MysqlConfig( + host="db.example.test:3307", + user="fixture-user", + password="fixture@:/", + database="db", + **config, + ) + return make_url(MysqlSTMBackend(mysql_config=mysql_config)._db_url) + + +@pytest.mark.parametrize("async_driver", [False, True]) +def test_default_charset_is_sent_to_driver(async_driver, monkeypatch): + url = _backend_url(monkeypatch, async_driver) + assert url.drivername == ("mysql+aiomysql" if async_driver else "mysql+pymysql") + assert url.query == {"charset": "utf8mb4"} + assert (url.host, url.port, url.database) == ("db.example.test", 3307, "db") + assert (url.username, url.password) == ("fixture-user", "fixture@:/") + + +@pytest.mark.parametrize("async_driver", [False, True]) +def test_configured_charset_is_sent_to_driver(async_driver, monkeypatch): + url = _backend_url(monkeypatch, async_driver, charset="latin1") + assert url.query == {"charset": "latin1"} + + +def test_charset_from_environment(monkeypatch): + monkeypatch.setenv("DATABASE_MYSQL_CHARSET", "gbk") + url = _backend_url(monkeypatch, False) + assert url.query == {"charset": "gbk"} + + +@pytest.mark.parametrize("async_driver", [False, True]) +def test_empty_charset_leaves_choice_to_driver(async_driver, monkeypatch): + url = _backend_url(monkeypatch, async_driver, charset="") + assert url.query == {} diff --git a/veadk/cli/cli_harness.py b/veadk/cli/cli_harness.py index e29b1be8e..1d428002e 100644 --- a/veadk/cli/cli_harness.py +++ b/veadk/cli/cli_harness.py @@ -163,7 +163,7 @@ # user: root # password: "" # database: harness - # charset: utf8 + # charset: utf8mb4 # -- postgresql -- env DATABASE_POSTGRESQL_* flags: --short-term-memory-host / -port / -user / -password / -database # host: 1.2.3.4 # port: 5432 diff --git a/veadk/configs/database_configs.py b/veadk/configs/database_configs.py index 7023a1222..0852fc378 100644 --- a/veadk/configs/database_configs.py +++ b/veadk/configs/database_configs.py @@ -65,7 +65,10 @@ class MysqlConfig(BaseSettings): database: str = "" - charset: str = "utf8" + charset: str = "utf8mb4" + """Connection character set, sent to the driver as ``?charset=``. The + default matches what PyMySQL and aiomysql pick when none is given. Set it + to an empty string to leave the choice to the driver.""" secret_token: str = "" """STS token for MySQL auth, not supported yet.""" diff --git a/veadk/memory/short_term_memory_backends/mysql_backend.py b/veadk/memory/short_term_memory_backends/mysql_backend.py index e3e92d91b..6f1ffa727 100644 --- a/veadk/memory/short_term_memory_backends/mysql_backend.py +++ b/veadk/memory/short_term_memory_backends/mysql_backend.py @@ -38,10 +38,16 @@ class MysqlSTMBackend(BaseShortTermMemoryBackend): def model_post_init(self, context: Any) -> None: encoded_username = quote_plus(self.mysql_config.user) encoded_password = quote_plus(self.mysql_config.password) + # An empty charset leaves the choice to the driver. + query = ( + f"?charset={quote_plus(self.mysql_config.charset)}" + if self.mysql_config.charset + else "" + ) if should_use_async_db_drivers(): - self._db_url = f"mysql+aiomysql://{encoded_username}:{encoded_password}@{self.mysql_config.host}/{self.mysql_config.database}" + self._db_url = f"mysql+aiomysql://{encoded_username}:{encoded_password}@{self.mysql_config.host}/{self.mysql_config.database}{query}" else: - self._db_url = f"mysql+pymysql://{encoded_username}:{encoded_password}@{self.mysql_config.host}/{self.mysql_config.database}" + self._db_url = f"mysql+pymysql://{encoded_username}:{encoded_password}@{self.mysql_config.host}/{self.mysql_config.database}{query}" @cached_property @override