diff --git a/config.yaml.full b/config.yaml.full index 196cc143e..b41b5a2da 100644 --- a/config.yaml.full +++ b/config.yaml.full @@ -143,7 +143,7 @@ database: user: password: database: - charset: utf8 + charset: utf8mb4 # [optional] postgresql: host: # host or ip diff --git a/docs/content/docs/framework/memory/short-term/mysql.en.mdx b/docs/content/docs/framework/memory/short-term/mysql.en.mdx index d7f0d3bc4..ec387fa26 100644 --- a/docs/content/docs/framework/memory/short-term/mysql.en.mdx +++ b/docs/content/docs/framework/memory/short-term/mysql.en.mdx @@ -20,7 +20,7 @@ database: user: password: database: - charset: utf8 + charset: utf8mb4 ``` | Field | Env var | Default | Description | @@ -29,7 +29,7 @@ database: | `user` | `DATABASE_MYSQL_USER` | `""` | Username. | | `password` | `DATABASE_MYSQL_PASSWORD` | `""` | Password. | | `database` | `DATABASE_MYSQL_DATABASE` | `""` | Database name. | -| `charset` | `DATABASE_MYSQL_CHARSET` | `utf8` | Character set. | +| `charset` | `DATABASE_MYSQL_CHARSET` | `utf8mb4` | Character set, passed to the driver as `?charset=`. An empty string leaves it to the driver. | ## Usage diff --git a/docs/content/docs/framework/memory/short-term/mysql.mdx b/docs/content/docs/framework/memory/short-term/mysql.mdx index 3ebb492cf..a60e7c681 100644 --- a/docs/content/docs/framework/memory/short-term/mysql.mdx +++ b/docs/content/docs/framework/memory/short-term/mysql.mdx @@ -20,7 +20,7 @@ database: user: password: database: - charset: utf8 + charset: utf8mb4 ``` | 配置项 | 环境变量 | 默认值 | 说明 | @@ -29,7 +29,7 @@ database: | `user` | `DATABASE_MYSQL_USER` | `""` | 用户名。 | | `password` | `DATABASE_MYSQL_PASSWORD` | `""` | 密码。 | | `database` | `DATABASE_MYSQL_DATABASE` | `""` | 数据库名。 | -| `charset` | `DATABASE_MYSQL_CHARSET` | `utf8` | 字符集。 | +| `charset` | `DATABASE_MYSQL_CHARSET` | `utf8mb4` | 字符集,以 `?charset=` 传给驱动;设为空字符串则由驱动决定。 | ## 用法 diff --git a/docs/content/docs/references/configuration/environment-variables.en.mdx b/docs/content/docs/references/configuration/environment-variables.en.mdx index 0207053df..523c108c1 100644 --- a/docs/content/docs/references/configuration/environment-variables.en.mdx +++ b/docs/content/docs/references/configuration/environment-variables.en.mdx @@ -95,7 +95,7 @@ Prefix `DATABASE_`, grouped by storage type. Memory and the knowledge base read | | `DATABASE_MYSQL_USER` | Username | | | `DATABASE_MYSQL_PASSWORD` | Password | | | `DATABASE_MYSQL_DATABASE` | Database name | -| | `DATABASE_MYSQL_CHARSET` | Charset, default utf8 | +| | `DATABASE_MYSQL_CHARSET` | Charset, default utf8mb4 | | Redis | `DATABASE_REDIS_HOST` | Host | | | `DATABASE_REDIS_PORT` | Port, default 6379 | | | `DATABASE_REDIS_PASSWORD` | Password | diff --git a/docs/content/docs/references/configuration/environment-variables.mdx b/docs/content/docs/references/configuration/environment-variables.mdx index d1209350b..c45fc4a69 100644 --- a/docs/content/docs/references/configuration/environment-variables.mdx +++ b/docs/content/docs/references/configuration/environment-variables.mdx @@ -95,7 +95,7 @@ volcengine: | | `DATABASE_MYSQL_USER` | 用户名 | | | `DATABASE_MYSQL_PASSWORD` | 密码 | | | `DATABASE_MYSQL_DATABASE` | 数据库名 | -| | `DATABASE_MYSQL_CHARSET` | 字符集,默认 utf8 | +| | `DATABASE_MYSQL_CHARSET` | 字符集,默认 utf8mb4 | | Redis | `DATABASE_REDIS_HOST` | 主机地址 | | | `DATABASE_REDIS_PORT` | 端口,默认 6379 | | | `DATABASE_REDIS_PASSWORD` | 密码 | diff --git a/tests/memory/test_mysql_charset.py b/tests/memory/test_mysql_charset.py new file mode 100644 index 000000000..3311d70c6 --- /dev/null +++ b/tests/memory/test_mysql_charset.py @@ -0,0 +1,69 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pytest +from sqlalchemy.engine import make_url + +from veadk.configs.database_configs import MysqlConfig +from veadk.memory.short_term_memory_backends.mysql_backend import MysqlSTMBackend + +BACKEND = "veadk.memory.short_term_memory_backends.mysql_backend" + + +@pytest.fixture(autouse=True) +def isolated_database_environment(monkeypatch): + for key in tuple(os.environ): + if key.startswith("DATABASE_MYSQL_"): + monkeypatch.delenv(key) + + +def _backend_url(monkeypatch, async_driver, **config): + monkeypatch.setattr(f"{BACKEND}.should_use_async_db_drivers", lambda: async_driver) + mysql_config = MysqlConfig( + host="db.example.test:3307", + user="fixture-user", + password="fixture@:/", + database="db", + **config, + ) + return make_url(MysqlSTMBackend(mysql_config=mysql_config)._db_url) + + +@pytest.mark.parametrize("async_driver", [False, True]) +def test_default_charset_is_sent_to_driver(async_driver, monkeypatch): + url = _backend_url(monkeypatch, async_driver) + assert url.drivername == ("mysql+aiomysql" if async_driver else "mysql+pymysql") + assert url.query == {"charset": "utf8mb4"} + assert (url.host, url.port, url.database) == ("db.example.test", 3307, "db") + assert (url.username, url.password) == ("fixture-user", "fixture@:/") + + +@pytest.mark.parametrize("async_driver", [False, True]) +def test_configured_charset_is_sent_to_driver(async_driver, monkeypatch): + url = _backend_url(monkeypatch, async_driver, charset="latin1") + assert url.query == {"charset": "latin1"} + + +def test_charset_from_environment(monkeypatch): + monkeypatch.setenv("DATABASE_MYSQL_CHARSET", "gbk") + url = _backend_url(monkeypatch, False) + assert url.query == {"charset": "gbk"} + + +@pytest.mark.parametrize("async_driver", [False, True]) +def test_empty_charset_leaves_choice_to_driver(async_driver, monkeypatch): + url = _backend_url(monkeypatch, async_driver, charset="") + assert url.query == {} diff --git a/veadk/cli/cli_harness.py b/veadk/cli/cli_harness.py index e29b1be8e..1d428002e 100644 --- a/veadk/cli/cli_harness.py +++ b/veadk/cli/cli_harness.py @@ -163,7 +163,7 @@ # user: root # password: "" # database: harness - # charset: utf8 + # charset: utf8mb4 # -- postgresql -- env DATABASE_POSTGRESQL_* flags: --short-term-memory-host / -port / -user / -password / -database # host: 1.2.3.4 # port: 5432 diff --git a/veadk/configs/database_configs.py b/veadk/configs/database_configs.py index 7023a1222..0852fc378 100644 --- a/veadk/configs/database_configs.py +++ b/veadk/configs/database_configs.py @@ -65,7 +65,10 @@ class MysqlConfig(BaseSettings): database: str = "" - charset: str = "utf8" + charset: str = "utf8mb4" + """Connection character set, sent to the driver as ``?charset=``. The + default matches what PyMySQL and aiomysql pick when none is given. Set it + to an empty string to leave the choice to the driver.""" secret_token: str = "" """STS token for MySQL auth, not supported yet.""" diff --git a/veadk/memory/short_term_memory_backends/mysql_backend.py b/veadk/memory/short_term_memory_backends/mysql_backend.py index e3e92d91b..6f1ffa727 100644 --- a/veadk/memory/short_term_memory_backends/mysql_backend.py +++ b/veadk/memory/short_term_memory_backends/mysql_backend.py @@ -38,10 +38,16 @@ class MysqlSTMBackend(BaseShortTermMemoryBackend): def model_post_init(self, context: Any) -> None: encoded_username = quote_plus(self.mysql_config.user) encoded_password = quote_plus(self.mysql_config.password) + # An empty charset leaves the choice to the driver. + query = ( + f"?charset={quote_plus(self.mysql_config.charset)}" + if self.mysql_config.charset + else "" + ) if should_use_async_db_drivers(): - self._db_url = f"mysql+aiomysql://{encoded_username}:{encoded_password}@{self.mysql_config.host}/{self.mysql_config.database}" + self._db_url = f"mysql+aiomysql://{encoded_username}:{encoded_password}@{self.mysql_config.host}/{self.mysql_config.database}{query}" else: - self._db_url = f"mysql+pymysql://{encoded_username}:{encoded_password}@{self.mysql_config.host}/{self.mysql_config.database}" + self._db_url = f"mysql+pymysql://{encoded_username}:{encoded_password}@{self.mysql_config.host}/{self.mysql_config.database}{query}" @cached_property @override