Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion config.yaml.full
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ database:
user:
password:
database:
charset: utf8
charset: utf8mb4
# [optional]
postgresql:
host: # host or ip
Expand Down
4 changes: 2 additions & 2 deletions docs/content/docs/framework/memory/short-term/mysql.en.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ database:
user:
password:
database:
charset: utf8
charset: utf8mb4
```

| Field | Env var | Default | Description |
Expand All @@ -29,7 +29,7 @@ database:
| `user` | `DATABASE_MYSQL_USER` | `""` | Username. |
| `password` | `DATABASE_MYSQL_PASSWORD` | `""` | Password. |
| `database` | `DATABASE_MYSQL_DATABASE` | `""` | Database name. |
| `charset` | `DATABASE_MYSQL_CHARSET` | `utf8` | Character set. |
| `charset` | `DATABASE_MYSQL_CHARSET` | `utf8mb4` | Character set, passed to the driver as `?charset=`. An empty string leaves it to the driver. |

## Usage

Expand Down
4 changes: 2 additions & 2 deletions docs/content/docs/framework/memory/short-term/mysql.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ database:
user:
password:
database:
charset: utf8
charset: utf8mb4
```

| 配置项 | 环境变量 | 默认值 | 说明 |
Expand All @@ -29,7 +29,7 @@ database:
| `user` | `DATABASE_MYSQL_USER` | `""` | 用户名。 |
| `password` | `DATABASE_MYSQL_PASSWORD` | `""` | 密码。 |
| `database` | `DATABASE_MYSQL_DATABASE` | `""` | 数据库名。 |
| `charset` | `DATABASE_MYSQL_CHARSET` | `utf8` | 字符集。 |
| `charset` | `DATABASE_MYSQL_CHARSET` | `utf8mb4` | 字符集,以 `?charset=` 传给驱动;设为空字符串则由驱动决定。 |

## 用法

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ Prefix `DATABASE_`, grouped by storage type. Memory and the knowledge base read
| | `DATABASE_MYSQL_USER` | Username |
| | `DATABASE_MYSQL_PASSWORD` | Password |
| | `DATABASE_MYSQL_DATABASE` | Database name |
| | `DATABASE_MYSQL_CHARSET` | Charset, default utf8 |
| | `DATABASE_MYSQL_CHARSET` | Charset, default utf8mb4 |
| Redis | `DATABASE_REDIS_HOST` | Host |
| | `DATABASE_REDIS_PORT` | Port, default 6379 |
| | `DATABASE_REDIS_PASSWORD` | Password |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ volcengine:
| | `DATABASE_MYSQL_USER` | 用户名 |
| | `DATABASE_MYSQL_PASSWORD` | 密码 |
| | `DATABASE_MYSQL_DATABASE` | 数据库名 |
| | `DATABASE_MYSQL_CHARSET` | 字符集,默认 utf8 |
| | `DATABASE_MYSQL_CHARSET` | 字符集,默认 utf8mb4 |
| Redis | `DATABASE_REDIS_HOST` | 主机地址 |
| | `DATABASE_REDIS_PORT` | 端口,默认 6379 |
| | `DATABASE_REDIS_PASSWORD` | 密码 |
Expand Down
69 changes: 69 additions & 0 deletions tests/memory/test_mysql_charset.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os

import pytest
from sqlalchemy.engine import make_url

from veadk.configs.database_configs import MysqlConfig
from veadk.memory.short_term_memory_backends.mysql_backend import MysqlSTMBackend

BACKEND = "veadk.memory.short_term_memory_backends.mysql_backend"


@pytest.fixture(autouse=True)
def isolated_database_environment(monkeypatch):
for key in tuple(os.environ):
if key.startswith("DATABASE_MYSQL_"):
monkeypatch.delenv(key)


def _backend_url(monkeypatch, async_driver, **config):
monkeypatch.setattr(f"{BACKEND}.should_use_async_db_drivers", lambda: async_driver)
mysql_config = MysqlConfig(
host="db.example.test:3307",
user="fixture-user",
password="fixture@:/",
database="db",
**config,
)
return make_url(MysqlSTMBackend(mysql_config=mysql_config)._db_url)


@pytest.mark.parametrize("async_driver", [False, True])
def test_default_charset_is_sent_to_driver(async_driver, monkeypatch):
url = _backend_url(monkeypatch, async_driver)
assert url.drivername == ("mysql+aiomysql" if async_driver else "mysql+pymysql")
assert url.query == {"charset": "utf8mb4"}
assert (url.host, url.port, url.database) == ("db.example.test", 3307, "db")
assert (url.username, url.password) == ("fixture-user", "fixture@:/")


@pytest.mark.parametrize("async_driver", [False, True])
def test_configured_charset_is_sent_to_driver(async_driver, monkeypatch):
url = _backend_url(monkeypatch, async_driver, charset="latin1")
assert url.query == {"charset": "latin1"}


def test_charset_from_environment(monkeypatch):
monkeypatch.setenv("DATABASE_MYSQL_CHARSET", "gbk")
url = _backend_url(monkeypatch, False)
assert url.query == {"charset": "gbk"}


@pytest.mark.parametrize("async_driver", [False, True])
def test_empty_charset_leaves_choice_to_driver(async_driver, monkeypatch):
url = _backend_url(monkeypatch, async_driver, charset="")
assert url.query == {}
2 changes: 1 addition & 1 deletion veadk/cli/cli_harness.py
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@
# user: root
# password: ""
# database: harness
# charset: utf8
# charset: utf8mb4
# -- postgresql -- env DATABASE_POSTGRESQL_* flags: --short-term-memory-host / -port / -user / -password / -database
# host: 1.2.3.4
# port: 5432
Expand Down
5 changes: 4 additions & 1 deletion veadk/configs/database_configs.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,10 @@ class MysqlConfig(BaseSettings):

database: str = ""

charset: str = "utf8"
charset: str = "utf8mb4"
"""Connection character set, sent to the driver as ``?charset=``. The
default matches what PyMySQL and aiomysql pick when none is given. Set it
to an empty string to leave the choice to the driver."""

secret_token: str = ""
"""STS token for MySQL auth, not supported yet."""
Expand Down
10 changes: 8 additions & 2 deletions veadk/memory/short_term_memory_backends/mysql_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,16 @@ class MysqlSTMBackend(BaseShortTermMemoryBackend):
def model_post_init(self, context: Any) -> None:
encoded_username = quote_plus(self.mysql_config.user)
encoded_password = quote_plus(self.mysql_config.password)
# An empty charset leaves the choice to the driver.
query = (
f"?charset={quote_plus(self.mysql_config.charset)}"
if self.mysql_config.charset
else ""
)
if should_use_async_db_drivers():
self._db_url = f"mysql+aiomysql://{encoded_username}:{encoded_password}@{self.mysql_config.host}/{self.mysql_config.database}"
self._db_url = f"mysql+aiomysql://{encoded_username}:{encoded_password}@{self.mysql_config.host}/{self.mysql_config.database}{query}"
else:
self._db_url = f"mysql+pymysql://{encoded_username}:{encoded_password}@{self.mysql_config.host}/{self.mysql_config.database}"
self._db_url = f"mysql+pymysql://{encoded_username}:{encoded_password}@{self.mysql_config.host}/{self.mysql_config.database}{query}"

@cached_property
@override
Expand Down