From f2796c3632151ddbb09385f9f48c3ec89716d273 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 16 Nov 2026 10:29:20 +0800 Subject: [PATCH 01/16] feat(python): write TsFiles to file-like and object-storage targets Expose the C++ write sink to Python so TsFileWriter(target) accepts a writable binary file object (fsspec/S3-style) as well as a path. - Add the C API in-memory handle (write_file_new_memory / write_file_get_buffer) on top of MemoryRandomAccessWriteFile, and take ownership of the sink through TsFileWriter::init(unique_ptr). - free_write_file() and the writer factories now go through RandomAccessWriteFile so an in-memory handle is destroyed correctly. - Add the Python bridge (python_random_access_write_file) implementing RandomAccessWriteFile over write/tell/seek/flush/truncate, wire it into the tsfile_writer extension, and let TsFileWriter(target) dispatch on the target type. - Tests: in-memory and file-like round trips, plus a MinIO compose file and an env-gated integration test for a real object store. --- .gitignore | 1 + cpp/src/cwrapper/tsfile_cwrapper.cc | 53 +++- cpp/src/cwrapper/tsfile_cwrapper.h | 25 ++ cpp/src/writer/tsfile_writer.cc | 16 + cpp/src/writer/tsfile_writer.h | 3 + python/setup.py | 7 +- python/tests/integration/minio/README.md | 53 ++++ .../integration/minio/docker-compose.yml | 40 +++ python/tests/test_minio_integration.py | 87 ++++++ python/tests/test_write_sources.py | 86 ++++++ .../tsfile/python_random_access_write_file.cc | 282 ++++++++++++++++++ .../tsfile/python_random_access_write_file.h | 30 ++ python/tsfile/tsfile_writer.pyx | 22 +- 13 files changed, 698 insertions(+), 7 deletions(-) create mode 100644 python/tests/integration/minio/README.md create mode 100644 python/tests/integration/minio/docker-compose.yml create mode 100644 python/tests/test_minio_integration.py create mode 100644 python/tests/test_write_sources.py create mode 100644 python/tsfile/python_random_access_write_file.cc create mode 100644 python/tsfile/python_random_access_write_file.h diff --git a/.gitignore b/.gitignore index dc16352bd..3c5fe07ea 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ python/tsfile/*dll* python/tsfile/*dylib* python/tsfile/*.h !python/tsfile/python_random_access_read_file.h +!python/tsfile/python_random_access_write_file.h python/tsfile/*.cpp python/tsfile/**/*.cpp python/tsfile/**/*so* diff --git a/cpp/src/cwrapper/tsfile_cwrapper.cc b/cpp/src/cwrapper/tsfile_cwrapper.cc index 588561e3e..94f6092eb 100644 --- a/cpp/src/cwrapper/tsfile_cwrapper.cc +++ b/cpp/src/cwrapper/tsfile_cwrapper.cc @@ -20,6 +20,7 @@ #include "cwrapper/tsfile_cwrapper.h" #include +#include #include #include #include @@ -226,6 +227,49 @@ WriteFile write_file_new(const char* pathname, ERRNO* err_code) { return file; } +WriteFile write_file_new_memory(ERRNO* err_code) { + if (err_code == nullptr) { + return nullptr; + } + storage::MemoryRandomAccessWriteFile* file = + new (std::nothrow) storage::MemoryRandomAccessWriteFile(); + if (file == nullptr) { + *err_code = common::E_OOM; + return nullptr; + } + *err_code = common::E_OK; + return file; +} + +ERRNO write_file_get_buffer(WriteFile write_file, uint8_t** buffer, + uint32_t* length) { + if (buffer == nullptr || length == nullptr) { + return common::E_INVALID_ARG; + } + *buffer = nullptr; + *length = 0; + if (write_file == nullptr) { + return common::E_INVALID_ARG; + } + auto* sink = static_cast(write_file); + auto* mem = dynamic_cast(sink); + if (mem == nullptr) { + return common::E_NOT_SUPPORT; + } + const std::vector& data = mem->data(); + if (data.empty()) { + return common::E_OK; + } + void* copy = malloc(data.size()); + if (copy == nullptr) { + return common::E_OOM; + } + std::memcpy(copy, data.data(), data.size()); + *buffer = static_cast(copy); + *length = static_cast(data.size()); + return common::E_OK; +} + TsFileWriter tsfile_writer_new(WriteFile file, TableSchema* schema, ERRNO* err_code) { // C API: every public entry must defend against null callers — a null @@ -259,7 +303,8 @@ TsFileWriter tsfile_writer_new(WriteFile file, TableSchema* schema, new storage::TableSchema(schema->table_name, column_schemas)); std::unique_ptr table_writer( new storage::TsFileTableWriter( - static_cast(file), table_schema.get())); + static_cast(file), + table_schema.get())); *err_code = table_writer->get_error(); if (*err_code != common::E_OK) { return nullptr; @@ -306,8 +351,8 @@ TsFileWriter tsfile_writer_new_with_memory_threshold(WriteFile file, new storage::TableSchema(schema->table_name, column_schemas)); std::unique_ptr table_writer( new storage::TsFileTableWriter( - static_cast(file), table_schema.get(), - memory_threshold)); + static_cast(file), + table_schema.get(), memory_threshold)); *err_code = table_writer->get_error(); if (*err_code != common::E_OK) { return nullptr; @@ -2341,7 +2386,7 @@ void free_column_schema(ColumnSchema schema) { free(schema.column_name); } void tsfile_free_buffer(void* buffer) { free(buffer); } void free_write_file(WriteFile* write_file) { - auto f = static_cast(*write_file); + auto f = static_cast(*write_file); delete f; *write_file = nullptr; } diff --git a/cpp/src/cwrapper/tsfile_cwrapper.h b/cpp/src/cwrapper/tsfile_cwrapper.h index d2982ac36..cda753223 100644 --- a/cpp/src/cwrapper/tsfile_cwrapper.h +++ b/cpp/src/cwrapper/tsfile_cwrapper.h @@ -517,6 +517,31 @@ int set_allow_non_decreasing_timestamps(bool enabled); WriteFile write_file_new(const char* pathname, ERRNO* err_code); +/** + * @brief Creates an in-memory WriteFile that accumulates bytes instead of + * touching the filesystem. + * + * @param err_code [out] E_OK(0), or check error code in errno_define_c.h. + * @return WriteFile Valid handle on success. + * @note Use write_file_get_buffer() to obtain the serialized bytes (for + * example, to upload to object storage). Free the buffer with + * tsfile_free_buffer(). Free the handle with free_write_file() after the + * TsFileWriter has been closed. + */ +WriteFile write_file_new_memory(ERRNO* err_code); + +/** + * @brief Copies the bytes written so far to a caller-owned buffer. + * + * @param write_file Valid in-memory WriteFile handle. + * @param buffer [out] malloc'd copy of the bytes; free with + * tsfile_free_buffer(). + * @param length [out] Number of bytes in @p buffer. + * @return E_OK(0), or check error code in errno_define_c.h. + */ +ERRNO write_file_get_buffer(WriteFile write_file, uint8_t** buffer, + uint32_t* length); + /** * @brief Creates a TsFileWriter for writing a TsFile. * diff --git a/cpp/src/writer/tsfile_writer.cc b/cpp/src/writer/tsfile_writer.cc index d1e06a0ba..1d84d78bc 100644 --- a/cpp/src/writer/tsfile_writer.cc +++ b/cpp/src/writer/tsfile_writer.cc @@ -151,6 +151,22 @@ int TsFileWriter::init(RandomAccessWriteFile* write_file) { // table_schema_map_ in do_check_schema_table when a new device appears. // All new MeasurementSchemaGroup/MeasurementSchema are freed in destroy(). // ----------------------------------------------------------------------------- +int TsFileWriter::init(std::unique_ptr write_file) { + if (!write_file || !write_file->is_opened()) { + return E_INVALID_ARG; + } + RandomAccessWriteFile* raw = write_file.release(); + const int ret = init(raw); + if (ret != E_OK) { + delete raw; + return ret; + } + // init(raw) stored the pointer without taking ownership; transfer it now + // so destroy() deletes it exactly once. + write_file_created_ = true; + return E_OK; +} + int TsFileWriter::init(RestorableTsFileIOWriter* rw) { if (rw == nullptr || !rw->can_write()) { return E_INVALID_ARG; diff --git a/cpp/src/writer/tsfile_writer.h b/cpp/src/writer/tsfile_writer.h index 39645f2d3..190826641 100644 --- a/cpp/src/writer/tsfile_writer.h +++ b/cpp/src/writer/tsfile_writer.h @@ -61,6 +61,9 @@ class TsFileWriter { int open(const std::string& file_path, int flags, mode_t mode); int open(const std::string& file_path); int init(storage::RandomAccessWriteFile* write_file); + /** Like init(RandomAccessWriteFile*) but the writer takes ownership of + * @p write_file and deletes it on destroy(). */ + int init(std::unique_ptr write_file); int init(storage::RestorableTsFileIOWriter* rw); void set_generate_table_schema(bool generate_table_schema); diff --git a/python/setup.py b/python/setup.py index 2672565bf..a983a66f0 100644 --- a/python/setup.py +++ b/python/setup.py @@ -408,7 +408,12 @@ def finalize_options(self): depends=["tsfile/python_random_access_read_file.h"], **common, ), - Extension("tsfile.tsfile_writer", ["tsfile/tsfile_writer.pyx"], **common), + Extension( + "tsfile.tsfile_writer", + ["tsfile/tsfile_writer.pyx", "tsfile/python_random_access_write_file.cc"], + depends=["tsfile/python_random_access_write_file.h"], + **common, + ), ] setup( diff --git a/python/tests/integration/minio/README.md b/python/tests/integration/minio/README.md new file mode 100644 index 000000000..6aeb19f62 --- /dev/null +++ b/python/tests/integration/minio/README.md @@ -0,0 +1,53 @@ +# MinIO integration test + +This directory provisions a MinIO test cluster and documents how to run the +object-storage write/read integration test. + +## Why MinIO + +The C++ core has no HTTP client, so TsFile writes to object storage go through +the Python binding: `TsFileWriter(target)` accepts a writable binary file +object, and fsspec/s3fs supplies one that uploads to S3-compatible storage on +close. MinIO provides that S3-compatible endpoint locally, without a cloud +account. + +## Bring up the cluster + +```bash +docker compose up -d +``` + +This starts a single MinIO node with four drives (erasure coding enabled), the +lightweight equivalent of a distributed cluster for test purposes. The S3 +endpoint is `http://127.0.0.1:9000` and the console is +`http://127.0.0.1:9001`. Credentials are the MinIO defaults (`minioadmin` / +`minioadmin`). + +## Install the S3 client + +The test needs `s3fs` (and its `fsspec` dependency), which is not part of the +core Python dependencies: + +```bash +pip install s3fs +``` + +## Run the integration test + +```bash +TSFILE_MINIO_ENDPOINT=http://127.0.0.1:9000 \ +TSFILE_MINIO_ACCESS_KEY=minioadmin \ +TSFILE_MINIO_SECRET_KEY=minioadmin \ +TSFILE_MINIO_BUCKET=tsfile-test \ +pytest tests/test_minio_integration.py -v +``` + +Without `s3fs` installed or the `TSFILE_MINIO_*` variables set, the test is +skipped rather than failed. + +## What the test covers + +The test writes a TsFile into the object store through `TsFileWriter` (the new +write-sink bridge), lets s3fs upload it on close, then reads it back through +`TsFileReader` and verifies the rows. This exercises the same C++ write/read +paths a non-local filesystem would. diff --git a/python/tests/integration/minio/docker-compose.yml b/python/tests/integration/minio/docker-compose.yml new file mode 100644 index 000000000..541a6b245 --- /dev/null +++ b/python/tests/integration/minio/docker-compose.yml @@ -0,0 +1,40 @@ +# MinIO test cluster for the TsFile object-storage write/read integration test. +# +# A single node with four drives enables MinIO erasure coding and presents the +# same S3 API a multi-node distributed deployment does, without needing four +# VMs. This is the standard lightweight "cluster" for a test environment. +# +# Bring it up: +# docker compose up -d +# Then run the gated Python test: +# TSFILE_MINIO_ENDPOINT=http://127.0.0.1:9000 \ +# TSFILE_MINIO_ACCESS_KEY=minioadmin \ +# TSFILE_MINIO_SECRET_KEY=minioadmin \ +# TSFILE_MINIO_BUCKET=tsfile-test \ +# pytest python/tests/test_minio_integration.py -v +services: + minio: + image: minio/minio:latest + command: server --address :9000 --console-address :9001 /data1 /data2 /data3 /data4 + ports: + - "9000:9000" + - "9001:9001" + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + volumes: + - minio-data1:/data1 + - minio-data2:/data2 + - minio-data3:/data3 + - minio-data4:/data4 + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 5s + timeout: 5s + retries: 12 + +volumes: + minio-data1: + minio-data2: + minio-data3: + minio-data4: diff --git a/python/tests/test_minio_integration.py b/python/tests/test_minio_integration.py new file mode 100644 index 000000000..2fbc0b524 --- /dev/null +++ b/python/tests/test_minio_integration.py @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os + +import pytest + +s3fs = pytest.importorskip("s3fs") + +from tsfile import ( + ColumnCategory, + ColumnSchema, + TableSchema, + Tablet, + TSDataType, + TsFileReader, + TsFileWriter, +) + +ENDPOINT = os.environ.get("TSFILE_MINIO_ENDPOINT") +ACCESS_KEY = os.environ.get("TSFILE_MINIO_ACCESS_KEY") +SECRET_KEY = os.environ.get("TSFILE_MINIO_SECRET_KEY") +BUCKET = os.environ.get("TSFILE_MINIO_BUCKET", "tsfile-test") + +pytestmark = pytest.mark.skipif( + not (ENDPOINT and ACCESS_KEY and SECRET_KEY), + reason="MinIO integration test needs TSFILE_MINIO_* env vars", +) + + +@pytest.fixture +def fs(): + return s3fs.S3FileSystem( + endpoint_url=ENDPOINT, + key=ACCESS_KEY, + secret=SECRET_KEY, + ) + + +def _write_table(writer): + column1 = ColumnSchema("item_id", TSDataType.STRING, ColumnCategory.TAG) + column2 = ColumnSchema("value", TSDataType.DOUBLE) + writer.register_table(TableSchema("test_table", [column1, column2])) + + tablet = Tablet(["item_id", "value"], [TSDataType.STRING, TSDataType.DOUBLE], 10) + tablet.set_table_name("test_table") + for i in range(10): + tablet.add_timestamp(i, i) + tablet.add_value_by_name("item_id", i, f"id{i}") + tablet.add_value_by_name("value", i, i * 10.0) + writer.write_table(tablet) + + +def test_write_to_object_storage_and_read_back(fs): + fs.mkdir(BUCKET, exist_ok=True) + key = f"{BUCKET}/roundtrip.tsfile" + + # The writer fills the s3fs write buffer; closing `sink` (the outer `with`) + # triggers the multipart upload. + with fs.open(key, "wb") as sink: + with TsFileWriter(sink) as writer: + _write_table(writer) + + with fs.open(key, "rb") as source: + with TsFileReader(source) as reader: + result = reader.query_table("test_table", ["item_id", "value"]) + try: + seen = [] + while result.next(): + seen.append(result.get_value_by_name("item_id")) + assert seen == [f"id{i}" for i in range(10)] + finally: + result.close() diff --git a/python/tests/test_write_sources.py b/python/tests/test_write_sources.py new file mode 100644 index 000000000..27ded9444 --- /dev/null +++ b/python/tests/test_write_sources.py @@ -0,0 +1,86 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import io + +import pytest + +from tsfile import ( + ColumnCategory, + ColumnSchema, + TableSchema, + Tablet, + TSDataType, + TsFileReader, + TsFileWriter, +) + + +def _write_table(writer): + column1 = ColumnSchema("item_id", TSDataType.STRING, ColumnCategory.TAG) + column2 = ColumnSchema("value", TSDataType.DOUBLE) + table = TableSchema("test_table", [column1, column2]) + writer.register_table(table) + + tablet = Tablet(["item_id", "value"], [TSDataType.STRING, TSDataType.DOUBLE], 10) + tablet.set_table_name("test_table") + for i in range(10): + tablet.add_timestamp(i, i) + tablet.add_value_by_name("item_id", i, f"id{i}") + tablet.add_value_by_name("value", i, i * 10.0) + writer.write_table(tablet) + + +def test_write_to_bytesio_and_read_back(): + buf = io.BytesIO() + with TsFileWriter(buf) as writer: + _write_table(writer) + + data = buf.getvalue() + assert len(data) > 0 + + source = io.BytesIO(data) + with TsFileReader(source) as reader: + result = reader.query_table("test_table", ["item_id", "value"]) + try: + seen = [] + while result.next(): + seen.append(result.get_value_by_name("item_id")) + assert seen == [f"id{i}" for i in range(10)] + finally: + result.close() + + +def test_write_target_must_be_writable(): + class NotWritable: + pass + + with pytest.raises(TypeError): + TsFileWriter(NotWritable()) + + +def test_path_and_file_object_produce_same_bytes(tmp_path): + path = tmp_path / "path_write.tsfile" + with TsFileWriter(str(path)) as writer: + _write_table(writer) + path_bytes = path.read_bytes() + + buf = io.BytesIO() + with TsFileWriter(buf) as writer: + _write_table(writer) + + assert buf.getvalue() == path_bytes diff --git a/python/tsfile/python_random_access_write_file.cc b/python/tsfile/python_random_access_write_file.cc new file mode 100644 index 000000000..5368ddbb0 --- /dev/null +++ b/python/tsfile/python_random_access_write_file.cc @@ -0,0 +1,282 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "python_random_access_write_file.h" + +#include +#include +#include +#include +#include +#include + +#include "file/random_access_write_file.h" +#include "utils/errno_define.h" +#include "writer/tsfile_writer.h" + +namespace { + +class PythonTargetLock { + public: + explicit PythonTargetLock(std::mutex& mutex) + : gil_state_(PyGILState_Ensure()), lock_(mutex, std::defer_lock) { + PyThreadState* thread_state = PyEval_SaveThread(); + lock_.lock(); + PyEval_RestoreThread(thread_state); + } + + ~PythonTargetLock() { + lock_.unlock(); + PyGILState_Release(gil_state_); + } + + private: + PyGILState_STATE gil_state_; + std::unique_lock lock_; +}; + +bool has_callable_attribute(PyObject* target, const char* name) { + PyObject* attribute = PyObject_GetAttrString(target, name); + if (attribute == nullptr) { + PyErr_Clear(); + return false; + } + const bool callable = PyCallable_Check(attribute) != 0; + Py_DECREF(attribute); + return callable; +} + +bool seek_to(PyObject* target, int64_t offset, int whence) { + PyObject* result = PyObject_CallMethod( + target, "seek", "Li", static_cast(offset), whence); + if (result == nullptr) { + return false; + } + Py_DECREF(result); + return true; +} + +bool tell_position(PyObject* target, int64_t& position) { + PyObject* result = + PyObject_CallMethod(target, const_cast("tell"), nullptr); + if (result == nullptr) { + return false; + } + const long long value = PyLong_AsLongLong(result); + Py_DECREF(result); + if (value == -1 && PyErr_Occurred()) { + return false; + } + position = static_cast(value); + return true; +} + +int64_t write_bytes(PyObject* target, const char* buf, Py_ssize_t len) { + PyObject* result = + PyObject_CallMethod(target, const_cast("write"), "y#", buf, len); + if (result == nullptr) { + return -1; + } + const long long written = PyLong_AsLongLong(result); + Py_DECREF(result); + if (written < 0 && PyErr_Occurred()) { + return -1; + } + return static_cast(written); +} + +std::string target_name(PyObject* target) { + std::string name(""); + PyObject* value = PyObject_GetAttrString(target, "name"); + if (value == nullptr) { + PyErr_Clear(); + return name; + } + if (PyUnicode_Check(value)) { + const char* text = PyUnicode_AsUTF8(value); + if (text != nullptr) { + name.assign(text); + } else { + PyErr_Clear(); + } + } else if (PyBytes_Check(value)) { + char* text = nullptr; + Py_ssize_t length = 0; + if (PyBytes_AsStringAndSize(value, &text, &length) == 0) { + name.assign(text, static_cast(length)); + } else { + PyErr_Clear(); + } + } + Py_DECREF(value); + return name; +} + +class PythonRandomAccessWriteFile : public storage::RandomAccessWriteFile { + public: + PythonRandomAccessWriteFile(PyObject* target, std::string name) + : target_(target), name_(std::move(name)) { + Py_INCREF(target_); + } + + ~PythonRandomAccessWriteFile() override { close(); } + + bool is_opened() const override { + PythonTargetLock lock(mutex_); + return target_ != nullptr; + } + + const std::string& file_path() const override { return name_; } + + int write(const char* buf, uint32_t len) override { + if (len == 0) { + return common::E_OK; + } + if (buf == nullptr) { + return common::E_INVALID_ARG; + } + PythonTargetLock lock(mutex_); + if (target_ == nullptr) { + return common::E_FILE_WRITE_ERR; + } + uint32_t written = 0; + while (written < len) { + const int64_t cur = write_bytes( + target_, buf + written, static_cast(len - written)); + if (cur <= 0) { + PyErr_Clear(); + return common::E_FILE_WRITE_ERR; + } + written += static_cast(cur); + } + return common::E_OK; + } + + int sync() override { + PythonTargetLock lock(mutex_); + if (target_ == nullptr) { + return common::E_FILE_WRITE_ERR; + } + // flush() is optional for a non-durable sink; ignore a missing method. + PyObject* result = PyObject_CallMethod(target_, "flush", nullptr); + if (result == nullptr) { + PyErr_Clear(); + return common::E_OK; + } + Py_DECREF(result); + return common::E_OK; + } + + int truncate(int64_t size) override { + if (size < 0) { + return common::E_INVALID_ARG; + } + PythonTargetLock lock(mutex_); + if (target_ == nullptr) { + return common::E_FILE_WRITE_ERR; + } + PyObject* result = PyObject_CallMethod(target_, "truncate", "L", + static_cast(size)); + if (result == nullptr) { + PyErr_Clear(); + return common::E_NOT_SUPPORT; + } + Py_DECREF(result); + return common::E_OK; + } + + int seek_to_end() override { + PythonTargetLock lock(mutex_); + if (target_ == nullptr) { + return common::E_FILE_WRITE_ERR; + } + if (!seek_to(target_, 0, SEEK_END)) { + PyErr_Clear(); + return common::E_NOT_SUPPORT; + } + return common::E_OK; + } + + int64_t get_position() override { + PythonTargetLock lock(mutex_); + if (target_ == nullptr) { + return 0; + } + int64_t position = 0; + if (!tell_position(target_, position)) { + PyErr_Clear(); + return 0; + } + return position; + } + + int close() override { + PythonTargetLock lock(mutex_); + if (target_ == nullptr) { + return common::E_OK; + } + Py_DECREF(target_); + target_ = nullptr; + return common::E_OK; + } + + private: + PyObject* target_; + std::string name_; + mutable std::mutex mutex_; +}; + +} // namespace + +void* create_tsfile_writer_from_python_file(PyObject* target, + int32_t* error_code) { + if (error_code == nullptr) { + PyErr_SetString(PyExc_ValueError, "error_code must not be null"); + return nullptr; + } + *error_code = common::E_OK; + if (target == nullptr || !has_callable_attribute(target, "write") || + !has_callable_attribute(target, "tell")) { + *error_code = common::E_INVALID_ARG; + PyErr_SetString(PyExc_TypeError, + "target must be a writable binary file object"); + return nullptr; + } + + try { + *error_code = storage::libtsfile_init(); + if (*error_code != common::E_OK) { + return nullptr; + } + std::unique_ptr write_file( + new PythonRandomAccessWriteFile(target, target_name(target))); + std::unique_ptr writer( + new storage::TsFileWriter()); + *error_code = writer->init(std::move(write_file)); + if (*error_code != common::E_OK) { + return nullptr; + } + return writer.release(); + } catch (const std::bad_alloc&) { + *error_code = common::E_OOM; + } catch (...) { + *error_code = common::E_FILE_OPEN_ERR; + } + return nullptr; +} diff --git a/python/tsfile/python_random_access_write_file.h b/python/tsfile/python_random_access_write_file.h new file mode 100644 index 000000000..44b9ac905 --- /dev/null +++ b/python/tsfile/python_random_access_write_file.h @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef PYTHON_TSFILE_PYTHON_RANDOM_ACCESS_WRITE_FILE_H +#define PYTHON_TSFILE_PYTHON_RANDOM_ACCESS_WRITE_FILE_H + +#define PY_SSIZE_T_CLEAN +#include +#include + +void* create_tsfile_writer_from_python_file(PyObject* target, + int32_t* error_code); + +#endif // PYTHON_TSFILE_PYTHON_RANDOM_ACCESS_WRITE_FILE_H diff --git a/python/tsfile/tsfile_writer.pyx b/python/tsfile/tsfile_writer.pyx index 7f0bd57ff..62612259c 100644 --- a/python/tsfile/tsfile_writer.pyx +++ b/python/tsfile/tsfile_writer.pyx @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. # +import os + import pandas import pyarrow as pa @@ -25,20 +27,36 @@ from tsfile.tablet import Tablet as TabletPy from libc.string cimport memset from libc.stdint cimport uint32_t, uint8_t, uintptr_t from cpython.bytes cimport PyBytes_AsStringAndSize +from cpython.ref cimport PyObject from .tsfile_cpp cimport * from .tsfile_py_cpp cimport * +cdef extern from "python_random_access_write_file.h": + TsFileWriter create_tsfile_writer_from_python_file( + PyObject* target, int32_t* error_code) except? NULL + #cython: language_level=3 cdef class TsFileWriterPy: cdef TsFileWriter writer - def __init__(self, pathname: str, memory_threshold: int = 128 * 1024 * 1024, + def __init__(self, target, memory_threshold: int = 128 * 1024 * 1024, allow_non_decreasing_timestamps: bool = False): + """ + Initialize from a str/bytes/PathLike path, or a writable binary file + object (must have write/tell; seek/flush/truncate are used when + present) so fsspec-style remote targets work. + """ self.writer = NULL set_allow_non_decreasing_timestamps( allow_non_decreasing_timestamps) - self.writer = tsfile_writer_new_c(pathname, memory_threshold) + cdef ErrorCode error_code = 0 + if isinstance(target, (str, bytes, os.PathLike)): + self.writer = tsfile_writer_new_c(os.fspath(target), memory_threshold) + return + self.writer = create_tsfile_writer_from_python_file( + target, &error_code) + check_error(error_code) def register_timeseries(self, device_name : str, timeseries_schema : TimeseriesSchemaPy): """ From 881b624bb1078f235112b5b84dcde8883045411b Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 16 Nov 2026 10:37:26 +0800 Subject: [PATCH 02/16] chore(python): add Apache license headers to the MinIO test files The RAT check failed on the new README.md and docker-compose.yml; add the standard headers. --- python/tests/integration/minio/README.md | 19 +++++++++++++++++++ .../integration/minio/docker-compose.yml | 18 ++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/python/tests/integration/minio/README.md b/python/tests/integration/minio/README.md index 6aeb19f62..14a575980 100644 --- a/python/tests/integration/minio/README.md +++ b/python/tests/integration/minio/README.md @@ -1,3 +1,22 @@ + + # MinIO integration test This directory provisions a MinIO test cluster and documents how to run the diff --git a/python/tests/integration/minio/docker-compose.yml b/python/tests/integration/minio/docker-compose.yml index 541a6b245..fe23ce666 100644 --- a/python/tests/integration/minio/docker-compose.yml +++ b/python/tests/integration/minio/docker-compose.yml @@ -1,3 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + # MinIO test cluster for the TsFile object-storage write/read integration test. # # A single node with four drives enables MinIO erasure coding and presents the From e99af270884a9785b8bb498d27987f0689f668e0 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 16 Nov 2026 10:42:03 +0800 Subject: [PATCH 03/16] fix(python): propagate sink errors and keep pathname compatibility Review follow-ups on the write bridge: - sync() only ignores a missing flush(); a flush that raises now returns E_FILE_SYNC_ERR instead of pretending the data was committed. - get_position() reports failure instead of returning 0, the factory validates tell() and starts from an empty file (seek+truncate when the target supports them), and it applies memory_threshold like the path-based writer. - TsFileWriter keeps the pathname keyword (it still accepts bytes and PathLike) and tsfile_writer_new_c now uses os.fsencode, so bytes paths work. - write_file_get_buffer() rejects a sink larger than 4 GiB instead of truncating the reported length, and init(unique_ptr) keeps ownership until initialization succeeds so a throw cannot leak the sink. --- cpp/src/cwrapper/tsfile_cwrapper.cc | 3 ++ cpp/src/writer/tsfile_writer.cc | 6 ++- python/tests/test_write_sources.py | 42 +++++++++++++++++ .../tsfile/python_random_access_write_file.cc | 47 +++++++++++++++++-- .../tsfile/python_random_access_write_file.h | 1 + python/tsfile/tsfile_py_cpp.pyx | 2 +- python/tsfile/tsfile_writer.pyx | 16 ++++--- 7 files changed, 104 insertions(+), 13 deletions(-) diff --git a/cpp/src/cwrapper/tsfile_cwrapper.cc b/cpp/src/cwrapper/tsfile_cwrapper.cc index 94f6092eb..f2fcaf9dd 100644 --- a/cpp/src/cwrapper/tsfile_cwrapper.cc +++ b/cpp/src/cwrapper/tsfile_cwrapper.cc @@ -260,6 +260,9 @@ ERRNO write_file_get_buffer(WriteFile write_file, uint8_t** buffer, if (data.empty()) { return common::E_OK; } + if (data.size() > 0xFFFFFFFFULL) { + return common::E_OVERFLOW; + } void* copy = malloc(data.size()); if (copy == nullptr) { return common::E_OOM; diff --git a/cpp/src/writer/tsfile_writer.cc b/cpp/src/writer/tsfile_writer.cc index 1d84d78bc..5a6152128 100644 --- a/cpp/src/writer/tsfile_writer.cc +++ b/cpp/src/writer/tsfile_writer.cc @@ -155,12 +155,14 @@ int TsFileWriter::init(std::unique_ptr write_file) { if (!write_file || !write_file->is_opened()) { return E_INVALID_ARG; } - RandomAccessWriteFile* raw = write_file.release(); + // Keep the unique_ptr owning the sink until init() succeeds; if it throws + // (e.g. while allocating TsFileIOWriter) the sink is still released. + RandomAccessWriteFile* raw = write_file.get(); const int ret = init(raw); if (ret != E_OK) { - delete raw; return ret; } + write_file.release(); // init(raw) stored the pointer without taking ownership; transfer it now // so destroy() deletes it exactly once. write_file_created_ = true; diff --git a/python/tests/test_write_sources.py b/python/tests/test_write_sources.py index 27ded9444..49992a8f2 100644 --- a/python/tests/test_write_sources.py +++ b/python/tests/test_write_sources.py @@ -84,3 +84,45 @@ def test_path_and_file_object_produce_same_bytes(tmp_path): _write_table(writer) assert buf.getvalue() == path_bytes + + +def test_pathname_keyword_and_bytes_path(tmp_path): + keyword_path = tmp_path / "keyword.tsfile" + with TsFileWriter(pathname=str(keyword_path)) as writer: + _write_table(writer) + keyword_bytes = keyword_path.read_bytes() + assert len(keyword_bytes) > 0 + + bytes_path = str(tmp_path / "bytes.tsfile").encode() + with TsFileWriter(bytes_path) as writer: + _write_table(writer) + assert (tmp_path / "bytes.tsfile").read_bytes() == keyword_bytes + + +def test_file_object_with_existing_content_is_reset(): + buf = io.BytesIO(b"stale trailing bytes" * 100) + buf.seek(1000) + with TsFileWriter(buf) as writer: + _write_table(writer) + + data = buf.getvalue() + with TsFileReader(io.BytesIO(data)) as reader: + result = reader.query_table("test_table", ["item_id", "value"]) + try: + seen = [] + while result.next(): + seen.append(result.get_value_by_name("item_id")) + assert seen == [f"id{i}" for i in range(10)] + finally: + result.close() + + +def test_flush_failure_is_reported(): + class FailingFlush(io.BytesIO): + def flush(self): + raise OSError("disk full") + + target = FailingFlush() + with pytest.raises(Exception): + with TsFileWriter(target) as writer: + _write_table(writer) diff --git a/python/tsfile/python_random_access_write_file.cc b/python/tsfile/python_random_access_write_file.cc index 5368ddbb0..c6f5a83aa 100644 --- a/python/tsfile/python_random_access_write_file.cc +++ b/python/tsfile/python_random_access_write_file.cc @@ -26,6 +26,7 @@ #include #include +#include "common/global.h" #include "file/random_access_write_file.h" #include "utils/errno_define.h" #include "writer/tsfile_writer.h" @@ -173,11 +174,15 @@ class PythonRandomAccessWriteFile : public storage::RandomAccessWriteFile { if (target_ == nullptr) { return common::E_FILE_WRITE_ERR; } - // flush() is optional for a non-durable sink; ignore a missing method. + // flush() is optional for a non-durable sink, but if the target has + // one, a failure means the data may not be committed. + if (!has_callable_attribute(target_, "flush")) { + return common::E_OK; + } PyObject* result = PyObject_CallMethod(target_, "flush", nullptr); if (result == nullptr) { PyErr_Clear(); - return common::E_OK; + return common::E_FILE_SYNC_ERR; } Py_DECREF(result); return common::E_OK; @@ -221,7 +226,7 @@ class PythonRandomAccessWriteFile : public storage::RandomAccessWriteFile { int64_t position = 0; if (!tell_position(target_, position)) { PyErr_Clear(); - return 0; + return -1; } return position; } @@ -245,6 +250,7 @@ class PythonRandomAccessWriteFile : public storage::RandomAccessWriteFile { } // namespace void* create_tsfile_writer_from_python_file(PyObject* target, + uint64_t memory_threshold, int32_t* error_code) { if (error_code == nullptr) { PyErr_SetString(PyExc_ValueError, "error_code must not be null"); @@ -259,11 +265,46 @@ void* create_tsfile_writer_from_python_file(PyObject* target, return nullptr; } + // The file must start empty: a header written at a nonzero cursor is not + // a standalone TsFile, and bytes left over from a longer previous file + // would trail the new one. + int64_t initial_position = 0; + if (!tell_position(target, initial_position)) { + PyErr_Clear(); + *error_code = common::E_INVALID_ARG; + PyErr_SetString(PyExc_TypeError, "target must support tell()"); + return nullptr; + } + if (has_callable_attribute(target, "seek")) { + if (!seek_to(target, 0, SEEK_SET)) { + PyErr_Clear(); + *error_code = common::E_INVALID_ARG; + PyErr_SetString(PyExc_ValueError, "target must be seekable"); + return nullptr; + } + } else if (initial_position != 0) { + *error_code = common::E_INVALID_ARG; + PyErr_SetString(PyExc_ValueError, + "target must be positioned at 0 before writing"); + return nullptr; + } + if (has_callable_attribute(target, "truncate")) { + PyObject* truncated = PyObject_CallMethod(target, "truncate", "i", 0); + if (truncated == nullptr) { + PyErr_Clear(); + *error_code = common::E_INVALID_ARG; + PyErr_SetString(PyExc_ValueError, "target could not be truncated"); + return nullptr; + } + Py_DECREF(truncated); + } + try { *error_code = storage::libtsfile_init(); if (*error_code != common::E_OK) { return nullptr; } + common::g_config_value_.chunk_group_size_threshold_ = memory_threshold; std::unique_ptr write_file( new PythonRandomAccessWriteFile(target, target_name(target))); std::unique_ptr writer( diff --git a/python/tsfile/python_random_access_write_file.h b/python/tsfile/python_random_access_write_file.h index 44b9ac905..e149bf5c9 100644 --- a/python/tsfile/python_random_access_write_file.h +++ b/python/tsfile/python_random_access_write_file.h @@ -25,6 +25,7 @@ #include void* create_tsfile_writer_from_python_file(PyObject* target, + uint64_t memory_threshold, int32_t* error_code); #endif // PYTHON_TSFILE_PYTHON_RANDOM_ACCESS_WRITE_FILE_H diff --git a/python/tsfile/tsfile_py_cpp.pyx b/python/tsfile/tsfile_py_cpp.pyx index b295a1ba9..92816f4b9 100644 --- a/python/tsfile/tsfile_py_cpp.pyx +++ b/python/tsfile/tsfile_py_cpp.pyx @@ -821,7 +821,7 @@ cdef void free_c_row_record(TsRecord record): cdef TsFileWriter tsfile_writer_new_c(object pathname, uint64_t memory_threshold) except NULL: cdef ErrorCode errno = 0 cdef TsFileWriter writer = NULL - cdef bytes encoded_path = PyUnicode_AsUTF8String(pathname) + cdef bytes encoded_path = os.fsencode(pathname) cdef const char * c_path = encoded_path writer = _tsfile_writer_new(c_path, memory_threshold, &errno) check_error(errno) diff --git a/python/tsfile/tsfile_writer.pyx b/python/tsfile/tsfile_writer.pyx index 62612259c..9f491e72d 100644 --- a/python/tsfile/tsfile_writer.pyx +++ b/python/tsfile/tsfile_writer.pyx @@ -33,29 +33,31 @@ from .tsfile_py_cpp cimport * cdef extern from "python_random_access_write_file.h": TsFileWriter create_tsfile_writer_from_python_file( - PyObject* target, int32_t* error_code) except? NULL + PyObject* target, uint64_t memory_threshold, + int32_t* error_code) except? NULL #cython: language_level=3 cdef class TsFileWriterPy: cdef TsFileWriter writer - def __init__(self, target, memory_threshold: int = 128 * 1024 * 1024, + def __init__(self, pathname, memory_threshold: int = 128 * 1024 * 1024, allow_non_decreasing_timestamps: bool = False): """ Initialize from a str/bytes/PathLike path, or a writable binary file - object (must have write/tell; seek/flush/truncate are used when - present) so fsspec-style remote targets work. + object for fsspec-style remote targets. File objects must support + write/tell; seek (when present) and truncate (when present) are used + to start from an empty file. """ self.writer = NULL set_allow_non_decreasing_timestamps( allow_non_decreasing_timestamps) cdef ErrorCode error_code = 0 - if isinstance(target, (str, bytes, os.PathLike)): - self.writer = tsfile_writer_new_c(os.fspath(target), memory_threshold) + if isinstance(pathname, (str, bytes, os.PathLike)): + self.writer = tsfile_writer_new_c(pathname, memory_threshold) return self.writer = create_tsfile_writer_from_python_file( - target, &error_code) + pathname, memory_threshold, &error_code) check_error(error_code) def register_timeseries(self, device_name : str, timeseries_schema : TimeseriesSchemaPy): From 2ec4b69a35b0633892251446317d88459ee11a8e Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 16 Nov 2026 10:45:32 +0800 Subject: [PATCH 04/16] test(python): drop the MinIO integration test It would add a Docker/MinIO dependency to the test tree for a path the file-like unit tests already cover; keep only test_write_sources.py. --- python/tests/integration/minio/README.md | 72 --------------- .../integration/minio/docker-compose.yml | 58 ------------- python/tests/test_minio_integration.py | 87 ------------------- 3 files changed, 217 deletions(-) delete mode 100644 python/tests/integration/minio/README.md delete mode 100644 python/tests/integration/minio/docker-compose.yml delete mode 100644 python/tests/test_minio_integration.py diff --git a/python/tests/integration/minio/README.md b/python/tests/integration/minio/README.md deleted file mode 100644 index 14a575980..000000000 --- a/python/tests/integration/minio/README.md +++ /dev/null @@ -1,72 +0,0 @@ - - -# MinIO integration test - -This directory provisions a MinIO test cluster and documents how to run the -object-storage write/read integration test. - -## Why MinIO - -The C++ core has no HTTP client, so TsFile writes to object storage go through -the Python binding: `TsFileWriter(target)` accepts a writable binary file -object, and fsspec/s3fs supplies one that uploads to S3-compatible storage on -close. MinIO provides that S3-compatible endpoint locally, without a cloud -account. - -## Bring up the cluster - -```bash -docker compose up -d -``` - -This starts a single MinIO node with four drives (erasure coding enabled), the -lightweight equivalent of a distributed cluster for test purposes. The S3 -endpoint is `http://127.0.0.1:9000` and the console is -`http://127.0.0.1:9001`. Credentials are the MinIO defaults (`minioadmin` / -`minioadmin`). - -## Install the S3 client - -The test needs `s3fs` (and its `fsspec` dependency), which is not part of the -core Python dependencies: - -```bash -pip install s3fs -``` - -## Run the integration test - -```bash -TSFILE_MINIO_ENDPOINT=http://127.0.0.1:9000 \ -TSFILE_MINIO_ACCESS_KEY=minioadmin \ -TSFILE_MINIO_SECRET_KEY=minioadmin \ -TSFILE_MINIO_BUCKET=tsfile-test \ -pytest tests/test_minio_integration.py -v -``` - -Without `s3fs` installed or the `TSFILE_MINIO_*` variables set, the test is -skipped rather than failed. - -## What the test covers - -The test writes a TsFile into the object store through `TsFileWriter` (the new -write-sink bridge), lets s3fs upload it on close, then reads it back through -`TsFileReader` and verifies the rows. This exercises the same C++ write/read -paths a non-local filesystem would. diff --git a/python/tests/integration/minio/docker-compose.yml b/python/tests/integration/minio/docker-compose.yml deleted file mode 100644 index fe23ce666..000000000 --- a/python/tests/integration/minio/docker-compose.yml +++ /dev/null @@ -1,58 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# MinIO test cluster for the TsFile object-storage write/read integration test. -# -# A single node with four drives enables MinIO erasure coding and presents the -# same S3 API a multi-node distributed deployment does, without needing four -# VMs. This is the standard lightweight "cluster" for a test environment. -# -# Bring it up: -# docker compose up -d -# Then run the gated Python test: -# TSFILE_MINIO_ENDPOINT=http://127.0.0.1:9000 \ -# TSFILE_MINIO_ACCESS_KEY=minioadmin \ -# TSFILE_MINIO_SECRET_KEY=minioadmin \ -# TSFILE_MINIO_BUCKET=tsfile-test \ -# pytest python/tests/test_minio_integration.py -v -services: - minio: - image: minio/minio:latest - command: server --address :9000 --console-address :9001 /data1 /data2 /data3 /data4 - ports: - - "9000:9000" - - "9001:9001" - environment: - MINIO_ROOT_USER: minioadmin - MINIO_ROOT_PASSWORD: minioadmin - volumes: - - minio-data1:/data1 - - minio-data2:/data2 - - minio-data3:/data3 - - minio-data4:/data4 - healthcheck: - test: ["CMD", "mc", "ready", "local"] - interval: 5s - timeout: 5s - retries: 12 - -volumes: - minio-data1: - minio-data2: - minio-data3: - minio-data4: diff --git a/python/tests/test_minio_integration.py b/python/tests/test_minio_integration.py deleted file mode 100644 index 2fbc0b524..000000000 --- a/python/tests/test_minio_integration.py +++ /dev/null @@ -1,87 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import os - -import pytest - -s3fs = pytest.importorskip("s3fs") - -from tsfile import ( - ColumnCategory, - ColumnSchema, - TableSchema, - Tablet, - TSDataType, - TsFileReader, - TsFileWriter, -) - -ENDPOINT = os.environ.get("TSFILE_MINIO_ENDPOINT") -ACCESS_KEY = os.environ.get("TSFILE_MINIO_ACCESS_KEY") -SECRET_KEY = os.environ.get("TSFILE_MINIO_SECRET_KEY") -BUCKET = os.environ.get("TSFILE_MINIO_BUCKET", "tsfile-test") - -pytestmark = pytest.mark.skipif( - not (ENDPOINT and ACCESS_KEY and SECRET_KEY), - reason="MinIO integration test needs TSFILE_MINIO_* env vars", -) - - -@pytest.fixture -def fs(): - return s3fs.S3FileSystem( - endpoint_url=ENDPOINT, - key=ACCESS_KEY, - secret=SECRET_KEY, - ) - - -def _write_table(writer): - column1 = ColumnSchema("item_id", TSDataType.STRING, ColumnCategory.TAG) - column2 = ColumnSchema("value", TSDataType.DOUBLE) - writer.register_table(TableSchema("test_table", [column1, column2])) - - tablet = Tablet(["item_id", "value"], [TSDataType.STRING, TSDataType.DOUBLE], 10) - tablet.set_table_name("test_table") - for i in range(10): - tablet.add_timestamp(i, i) - tablet.add_value_by_name("item_id", i, f"id{i}") - tablet.add_value_by_name("value", i, i * 10.0) - writer.write_table(tablet) - - -def test_write_to_object_storage_and_read_back(fs): - fs.mkdir(BUCKET, exist_ok=True) - key = f"{BUCKET}/roundtrip.tsfile" - - # The writer fills the s3fs write buffer; closing `sink` (the outer `with`) - # triggers the multipart upload. - with fs.open(key, "wb") as sink: - with TsFileWriter(sink) as writer: - _write_table(writer) - - with fs.open(key, "rb") as source: - with TsFileReader(source) as reader: - result = reader.query_table("test_table", ["item_id", "value"]) - try: - seen = [] - while result.next(): - seen.append(result.get_value_by_name("item_id")) - assert seen == [f"id{i}" for i in range(10)] - finally: - result.close() From 088639b98a3ac06f909afc45642dc7dfed602ee6 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 16 Nov 2026 11:05:07 +0800 Subject: [PATCH 05/16] build(python): ship the write bridge header in the sdist setup.py lists python_random_access_write_file.cc as an Extension source, so setuptools copies it into an sdist automatically, but the header is only referenced through depends= and was left out; CI builds the sdist and failed with 'python_random_access_write_file.h: No such file'. --- python/MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/python/MANIFEST.in b/python/MANIFEST.in index 1981d9212..e90457964 100644 --- a/python/MANIFEST.in +++ b/python/MANIFEST.in @@ -19,4 +19,5 @@ include _build_support.py recursive-include tsfile *.pyx *.pxd *.pxi include tsfile/python_random_access_read_file.h +include tsfile/python_random_access_write_file.h prune tsfile/include From 68677dd93764eccdbe7094bac9622137f4d86fdb Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 16 Nov 2026 11:18:46 +0800 Subject: [PATCH 06/16] fix(python): accept forward-only sinks and release writers on close failure - A forward-only io.RawIOBase inherits callable seek()/truncate() that raise io.UnsupportedOperation; treat a failed rewind as "not seekable" and accept the target when it already starts at 0, and only require truncate when earlier content has to be cleared. Added a regression test with a write-only stream. - _tsfile_writer_close() now deletes the writer even when close() fails, and TsFileWriterPy drops its handle before raising, so a persistent flush error no longer leaks the native writer or its Python target. --- cpp/src/cwrapper/tsfile_cwrapper.cc | 5 +-- python/tests/test_write_sources.py | 40 +++++++++++++++++++ .../tsfile/python_random_access_write_file.cc | 31 +++++++++----- python/tsfile/tsfile_writer.pyx | 5 ++- 4 files changed, 66 insertions(+), 15 deletions(-) diff --git a/cpp/src/cwrapper/tsfile_cwrapper.cc b/cpp/src/cwrapper/tsfile_cwrapper.cc index f2fcaf9dd..2eca4d607 100644 --- a/cpp/src/cwrapper/tsfile_cwrapper.cc +++ b/cpp/src/cwrapper/tsfile_cwrapper.cc @@ -2529,9 +2529,8 @@ ERRNO _tsfile_writer_write_ts_record(TsFileWriter writer, TsRecord data) { ERRNO _tsfile_writer_close(TsFileWriter writer) { auto* w = static_cast(writer); int ret = w->close(); - if (ret != common::E_OK) { - return ret; - } + // Release the writer even when close() failed: the handle must not leak, + // and destroy() is the documented cleanup path for a failed close. delete w; return ret; } diff --git a/python/tests/test_write_sources.py b/python/tests/test_write_sources.py index 49992a8f2..e25b036df 100644 --- a/python/tests/test_write_sources.py +++ b/python/tests/test_write_sources.py @@ -126,3 +126,43 @@ def flush(self): with pytest.raises(Exception): with TsFileWriter(target) as writer: _write_table(writer) + + +class _ForwardOnlyStream(io.RawIOBase): + """A write-only sink: seek/truncate exist but raise, like a pipe.""" + + def __init__(self): + self._buffer = bytearray() + + def writable(self): + return True + + def write(self, data): + self._buffer += data + return len(data) + + def tell(self): + return len(self._buffer) + + +def test_forward_only_stream_is_accepted(): + target = _ForwardOnlyStream() + with TsFileWriter(target) as writer: + _write_table(writer) + + with TsFileReader(io.BytesIO(bytes(target._buffer))) as reader: + result = reader.query_table("test_table", ["item_id", "value"]) + try: + seen = [] + while result.next(): + seen.append(result.get_value_by_name("item_id")) + assert seen == [f"id{i}" for i in range(10)] + finally: + result.close() + + +def test_forward_only_stream_at_nonzero_position_is_rejected(): + target = _ForwardOnlyStream() + target._buffer += b"prefix" + with pytest.raises(ValueError): + TsFileWriter(target) diff --git a/python/tsfile/python_random_access_write_file.cc b/python/tsfile/python_random_access_write_file.cc index c6f5a83aa..66839c83a 100644 --- a/python/tsfile/python_random_access_write_file.cc +++ b/python/tsfile/python_random_access_write_file.cc @@ -275,28 +275,39 @@ void* create_tsfile_writer_from_python_file(PyObject* target, PyErr_SetString(PyExc_TypeError, "target must support tell()"); return nullptr; } + // Rewind when the target allows it. A forward-only stream inherits + // callable seek()/truncate() from io.RawIOBase that raise + // io.UnsupportedOperation, so treating a callable method as support would + // reject freshly opened streams; only the position check matters then. + bool rewound = false; if (has_callable_attribute(target, "seek")) { - if (!seek_to(target, 0, SEEK_SET)) { + if (seek_to(target, 0, SEEK_SET)) { + rewound = true; + } else { PyErr_Clear(); - *error_code = common::E_INVALID_ARG; - PyErr_SetString(PyExc_ValueError, "target must be seekable"); - return nullptr; } - } else if (initial_position != 0) { + } + if (!rewound && initial_position != 0) { *error_code = common::E_INVALID_ARG; PyErr_SetString(PyExc_ValueError, "target must be positioned at 0 before writing"); return nullptr; } - if (has_callable_attribute(target, "truncate")) { + if (rewound && has_callable_attribute(target, "truncate")) { PyObject* truncated = PyObject_CallMethod(target, "truncate", "i", 0); if (truncated == nullptr) { PyErr_Clear(); - *error_code = common::E_INVALID_ARG; - PyErr_SetString(PyExc_ValueError, "target could not be truncated"); - return nullptr; + // At position 0 there may be nothing stale to clear; a target + // that started with content must support truncate. + if (initial_position != 0) { + *error_code = common::E_INVALID_ARG; + PyErr_SetString(PyExc_ValueError, + "target could not be truncated"); + return nullptr; + } + } else { + Py_DECREF(truncated); } - Py_DECREF(truncated); } try { diff --git a/python/tsfile/tsfile_writer.pyx b/python/tsfile/tsfile_writer.pyx index 9f491e72d..d02a6a5d6 100644 --- a/python/tsfile/tsfile_writer.pyx +++ b/python/tsfile/tsfile_writer.pyx @@ -228,9 +228,10 @@ cdef class TsFileWriterPy: cdef ErrorCode errno if self.writer == NULL: return - errno = _tsfile_writer_close(self.writer) - check_error(errno) + cdef TsFileWriter closing = self.writer self.writer = NULL + errno = _tsfile_writer_close(closing) + check_error(errno) cpdef flush(self): """ From fd4e3fc60cce5796f2fade45482b2dcd05f02e6c Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 16 Nov 2026 11:31:25 +0800 Subject: [PATCH 07/16] build(python): keep the write bridge header through mvn clean python/pom.xml's clean plugin deletes tsfile/*.h to drop generated SDK headers, with an exception only for the read bridge header. CI runs clean verify, so the write header was removed before build_ext. Add it to the excludes. --- python/pom.xml | 1 + python/tests/test_write_sources.py | 15 +++++++++++++++ .../tsfile/python_random_access_write_file.cc | 18 ++++++++++++++---- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/python/pom.xml b/python/pom.xml index 33d3338cb..f3fa349fa 100644 --- a/python/pom.xml +++ b/python/pom.xml @@ -244,6 +244,7 @@ python_random_access_read_file.h + python_random_access_write_file.h diff --git a/python/tests/test_write_sources.py b/python/tests/test_write_sources.py index e25b036df..94730d1cc 100644 --- a/python/tests/test_write_sources.py +++ b/python/tests/test_write_sources.py @@ -166,3 +166,18 @@ def test_forward_only_stream_at_nonzero_position_is_rejected(): target._buffer += b"prefix" with pytest.raises(ValueError): TsFileWriter(target) + + +class _SeekableWithoutTruncate(io.BytesIO): + def truncate(self, size=None): + raise io.UnsupportedOperation("truncate") + + +def test_seekable_target_without_truncate_must_be_empty(): + with pytest.raises(ValueError): + TsFileWriter(_SeekableWithoutTruncate(b"stale")) + + target = _SeekableWithoutTruncate() + with TsFileWriter(target) as writer: + _write_table(writer) + assert len(target.getvalue()) > 0 diff --git a/python/tsfile/python_random_access_write_file.cc b/python/tsfile/python_random_access_write_file.cc index 66839c83a..87369874f 100644 --- a/python/tsfile/python_random_access_write_file.cc +++ b/python/tsfile/python_random_access_write_file.cc @@ -297,12 +297,22 @@ void* create_tsfile_writer_from_python_file(PyObject* target, PyObject* truncated = PyObject_CallMethod(target, "truncate", "i", 0); if (truncated == nullptr) { PyErr_Clear(); - // At position 0 there may be nothing stale to clear; a target - // that started with content must support truncate. - if (initial_position != 0) { + // Truncation is unsupported: tolerate it only if the target is + // provably empty, otherwise a shorter write would leave stale + // trailing bytes behind a valid-looking file. + int64_t end_position = 0; + const bool sized = seek_to(target, 0, SEEK_END) && + tell_position(target, end_position); + if (!sized) { + PyErr_Clear(); + } + if (!sized || end_position != 0 || + !seek_to(target, 0, SEEK_SET)) { + PyErr_Clear(); *error_code = common::E_INVALID_ARG; PyErr_SetString(PyExc_ValueError, - "target could not be truncated"); + "target has existing content and cannot be " + "truncated"); return nullptr; } } else { From db6e1d13c19f55a9bee59d756a8e1421f5808c5a Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 16 Nov 2026 11:35:12 +0800 Subject: [PATCH 08/16] build(python): clean only generated artifacts in python/tsfile mvn clean used to delete every *.h there, which removed the hand-written bridge headers (each needed its own exception) along with the generated Cython public-API headers. Switch to an explicit list: compiled extensions, generated .cpp, and the two Cython-generated headers; the bridge headers now survive clean without per-file exceptions. --- python/pom.xml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/python/pom.xml b/python/pom.xml index f3fa349fa..7be934308 100644 --- a/python/pom.xml +++ b/python/pom.xml @@ -235,17 +235,20 @@ ${basedir}/tsfile + *.so* *.dylib *.dll - *.h *.cpp + + tsfile_py_cpp.h + tsfile_py_cpp_api.h - - python_random_access_read_file.h - python_random_access_write_file.h - tsfile.egg-info From 459977159c91012a0fb91b1ca26a7096f3419785 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 16 Nov 2026 11:58:33 +0800 Subject: [PATCH 09/16] fix(python): track the sink position and initialize memory handles Review pass over the write bridge: - get_position() no longer asks the target for tell() after creation. TsFileIOWriter records chunk offsets from it after every flush, and a stream whose tell() becomes unavailable mid-write would have silently produced -1 offsets; track bytes written internally instead (updated by seek_to_end too). - write_file_new_memory() now runs the same lazy init_tsfile_config() as write_file_new(), so a writer built on a memory handle in a fresh process does not read zero-initialized global config. - Document writable file objects in python/README (reset semantics, forward-only streams, FileSyncError), and add C-API and Python tests: memory-handle round trip through tsfile_writer_new, memory_threshold propagation, and a target whose tell() fails after the first call. --- cpp/src/cwrapper/tsfile_cwrapper.cc | 4 ++ cpp/test/cwrapper/cwrapper_test.cc | 71 +++++++++++++++++++ python/README.md | 19 +++++ python/tests/test_write_sources.py | 44 +++++++++++- .../tsfile/python_random_access_write_file.cc | 26 +++---- python/tsfile/tsfile_writer.pyx | 4 +- 6 files changed, 153 insertions(+), 15 deletions(-) diff --git a/cpp/src/cwrapper/tsfile_cwrapper.cc b/cpp/src/cwrapper/tsfile_cwrapper.cc index 2eca4d607..f1fcc2ba0 100644 --- a/cpp/src/cwrapper/tsfile_cwrapper.cc +++ b/cpp/src/cwrapper/tsfile_cwrapper.cc @@ -231,6 +231,10 @@ WriteFile write_file_new_memory(ERRNO* err_code) { if (err_code == nullptr) { return nullptr; } + // Same lazy initialization as write_file_new(): a writer built on this + // handle must not see the zero-initialized global config of a process + // that has not touched any other API yet. + init_tsfile_config(); storage::MemoryRandomAccessWriteFile* file = new (std::nothrow) storage::MemoryRandomAccessWriteFile(); if (file == nullptr) { diff --git a/cpp/test/cwrapper/cwrapper_test.cc b/cpp/test/cwrapper/cwrapper_test.cc index dfe13e101..ee9a25104 100644 --- a/cpp/test/cwrapper/cwrapper_test.cc +++ b/cpp/test/cwrapper/cwrapper_test.cc @@ -762,4 +762,75 @@ TEST(CApiCompatibilityTest, KeepsLegacySchemaAndTagFilterEntryPoints) { nullptr); } +// The in-memory write sink lets a C caller serialize a TsFile without a +// filesystem and upload the bytes themselves (write_file_get_buffer). +TEST_F(CWrapperTest, MemoryWriteFileRoundTrip) { + ERRNO code = RET_OK; + WriteFile file = write_file_new_memory(&code); + ASSERT_EQ(RET_OK, code); + ASSERT_NE(nullptr, file); + + std::vector columns; + columns.push_back( + ColumnSchema{const_cast("device"), TS_DATATYPE_STRING, TAG}); + columns.push_back( + ColumnSchema{const_cast("value"), TS_DATATYPE_INT64, FIELD}); + TableSchema schema = {const_cast("memtable"), columns.data(), + static_cast(columns.size())}; + + TsFileWriter writer = tsfile_writer_new(file, &schema, &code); + ASSERT_EQ(RET_OK, code); + ASSERT_NE(nullptr, writer); + + char* names[] = {const_cast("device"), const_cast("value")}; + TSDataType types[] = {TS_DATATYPE_STRING, TS_DATATYPE_INT64}; + Tablet tablet = tablet_new(names, types, 2, 4, &code); + ASSERT_EQ(RET_OK, code); + ASSERT_NE(nullptr, tablet); + for (uint32_t row = 0; row < 4; row++) { + ASSERT_EQ(RET_OK, tablet_add_timestamp(tablet, row, row)); + ASSERT_EQ(RET_OK, tablet_add_value_by_name_string_with_len( + tablet, row, names[0], "d0", 2)); + ASSERT_EQ(RET_OK, tablet_add_value_by_name_int64_t(tablet, row, + names[1], row * 10)); + } + ASSERT_EQ(RET_OK, tsfile_writer_write(writer, tablet)); + ASSERT_EQ(RET_OK, tsfile_writer_close(writer)); + free_tablet(&tablet); + + uint8_t* buffer = nullptr; + uint32_t length = 0; + ASSERT_EQ(RET_OK, write_file_get_buffer(file, &buffer, &length)); + ASSERT_NE(nullptr, buffer); + ASSERT_GT(length, 0u); + + const std::string path = "cwrapper_memory_write_round_trip.tsfile"; + { + FILE* out = fopen(path.c_str(), "wb"); + ASSERT_NE(nullptr, out); + ASSERT_EQ(length, fwrite(buffer, 1, length, out)); + fclose(out); + } + tsfile_free_buffer(buffer); + free_write_file(&file); + + TsFileReader reader = tsfile_reader_new(path.c_str(), &code); + ASSERT_EQ(RET_OK, code); + ASSERT_NE(nullptr, reader); + ResultSet result = + tsfile_query_table(reader, "memtable", names, 2, 0, 100, &code); + ASSERT_EQ(RET_OK, code); + int rows = 0; + while (tsfile_result_set_next(result, &code) && code == RET_OK) { + EXPECT_EQ(static_cast(rows * 10), + tsfile_result_set_get_value_by_name_int64_t(result, "value")); + rows++; + } + EXPECT_EQ(RET_OK, code); + EXPECT_EQ(4, rows); + free_tsfile_result_set(&result); + tsfile_reader_close(reader); + remove(path.c_str()); +} + } // namespace cwrapper diff --git a/python/README.md b/python/README.md index 7980e91db..f342d59a6 100644 --- a/python/README.md +++ b/python/README.md @@ -219,3 +219,22 @@ The object must provide `seek()`, `tell()`, and explicitly sized binary `read(size)` operations. The caller owns the object: `TsFileReader` keeps it alive while in use, preserves its cursor position, and does not close it. The local `FileReadBackend` setting does not apply to file objects. + +`TsFileWriter` accepts a writable binary file object as well, which lets a +TsFile be written straight to an object store: + +```python +import fsspec +from tsfile import TsFileWriter + +with fsspec.open("s3://bucket/output.tsfile", "wb") as target: + with TsFileWriter(target) as writer: + writer.register_table(table_schema) + writer.write_table(tablet) +``` + +The target must provide `write()` and `tell()`. When it is seekable its +existing content is discarded; a forward-only stream (for example a pipe) is +accepted only while it is positioned at 0. A failing `flush()` is reported as +`FileSyncError` instead of being ignored. As with the reader, the caller owns +the object and closes it. diff --git a/python/tests/test_write_sources.py b/python/tests/test_write_sources.py index 94730d1cc..01d71e8a7 100644 --- a/python/tests/test_write_sources.py +++ b/python/tests/test_write_sources.py @@ -118,16 +118,58 @@ def test_file_object_with_existing_content_is_reset(): def test_flush_failure_is_reported(): + from tsfile import FileSyncError + class FailingFlush(io.BytesIO): def flush(self): raise OSError("disk full") target = FailingFlush() - with pytest.raises(Exception): + with pytest.raises(FileSyncError): with TsFileWriter(target) as writer: _write_table(writer) +def test_memory_threshold_is_applied(): + from tsfile import get_tsfile_config + + from tsfile import set_tsfile_config + + original = get_tsfile_config()["chunk_group_size_threshold_"] + try: + with TsFileWriter(io.BytesIO(), memory_threshold=4096) as writer: + _write_table(writer) + assert get_tsfile_config()["chunk_group_size_threshold_"] == 4096 + finally: + set_tsfile_config({"chunk_group_size_threshold_": original}) + + +def test_position_does_not_depend_on_tell_after_start(): + class FlakyTell(io.BytesIO): + def __init__(self, *args): + super().__init__(*args) + self._calls = 0 + + def tell(self): + self._calls += 1 + if self._calls > 1: + raise OSError("tell is not supported while writing") + return super().tell() + + target = FlakyTell() + with TsFileWriter(target) as writer: + _write_table(writer) + with TsFileReader(io.BytesIO(target.getvalue())) as reader: + result = reader.query_table("test_table", ["item_id", "value"]) + try: + seen = [] + while result.next(): + seen.append(result.get_value_by_name("item_id")) + assert seen == [f"id{i}" for i in range(10)] + finally: + result.close() + + class _ForwardOnlyStream(io.RawIOBase): """A write-only sink: seek/truncate exist but raise, like a pipe.""" diff --git a/python/tsfile/python_random_access_write_file.cc b/python/tsfile/python_random_access_write_file.cc index 87369874f..34397071e 100644 --- a/python/tsfile/python_random_access_write_file.cc +++ b/python/tsfile/python_random_access_write_file.cc @@ -132,7 +132,7 @@ std::string target_name(PyObject* target) { class PythonRandomAccessWriteFile : public storage::RandomAccessWriteFile { public: PythonRandomAccessWriteFile(PyObject* target, std::string name) - : target_(target), name_(std::move(name)) { + : target_(target), name_(std::move(name)), position_(0) { Py_INCREF(target_); } @@ -166,6 +166,7 @@ class PythonRandomAccessWriteFile : public storage::RandomAccessWriteFile { } written += static_cast(cur); } + position_ += written; return common::E_OK; } @@ -215,20 +216,21 @@ class PythonRandomAccessWriteFile : public storage::RandomAccessWriteFile { PyErr_Clear(); return common::E_NOT_SUPPORT; } + int64_t end_position = 0; + if (!tell_position(target_, end_position)) { + PyErr_Clear(); + return common::E_NOT_SUPPORT; + } + position_ = end_position; return common::E_OK; } + // The writer asks for the position after every flush to record chunk + // offsets; tell() may fail on streams that only support it while idle, so + // track the bytes written instead of asking the target each time. int64_t get_position() override { PythonTargetLock lock(mutex_); - if (target_ == nullptr) { - return 0; - } - int64_t position = 0; - if (!tell_position(target_, position)) { - PyErr_Clear(); - return -1; - } - return position; + return position_; } int close() override { @@ -244,6 +246,7 @@ class PythonRandomAccessWriteFile : public storage::RandomAccessWriteFile { private: PyObject* target_; std::string name_; + int64_t position_; mutable std::mutex mutex_; }; @@ -306,8 +309,7 @@ void* create_tsfile_writer_from_python_file(PyObject* target, if (!sized) { PyErr_Clear(); } - if (!sized || end_position != 0 || - !seek_to(target, 0, SEEK_SET)) { + if (!sized || end_position != 0 || !seek_to(target, 0, SEEK_SET)) { PyErr_Clear(); *error_code = common::E_INVALID_ARG; PyErr_SetString(PyExc_ValueError, diff --git a/python/tsfile/tsfile_writer.pyx b/python/tsfile/tsfile_writer.pyx index d02a6a5d6..c0be207be 100644 --- a/python/tsfile/tsfile_writer.pyx +++ b/python/tsfile/tsfile_writer.pyx @@ -46,8 +46,8 @@ cdef class TsFileWriterPy: """ Initialize from a str/bytes/PathLike path, or a writable binary file object for fsspec-style remote targets. File objects must support - write/tell; seek (when present) and truncate (when present) are used - to start from an empty file. + write/tell; existing content is discarded when the target is seekable + (a fresh file object simply starts empty). """ self.writer = NULL set_allow_non_decreasing_timestamps( From 279b73cb26d200f04d04efdb3a7e26c4a8647463 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 16 Nov 2026 12:23:43 +0800 Subject: [PATCH 10/16] fix(python): rely on the GIL for sink serialization; keep error text - The sink guard mixed PyGILState_Ensure/Release with PyEval_SaveThread/RestoreThread. Every access already happens while the GIL is held, so the GIL alone serializes target_/position_ access; use a plain GIL guard and drop the mutex. - A failing write()/flush() no longer discards the underlying exception: its text is stored per thread and passed as the context of the raised FileSyncError, so e.g. the OSError message survives. Test asserts it. --- python/tests/test_write_sources.py | 4 +- .../tsfile/python_random_access_write_file.cc | 73 ++++++++++++------- .../tsfile/python_random_access_write_file.h | 7 ++ python/tsfile/tsfile_writer.pyx | 5 +- 4 files changed, 61 insertions(+), 28 deletions(-) diff --git a/python/tests/test_write_sources.py b/python/tests/test_write_sources.py index 01d71e8a7..6b2101192 100644 --- a/python/tests/test_write_sources.py +++ b/python/tests/test_write_sources.py @@ -125,9 +125,11 @@ def flush(self): raise OSError("disk full") target = FailingFlush() - with pytest.raises(FileSyncError): + with pytest.raises(FileSyncError) as excinfo: with TsFileWriter(target) as writer: _write_table(writer) + # The original OSError text is kept, not replaced by a generic message. + assert "disk full" in str(excinfo.value) def test_memory_threshold_is_applied(): diff --git a/python/tsfile/python_random_access_write_file.cc b/python/tsfile/python_random_access_write_file.cc index 34397071e..5cb69db16 100644 --- a/python/tsfile/python_random_access_write_file.cc +++ b/python/tsfile/python_random_access_write_file.cc @@ -21,7 +21,6 @@ #include #include -#include #include #include #include @@ -33,25 +32,46 @@ namespace { -class PythonTargetLock { +// Every access to the target happens with the GIL held, so the GIL itself +// serializes them; mixing it with a separate mutex (and dropping the GIL while +// waiting for that mutex) is both unnecessary and easy to get wrong. +class PythonGILGuard { public: - explicit PythonTargetLock(std::mutex& mutex) - : gil_state_(PyGILState_Ensure()), lock_(mutex, std::defer_lock) { - PyThreadState* thread_state = PyEval_SaveThread(); - lock_.lock(); - PyEval_RestoreThread(thread_state); - } - - ~PythonTargetLock() { - lock_.unlock(); - PyGILState_Release(gil_state_); - } + PythonGILGuard() : state_(PyGILState_Ensure()) {} + ~PythonGILGuard() { PyGILState_Release(state_); } private: - PyGILState_STATE gil_state_; - std::unique_lock lock_; + PyGILState_STATE state_; }; +// Preserve the underlying Python error text for the caller: the C API can only +// return an errno, but dropping the exception entirely would hide e.g. the +// OSError message behind a generic FileSyncError. +thread_local std::string g_last_python_error; + +std::string take_python_error_message() { + PyObject* type = nullptr; + PyObject* value = nullptr; + PyObject* traceback = nullptr; + PyErr_Fetch(&type, &value, &traceback); + PyErr_NormalizeException(&type, &value, &traceback); + std::string message; + if (value != nullptr) { + PyObject* text = PyObject_Str(value); + if (text != nullptr) { + const char* utf8 = PyUnicode_AsUTF8(text); + if (utf8 != nullptr) { + message.assign(utf8); + } + Py_DECREF(text); + } + } + Py_XDECREF(type); + Py_XDECREF(value); + Py_XDECREF(traceback); + return message; +} + bool has_callable_attribute(PyObject* target, const char* name) { PyObject* attribute = PyObject_GetAttrString(target, name); if (attribute == nullptr) { @@ -139,7 +159,7 @@ class PythonRandomAccessWriteFile : public storage::RandomAccessWriteFile { ~PythonRandomAccessWriteFile() override { close(); } bool is_opened() const override { - PythonTargetLock lock(mutex_); + PythonGILGuard gil; return target_ != nullptr; } @@ -152,7 +172,7 @@ class PythonRandomAccessWriteFile : public storage::RandomAccessWriteFile { if (buf == nullptr) { return common::E_INVALID_ARG; } - PythonTargetLock lock(mutex_); + PythonGILGuard gil; if (target_ == nullptr) { return common::E_FILE_WRITE_ERR; } @@ -161,7 +181,7 @@ class PythonRandomAccessWriteFile : public storage::RandomAccessWriteFile { const int64_t cur = write_bytes( target_, buf + written, static_cast(len - written)); if (cur <= 0) { - PyErr_Clear(); + g_last_python_error = take_python_error_message(); return common::E_FILE_WRITE_ERR; } written += static_cast(cur); @@ -171,7 +191,7 @@ class PythonRandomAccessWriteFile : public storage::RandomAccessWriteFile { } int sync() override { - PythonTargetLock lock(mutex_); + PythonGILGuard gil; if (target_ == nullptr) { return common::E_FILE_WRITE_ERR; } @@ -182,7 +202,7 @@ class PythonRandomAccessWriteFile : public storage::RandomAccessWriteFile { } PyObject* result = PyObject_CallMethod(target_, "flush", nullptr); if (result == nullptr) { - PyErr_Clear(); + g_last_python_error = take_python_error_message(); return common::E_FILE_SYNC_ERR; } Py_DECREF(result); @@ -193,7 +213,7 @@ class PythonRandomAccessWriteFile : public storage::RandomAccessWriteFile { if (size < 0) { return common::E_INVALID_ARG; } - PythonTargetLock lock(mutex_); + PythonGILGuard gil; if (target_ == nullptr) { return common::E_FILE_WRITE_ERR; } @@ -208,7 +228,7 @@ class PythonRandomAccessWriteFile : public storage::RandomAccessWriteFile { } int seek_to_end() override { - PythonTargetLock lock(mutex_); + PythonGILGuard gil; if (target_ == nullptr) { return common::E_FILE_WRITE_ERR; } @@ -229,12 +249,12 @@ class PythonRandomAccessWriteFile : public storage::RandomAccessWriteFile { // offsets; tell() may fail on streams that only support it while idle, so // track the bytes written instead of asking the target each time. int64_t get_position() override { - PythonTargetLock lock(mutex_); + PythonGILGuard gil; return position_; } int close() override { - PythonTargetLock lock(mutex_); + PythonGILGuard gil; if (target_ == nullptr) { return common::E_OK; } @@ -247,11 +267,14 @@ class PythonRandomAccessWriteFile : public storage::RandomAccessWriteFile { PyObject* target_; std::string name_; int64_t position_; - mutable std::mutex mutex_; }; } // namespace +const char* python_random_access_write_file_last_error() { + return g_last_python_error.empty() ? nullptr : g_last_python_error.c_str(); +} + void* create_tsfile_writer_from_python_file(PyObject* target, uint64_t memory_threshold, int32_t* error_code) { diff --git a/python/tsfile/python_random_access_write_file.h b/python/tsfile/python_random_access_write_file.h index e149bf5c9..70c36c71c 100644 --- a/python/tsfile/python_random_access_write_file.h +++ b/python/tsfile/python_random_access_write_file.h @@ -28,4 +28,11 @@ void* create_tsfile_writer_from_python_file(PyObject* target, uint64_t memory_threshold, int32_t* error_code); +/** + * Returns the text of the last sink failure (for example the OSError raised by + * a target's flush()), or NULL when the last operation set no Python error. + * The pointer stays valid until the next sink operation on this thread. + */ +const char* python_random_access_write_file_last_error(void); + #endif // PYTHON_TSFILE_PYTHON_RANDOM_ACCESS_WRITE_FILE_H diff --git a/python/tsfile/tsfile_writer.pyx b/python/tsfile/tsfile_writer.pyx index c0be207be..84f4c922c 100644 --- a/python/tsfile/tsfile_writer.pyx +++ b/python/tsfile/tsfile_writer.pyx @@ -35,6 +35,7 @@ cdef extern from "python_random_access_write_file.h": TsFileWriter create_tsfile_writer_from_python_file( PyObject* target, uint64_t memory_threshold, int32_t* error_code) except? NULL + const char* python_random_access_write_file_last_error() #cython: language_level=3 @@ -231,7 +232,7 @@ cdef class TsFileWriterPy: cdef TsFileWriter closing = self.writer self.writer = NULL errno = _tsfile_writer_close(closing) - check_error(errno) + check_error(errno, python_random_access_write_file_last_error()) cpdef flush(self): """ @@ -242,7 +243,7 @@ cdef class TsFileWriterPy: if self.writer == NULL: return errno = _tsfile_writer_flush(self.writer) - check_error(errno) + check_error(errno, python_random_access_write_file_last_error()) def __dealloc__(self): try: From 260c8066f4bbc3d1e5a16d77896d1ffb82eaa4e3 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 16 Nov 2026 12:25:52 +0800 Subject: [PATCH 11/16] fix(python): clear the sink error per operation; init before resetting Second review pass: - The per-thread error text is cleared at the start of every sink operation, so a previous failure cannot be attached to an unrelated one; truncate/seek failures now record their message too. - libtsfile_init() runs before the target is rewound/truncated, so a failed initialization no longer destroys the caller's file content. - The C API memory round-trip test uses a per-process unique file name so parallel CTest processes cannot collide. --- cpp/test/cwrapper/cwrapper_test.cc | 11 +++++++++- .../tsfile/python_random_access_write_file.cc | 20 +++++++++++-------- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/cpp/test/cwrapper/cwrapper_test.cc b/cpp/test/cwrapper/cwrapper_test.cc index ee9a25104..36f530603 100644 --- a/cpp/test/cwrapper/cwrapper_test.cc +++ b/cpp/test/cwrapper/cwrapper_test.cc @@ -24,6 +24,7 @@ #endif #include +#include #include #include #include @@ -804,7 +805,15 @@ TEST_F(CWrapperTest, MemoryWriteFileRoundTrip) { ASSERT_NE(nullptr, buffer); ASSERT_GT(length, 0u); - const std::string path = "cwrapper_memory_write_round_trip.tsfile"; +#ifdef _WIN32 + const auto process_id = static_cast(_getpid()); +#else + const auto process_id = static_cast(getpid()); +#endif + static std::atomic round_trip_counter{0}; + const std::string path = + "cwrapper_memory_write_round_trip_" + std::to_string(process_id) + "_" + + std::to_string(round_trip_counter.fetch_add(1)) + ".tsfile"; { FILE* out = fopen(path.c_str(), "wb"); ASSERT_NE(nullptr, out); diff --git a/python/tsfile/python_random_access_write_file.cc b/python/tsfile/python_random_access_write_file.cc index 5cb69db16..eea929e8e 100644 --- a/python/tsfile/python_random_access_write_file.cc +++ b/python/tsfile/python_random_access_write_file.cc @@ -19,7 +19,6 @@ #include "python_random_access_write_file.h" -#include #include #include #include @@ -173,6 +172,7 @@ class PythonRandomAccessWriteFile : public storage::RandomAccessWriteFile { return common::E_INVALID_ARG; } PythonGILGuard gil; + g_last_python_error.clear(); if (target_ == nullptr) { return common::E_FILE_WRITE_ERR; } @@ -192,6 +192,7 @@ class PythonRandomAccessWriteFile : public storage::RandomAccessWriteFile { int sync() override { PythonGILGuard gil; + g_last_python_error.clear(); if (target_ == nullptr) { return common::E_FILE_WRITE_ERR; } @@ -217,10 +218,11 @@ class PythonRandomAccessWriteFile : public storage::RandomAccessWriteFile { if (target_ == nullptr) { return common::E_FILE_WRITE_ERR; } + g_last_python_error.clear(); PyObject* result = PyObject_CallMethod(target_, "truncate", "L", static_cast(size)); if (result == nullptr) { - PyErr_Clear(); + g_last_python_error = take_python_error_message(); return common::E_NOT_SUPPORT; } Py_DECREF(result); @@ -232,13 +234,14 @@ class PythonRandomAccessWriteFile : public storage::RandomAccessWriteFile { if (target_ == nullptr) { return common::E_FILE_WRITE_ERR; } + g_last_python_error.clear(); if (!seek_to(target_, 0, SEEK_END)) { - PyErr_Clear(); + g_last_python_error = take_python_error_message(); return common::E_NOT_SUPPORT; } int64_t end_position = 0; if (!tell_position(target_, end_position)) { - PyErr_Clear(); + g_last_python_error = take_python_error_message(); return common::E_NOT_SUPPORT; } position_ = end_position; @@ -291,6 +294,11 @@ void* create_tsfile_writer_from_python_file(PyObject* target, return nullptr; } + *error_code = storage::libtsfile_init(); + if (*error_code != common::E_OK) { + return nullptr; + } + // The file must start empty: a header written at a nonzero cursor is not // a standalone TsFile, and bytes left over from a longer previous file // would trail the new one. @@ -346,10 +354,6 @@ void* create_tsfile_writer_from_python_file(PyObject* target, } try { - *error_code = storage::libtsfile_init(); - if (*error_code != common::E_OK) { - return nullptr; - } common::g_config_value_.chunk_group_size_threshold_ = memory_threshold; std::unique_ptr write_file( new PythonRandomAccessWriteFile(target, target_name(target))); From 135e82d62f4f67f71530120b1db6ef38609f2587 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 16 Nov 2026 12:28:42 +0800 Subject: [PATCH 12/16] fix(python): harden sink shutdown and error attribution Third review pass: - The bridge destructor skips close() once the interpreter is finalized, where PyGILState_Ensure is undefined. - seek_to_end() restores the previous offset when tell() cannot report the end position, so a fallback append does not start at the wrong place. - The error text is read only after the failing close/flush call, and only for writers created from a file object, so a path writer cannot inherit an unrelated sink message. - write_file_get_buffer() documents E_OVERFLOW and has a test for the non-memory (RET_NOT_SUPPORT) path. --- cpp/src/cwrapper/tsfile_cwrapper.h | 4 +++- cpp/test/cwrapper/cwrapper_test.cc | 23 +++++++++++++++++++ .../tsfile/python_random_access_write_file.cc | 15 +++++++++++- python/tsfile/tsfile_writer.pyx | 13 +++++++++-- 4 files changed, 51 insertions(+), 4 deletions(-) diff --git a/cpp/src/cwrapper/tsfile_cwrapper.h b/cpp/src/cwrapper/tsfile_cwrapper.h index cda753223..8c1b621c7 100644 --- a/cpp/src/cwrapper/tsfile_cwrapper.h +++ b/cpp/src/cwrapper/tsfile_cwrapper.h @@ -537,7 +537,9 @@ WriteFile write_file_new_memory(ERRNO* err_code); * @param buffer [out] malloc'd copy of the bytes; free with * tsfile_free_buffer(). * @param length [out] Number of bytes in @p buffer. - * @return E_OK(0), or check error code in errno_define_c.h. + * @return E_OK(0), RET_NOT_SUPPORT for a non-memory handle, or RET_OVERFLOW + * when the file exceeds UINT32_MAX bytes (the length is 32-bit); + * other errors as listed in errno_define_c.h. */ ERRNO write_file_get_buffer(WriteFile write_file, uint8_t** buffer, uint32_t* length); diff --git a/cpp/test/cwrapper/cwrapper_test.cc b/cpp/test/cwrapper/cwrapper_test.cc index 36f530603..1adb70cb8 100644 --- a/cpp/test/cwrapper/cwrapper_test.cc +++ b/cpp/test/cwrapper/cwrapper_test.cc @@ -763,6 +763,29 @@ TEST(CApiCompatibilityTest, KeepsLegacySchemaAndTagFilterEntryPoints) { nullptr); } +TEST_F(CWrapperTest, WriteFileGetBufferRejectsPathHandle) { +#ifdef _WIN32 + const auto process_id = static_cast(_getpid()); +#else + const auto process_id = static_cast(getpid()); +#endif + static std::atomic path_counter{0}; + const std::string path = + "cwrapper_get_buffer_path_" + std::to_string(process_id) + "_" + + std::to_string(path_counter.fetch_add(1)) + ".tsfile"; + ERRNO code = RET_OK; + WriteFile file = write_file_new(path.c_str(), &code); + ASSERT_EQ(RET_OK, code); + ASSERT_NE(nullptr, file); + uint8_t* buffer = reinterpret_cast(0x1); + uint32_t length = 7; + EXPECT_EQ(RET_NOT_SUPPORT, write_file_get_buffer(file, &buffer, &length)); + EXPECT_EQ(nullptr, buffer); + EXPECT_EQ(0u, length); + free_write_file(&file); + remove(path.c_str()); +} + // The in-memory write sink lets a C caller serialize a TsFile without a // filesystem and upload the bytes themselves (write_file_get_buffer). TEST_F(CWrapperTest, MemoryWriteFileRoundTrip) { diff --git a/python/tsfile/python_random_access_write_file.cc b/python/tsfile/python_random_access_write_file.cc index eea929e8e..7b178a74b 100644 --- a/python/tsfile/python_random_access_write_file.cc +++ b/python/tsfile/python_random_access_write_file.cc @@ -155,7 +155,14 @@ class PythonRandomAccessWriteFile : public storage::RandomAccessWriteFile { Py_INCREF(target_); } - ~PythonRandomAccessWriteFile() override { close(); } + ~PythonRandomAccessWriteFile() override { + // A writer may outlive the interpreter (for example a module-level + // object); touching the Python API then is undefined. Leaking the + // reference during shutdown is the only safe option. + if (Py_IsInitialized()) { + close(); + } + } bool is_opened() const override { PythonGILGuard gil; @@ -235,6 +242,7 @@ class PythonRandomAccessWriteFile : public storage::RandomAccessWriteFile { return common::E_FILE_WRITE_ERR; } g_last_python_error.clear(); + const int64_t previous_position = position_; if (!seek_to(target_, 0, SEEK_END)) { g_last_python_error = take_python_error_message(); return common::E_NOT_SUPPORT; @@ -242,6 +250,11 @@ class PythonRandomAccessWriteFile : public storage::RandomAccessWriteFile { int64_t end_position = 0; if (!tell_position(target_, end_position)) { g_last_python_error = take_python_error_message(); + // The cursor already moved; put it back so a caller that falls + // back to appending does not land at the wrong offset. + if (seek_to(target_, previous_position, SEEK_SET)) { + PyErr_Clear(); + } return common::E_NOT_SUPPORT; } position_ = end_position; diff --git a/python/tsfile/tsfile_writer.pyx b/python/tsfile/tsfile_writer.pyx index 84f4c922c..18882e7e3 100644 --- a/python/tsfile/tsfile_writer.pyx +++ b/python/tsfile/tsfile_writer.pyx @@ -41,6 +41,7 @@ cdef extern from "python_random_access_write_file.h": cdef class TsFileWriterPy: cdef TsFileWriter writer + cdef bint from_file_object def __init__(self, pathname, memory_threshold: int = 128 * 1024 * 1024, allow_non_decreasing_timestamps: bool = False): @@ -51,12 +52,14 @@ cdef class TsFileWriterPy: (a fresh file object simply starts empty). """ self.writer = NULL + self.from_file_object = False set_allow_non_decreasing_timestamps( allow_non_decreasing_timestamps) cdef ErrorCode error_code = 0 if isinstance(pathname, (str, bytes, os.PathLike)): self.writer = tsfile_writer_new_c(pathname, memory_threshold) return + self.from_file_object = True self.writer = create_tsfile_writer_from_python_file( pathname, memory_threshold, &error_code) check_error(error_code) @@ -232,7 +235,10 @@ cdef class TsFileWriterPy: cdef TsFileWriter closing = self.writer self.writer = NULL errno = _tsfile_writer_close(closing) - check_error(errno, python_random_access_write_file_last_error()) + cdef const char* closing_context = NULL + if self.from_file_object: + closing_context = python_random_access_write_file_last_error() + check_error(errno, closing_context) cpdef flush(self): """ @@ -243,7 +249,10 @@ cdef class TsFileWriterPy: if self.writer == NULL: return errno = _tsfile_writer_flush(self.writer) - check_error(errno, python_random_access_write_file_last_error()) + cdef const char* flush_context = NULL + if self.from_file_object: + flush_context = python_random_access_write_file_last_error() + check_error(errno, flush_context) def __dealloc__(self): try: From 81c1fe36e1757fde2c225e37b16c62406805fcd8 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 16 Nov 2026 12:31:04 +0800 Subject: [PATCH 13/16] fix(cpp): free a previously adopted sink on re-init; more bridge tests Fourth review pass: - TsFileWriter::init(unique_ptr) deleted nothing when a writer that already owned a sink was initialized again, leaking the old sink (and its Python target); release it once the new init succeeds. - Tests: tree-model write (register_timeseries + write_row_record) to a file object, and an explicit flush() mid-write followed by close and a successful read-back. --- cpp/src/writer/tsfile_writer.cc | 7 ++++++ python/tests/test_write_sources.py | 38 ++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/cpp/src/writer/tsfile_writer.cc b/cpp/src/writer/tsfile_writer.cc index 5a6152128..dcf6afe17 100644 --- a/cpp/src/writer/tsfile_writer.cc +++ b/cpp/src/writer/tsfile_writer.cc @@ -155,6 +155,10 @@ int TsFileWriter::init(std::unique_ptr write_file) { if (!write_file || !write_file->is_opened()) { return E_INVALID_ARG; } + // A previously adopted sink would be overwritten by init(); release it + // once the new one is in place (a writer can be re-initialized). + RandomAccessWriteFile* previous = + write_file_created_ ? write_file_ : nullptr; // Keep the unique_ptr owning the sink until init() succeeds; if it throws // (e.g. while allocating TsFileIOWriter) the sink is still released. RandomAccessWriteFile* raw = write_file.get(); @@ -163,6 +167,9 @@ int TsFileWriter::init(std::unique_ptr write_file) { return ret; } write_file.release(); + if (previous != nullptr && previous != raw) { + delete previous; + } // init(raw) stored the pointer without taking ownership; transfer it now // so destroy() deletes it exactly once. write_file_created_ = true; diff --git a/python/tests/test_write_sources.py b/python/tests/test_write_sources.py index 6b2101192..d574fc6ff 100644 --- a/python/tests/test_write_sources.py +++ b/python/tests/test_write_sources.py @@ -225,3 +225,41 @@ def test_seekable_target_without_truncate_must_be_empty(): with TsFileWriter(target) as writer: _write_table(writer) assert len(target.getvalue()) > 0 + + +def test_tree_model_write_to_file_object(): + from tsfile import Field, RowRecord, TimeseriesSchema + + buf = io.BytesIO() + with TsFileWriter(buf) as writer: + writer.register_timeseries( + "root.device", TimeseriesSchema("temperature", TSDataType.DOUBLE) + ) + for second in range(5): + fields = [Field("temperature", second * 0.5, TSDataType.DOUBLE)] + writer.write_row_record(RowRecord("root.device", second, fields)) + + with TsFileReader(io.BytesIO(buf.getvalue())) as reader: + with reader.query_timeseries("root.device", ["temperature"], 0, 4) as result: + seen = [] + while result.next(): + seen.append(result.get_value_by_index(2)) + assert seen == [second * 0.5 for second in range(5)] + + +def test_explicit_flush_keeps_writing(): + buf = io.BytesIO() + writer = TsFileWriter(buf) + try: + _write_table(writer) + writer.flush() + assert len(buf.getvalue()) > 0 + finally: + writer.close() + + with TsFileReader(io.BytesIO(buf.getvalue())) as reader: + result = reader.query_table("test_table", ["item_id", "value"]) + try: + assert result.next() + finally: + result.close() From 3127f691c1765742e5a4137455343585f0257b53 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 16 Nov 2026 12:32:27 +0800 Subject: [PATCH 14/16] fix(python): reject text-mode targets with a clear error A text stream would fail on the first bytes with an opaque TypeError; check the target's mode (when it exposes one) and raise ValueError before touching the file. Added a test with a "w"-mode file. --- python/tests/test_write_sources.py | 7 ++++++ .../tsfile/python_random_access_write_file.cc | 23 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/python/tests/test_write_sources.py b/python/tests/test_write_sources.py index d574fc6ff..0d3b59ab9 100644 --- a/python/tests/test_write_sources.py +++ b/python/tests/test_write_sources.py @@ -263,3 +263,10 @@ def test_explicit_flush_keeps_writing(): assert result.next() finally: result.close() + + +def test_text_mode_file_is_rejected(tmp_path): + path = tmp_path / "text_mode.tsfile" + with open(path, "w", encoding="utf-8") as target: + with pytest.raises(ValueError, match="binary"): + TsFileWriter(target) diff --git a/python/tsfile/python_random_access_write_file.cc b/python/tsfile/python_random_access_write_file.cc index 7b178a74b..e66c23d8f 100644 --- a/python/tsfile/python_random_access_write_file.cc +++ b/python/tsfile/python_random_access_write_file.cc @@ -19,6 +19,7 @@ #include "python_random_access_write_file.h" +#include #include #include #include @@ -312,6 +313,28 @@ void* create_tsfile_writer_from_python_file(PyObject* target, return nullptr; } + // A text-mode stream would reject the first bytes with a TypeError; fail + // early with a message that says what is actually wrong. + PyObject* mode = PyObject_GetAttrString(target, "mode"); + if (mode == nullptr) { + PyErr_Clear(); + } else { + if (PyUnicode_Check(mode)) { + const char* text = PyUnicode_AsUTF8(mode); + if (text != nullptr && std::strchr(text, 'b') == nullptr) { + Py_DECREF(mode); + *error_code = common::E_INVALID_ARG; + PyErr_SetString(PyExc_ValueError, + "target must be opened in binary mode"); + return nullptr; + } + if (text == nullptr) { + PyErr_Clear(); + } + } + Py_DECREF(mode); + } + // The file must start empty: a header written at a nonzero cursor is not // a standalone TsFile, and bytes left over from a longer previous file // would trail the new one. From 3c431f10c70873d63944c31362e29435f9149b02 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 16 Nov 2026 13:00:39 +0800 Subject: [PATCH 15/16] test(cpp): include process.h for _getpid on Windows The new C API tests use _getpid() under _WIN32 but the file only included io.h, breaking the MSVC build (the py-msvc jobs build the C++ tests too). --- cpp/test/cwrapper/cwrapper_test.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/test/cwrapper/cwrapper_test.cc b/cpp/test/cwrapper/cwrapper_test.cc index 1adb70cb8..75fa6fffb 100644 --- a/cpp/test/cwrapper/cwrapper_test.cc +++ b/cpp/test/cwrapper/cwrapper_test.cc @@ -19,6 +19,7 @@ #include #ifdef _WIN32 #include +#include #else #include #endif From 49252b5906a9b015a9f9ed410a2ae0871b79a8bb Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 16 Nov 2026 13:47:22 +0800 Subject: [PATCH 16/16] docs(python): TsFileTableWriter accepts file objects too It forwards its target to TsFileWriter, so the new sink support already worked; update the annotation/docstring, the README and add a round-trip test through TsFileTableWriter. --- python/README.md | 3 +++ python/tests/test_write_sources.py | 28 ++++++++++++++++++++++++++++ python/tsfile/tsfile_table_writer.py | 6 ++++-- 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/python/README.md b/python/README.md index f342d59a6..7713f119a 100644 --- a/python/README.md +++ b/python/README.md @@ -233,6 +233,9 @@ with fsspec.open("s3://bucket/output.tsfile", "wb") as target: writer.write_table(tablet) ``` +`TsFileTableWriter` (the schema-bound table writer) accepts a writable file +object in the same way. + The target must provide `write()` and `tell()`. When it is seekable its existing content is discarded; a forward-only stream (for example a pipe) is accepted only while it is positioned at 0. A failing `flush()` is reported as diff --git a/python/tests/test_write_sources.py b/python/tests/test_write_sources.py index 0d3b59ab9..d47a387f4 100644 --- a/python/tests/test_write_sources.py +++ b/python/tests/test_write_sources.py @@ -270,3 +270,31 @@ def test_text_mode_file_is_rejected(tmp_path): with open(path, "w", encoding="utf-8") as target: with pytest.raises(ValueError, match="binary"): TsFileWriter(target) + + +def test_table_writer_accepts_file_object(): + from tsfile import TsFileTableWriter + + buf = io.BytesIO() + schema = TableSchema( + "table_writer_table", + [ + ColumnSchema("item_id", TSDataType.STRING, ColumnCategory.TAG), + ColumnSchema("value", TSDataType.DOUBLE), + ], + ) + with TsFileTableWriter(buf, schema) as writer: + tablet = Tablet(["item_id", "value"], [TSDataType.STRING, TSDataType.DOUBLE], 4) + tablet.set_table_name("table_writer_table") + for row in range(4): + tablet.add_timestamp(row, row) + tablet.add_value_by_name("item_id", row, f"id{row}") + tablet.add_value_by_name("value", row, row * 1.5) + writer.write_table(tablet) + + with TsFileReader(io.BytesIO(buf.getvalue())) as reader: + with reader.query_table("table_writer_table", ["item_id", "value"]) as result: + seen = [] + while result.next(): + seen.append(result.get_value_by_name("value")) + assert seen == [row * 1.5 for row in range(4)] diff --git a/python/tsfile/tsfile_table_writer.py b/python/tsfile/tsfile_table_writer.py index 6e525ff98..560e87604 100644 --- a/python/tsfile/tsfile_table_writer.py +++ b/python/tsfile/tsfile_table_writer.py @@ -99,13 +99,15 @@ class TsFileTableWriter: def __init__( self, - path: str, + path: object, table_schema: TableSchema, memory_threshold=128 * 1024 * 1024, allow_non_decreasing_timestamps=False, ): """ - :param path: The path of tsfile, will create if it doesn't exist. + :param path: The path of tsfile, will create if it doesn't exist; a + writable binary file object (for example one opened through + ``fsspec``) is accepted as well. :param table_schema: describes the schema of the tables they want to write. :param memory_threshold(Byte): memory usage threshold for flushing data. :param allow_non_decreasing_timestamps: when True, a timestamp equal to