diff --git a/.gitignore b/.gitignore index dc16352bd..3c5fe07ea 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ python/tsfile/*dll* python/tsfile/*dylib* python/tsfile/*.h !python/tsfile/python_random_access_read_file.h +!python/tsfile/python_random_access_write_file.h python/tsfile/*.cpp python/tsfile/**/*.cpp python/tsfile/**/*so* diff --git a/cpp/src/cwrapper/tsfile_cwrapper.cc b/cpp/src/cwrapper/tsfile_cwrapper.cc index 588561e3e..f1fcc2ba0 100644 --- a/cpp/src/cwrapper/tsfile_cwrapper.cc +++ b/cpp/src/cwrapper/tsfile_cwrapper.cc @@ -20,6 +20,7 @@ #include "cwrapper/tsfile_cwrapper.h" #include +#include #include #include #include @@ -226,6 +227,56 @@ WriteFile write_file_new(const char* pathname, ERRNO* err_code) { return file; } +WriteFile write_file_new_memory(ERRNO* err_code) { + if (err_code == nullptr) { + return nullptr; + } + // Same lazy initialization as write_file_new(): a writer built on this + // handle must not see the zero-initialized global config of a process + // that has not touched any other API yet. + init_tsfile_config(); + storage::MemoryRandomAccessWriteFile* file = + new (std::nothrow) storage::MemoryRandomAccessWriteFile(); + if (file == nullptr) { + *err_code = common::E_OOM; + return nullptr; + } + *err_code = common::E_OK; + return file; +} + +ERRNO write_file_get_buffer(WriteFile write_file, uint8_t** buffer, + uint32_t* length) { + if (buffer == nullptr || length == nullptr) { + return common::E_INVALID_ARG; + } + *buffer = nullptr; + *length = 0; + if (write_file == nullptr) { + return common::E_INVALID_ARG; + } + auto* sink = static_cast(write_file); + auto* mem = dynamic_cast(sink); + if (mem == nullptr) { + return common::E_NOT_SUPPORT; + } + const std::vector& data = mem->data(); + if (data.empty()) { + return common::E_OK; + } + if (data.size() > 0xFFFFFFFFULL) { + return common::E_OVERFLOW; + } + void* copy = malloc(data.size()); + if (copy == nullptr) { + return common::E_OOM; + } + std::memcpy(copy, data.data(), data.size()); + *buffer = static_cast(copy); + *length = static_cast(data.size()); + return common::E_OK; +} + TsFileWriter tsfile_writer_new(WriteFile file, TableSchema* schema, ERRNO* err_code) { // C API: every public entry must defend against null callers — a null @@ -259,7 +310,8 @@ TsFileWriter tsfile_writer_new(WriteFile file, TableSchema* schema, new storage::TableSchema(schema->table_name, column_schemas)); std::unique_ptr table_writer( new storage::TsFileTableWriter( - static_cast(file), table_schema.get())); + static_cast(file), + table_schema.get())); *err_code = table_writer->get_error(); if (*err_code != common::E_OK) { return nullptr; @@ -306,8 +358,8 @@ TsFileWriter tsfile_writer_new_with_memory_threshold(WriteFile file, new storage::TableSchema(schema->table_name, column_schemas)); std::unique_ptr table_writer( new storage::TsFileTableWriter( - static_cast(file), table_schema.get(), - memory_threshold)); + static_cast(file), + table_schema.get(), memory_threshold)); *err_code = table_writer->get_error(); if (*err_code != common::E_OK) { return nullptr; @@ -2341,7 +2393,7 @@ void free_column_schema(ColumnSchema schema) { free(schema.column_name); } void tsfile_free_buffer(void* buffer) { free(buffer); } void free_write_file(WriteFile* write_file) { - auto f = static_cast(*write_file); + auto f = static_cast(*write_file); delete f; *write_file = nullptr; } @@ -2481,9 +2533,8 @@ ERRNO _tsfile_writer_write_ts_record(TsFileWriter writer, TsRecord data) { ERRNO _tsfile_writer_close(TsFileWriter writer) { auto* w = static_cast(writer); int ret = w->close(); - if (ret != common::E_OK) { - return ret; - } + // Release the writer even when close() failed: the handle must not leak, + // and destroy() is the documented cleanup path for a failed close. delete w; return ret; } diff --git a/cpp/src/cwrapper/tsfile_cwrapper.h b/cpp/src/cwrapper/tsfile_cwrapper.h index d2982ac36..8c1b621c7 100644 --- a/cpp/src/cwrapper/tsfile_cwrapper.h +++ b/cpp/src/cwrapper/tsfile_cwrapper.h @@ -517,6 +517,33 @@ int set_allow_non_decreasing_timestamps(bool enabled); WriteFile write_file_new(const char* pathname, ERRNO* err_code); +/** + * @brief Creates an in-memory WriteFile that accumulates bytes instead of + * touching the filesystem. + * + * @param err_code [out] E_OK(0), or check error code in errno_define_c.h. + * @return WriteFile Valid handle on success. + * @note Use write_file_get_buffer() to obtain the serialized bytes (for + * example, to upload to object storage). Free the buffer with + * tsfile_free_buffer(). Free the handle with free_write_file() after the + * TsFileWriter has been closed. + */ +WriteFile write_file_new_memory(ERRNO* err_code); + +/** + * @brief Copies the bytes written so far to a caller-owned buffer. + * + * @param write_file Valid in-memory WriteFile handle. + * @param buffer [out] malloc'd copy of the bytes; free with + * tsfile_free_buffer(). + * @param length [out] Number of bytes in @p buffer. + * @return E_OK(0), RET_NOT_SUPPORT for a non-memory handle, or RET_OVERFLOW + * when the file exceeds UINT32_MAX bytes (the length is 32-bit); + * other errors as listed in errno_define_c.h. + */ +ERRNO write_file_get_buffer(WriteFile write_file, uint8_t** buffer, + uint32_t* length); + /** * @brief Creates a TsFileWriter for writing a TsFile. * diff --git a/cpp/src/writer/tsfile_writer.cc b/cpp/src/writer/tsfile_writer.cc index d1e06a0ba..dcf6afe17 100644 --- a/cpp/src/writer/tsfile_writer.cc +++ b/cpp/src/writer/tsfile_writer.cc @@ -151,6 +151,31 @@ int TsFileWriter::init(RandomAccessWriteFile* write_file) { // table_schema_map_ in do_check_schema_table when a new device appears. // All new MeasurementSchemaGroup/MeasurementSchema are freed in destroy(). // ----------------------------------------------------------------------------- +int TsFileWriter::init(std::unique_ptr write_file) { + if (!write_file || !write_file->is_opened()) { + return E_INVALID_ARG; + } + // A previously adopted sink would be overwritten by init(); release it + // once the new one is in place (a writer can be re-initialized). + RandomAccessWriteFile* previous = + write_file_created_ ? write_file_ : nullptr; + // Keep the unique_ptr owning the sink until init() succeeds; if it throws + // (e.g. while allocating TsFileIOWriter) the sink is still released. + RandomAccessWriteFile* raw = write_file.get(); + const int ret = init(raw); + if (ret != E_OK) { + return ret; + } + write_file.release(); + if (previous != nullptr && previous != raw) { + delete previous; + } + // init(raw) stored the pointer without taking ownership; transfer it now + // so destroy() deletes it exactly once. + write_file_created_ = true; + return E_OK; +} + int TsFileWriter::init(RestorableTsFileIOWriter* rw) { if (rw == nullptr || !rw->can_write()) { return E_INVALID_ARG; diff --git a/cpp/src/writer/tsfile_writer.h b/cpp/src/writer/tsfile_writer.h index 39645f2d3..190826641 100644 --- a/cpp/src/writer/tsfile_writer.h +++ b/cpp/src/writer/tsfile_writer.h @@ -61,6 +61,9 @@ class TsFileWriter { int open(const std::string& file_path, int flags, mode_t mode); int open(const std::string& file_path); int init(storage::RandomAccessWriteFile* write_file); + /** Like init(RandomAccessWriteFile*) but the writer takes ownership of + * @p write_file and deletes it on destroy(). */ + int init(std::unique_ptr write_file); int init(storage::RestorableTsFileIOWriter* rw); void set_generate_table_schema(bool generate_table_schema); diff --git a/cpp/test/cwrapper/cwrapper_test.cc b/cpp/test/cwrapper/cwrapper_test.cc index dfe13e101..75fa6fffb 100644 --- a/cpp/test/cwrapper/cwrapper_test.cc +++ b/cpp/test/cwrapper/cwrapper_test.cc @@ -19,11 +19,13 @@ #include #ifdef _WIN32 #include +#include #else #include #endif #include +#include #include #include #include @@ -762,4 +764,106 @@ TEST(CApiCompatibilityTest, KeepsLegacySchemaAndTagFilterEntryPoints) { nullptr); } +TEST_F(CWrapperTest, WriteFileGetBufferRejectsPathHandle) { +#ifdef _WIN32 + const auto process_id = static_cast(_getpid()); +#else + const auto process_id = static_cast(getpid()); +#endif + static std::atomic path_counter{0}; + const std::string path = + "cwrapper_get_buffer_path_" + std::to_string(process_id) + "_" + + std::to_string(path_counter.fetch_add(1)) + ".tsfile"; + ERRNO code = RET_OK; + WriteFile file = write_file_new(path.c_str(), &code); + ASSERT_EQ(RET_OK, code); + ASSERT_NE(nullptr, file); + uint8_t* buffer = reinterpret_cast(0x1); + uint32_t length = 7; + EXPECT_EQ(RET_NOT_SUPPORT, write_file_get_buffer(file, &buffer, &length)); + EXPECT_EQ(nullptr, buffer); + EXPECT_EQ(0u, length); + free_write_file(&file); + remove(path.c_str()); +} + +// The in-memory write sink lets a C caller serialize a TsFile without a +// filesystem and upload the bytes themselves (write_file_get_buffer). +TEST_F(CWrapperTest, MemoryWriteFileRoundTrip) { + ERRNO code = RET_OK; + WriteFile file = write_file_new_memory(&code); + ASSERT_EQ(RET_OK, code); + ASSERT_NE(nullptr, file); + + std::vector columns; + columns.push_back( + ColumnSchema{const_cast("device"), TS_DATATYPE_STRING, TAG}); + columns.push_back( + ColumnSchema{const_cast("value"), TS_DATATYPE_INT64, FIELD}); + TableSchema schema = {const_cast("memtable"), columns.data(), + static_cast(columns.size())}; + + TsFileWriter writer = tsfile_writer_new(file, &schema, &code); + ASSERT_EQ(RET_OK, code); + ASSERT_NE(nullptr, writer); + + char* names[] = {const_cast("device"), const_cast("value")}; + TSDataType types[] = {TS_DATATYPE_STRING, TS_DATATYPE_INT64}; + Tablet tablet = tablet_new(names, types, 2, 4, &code); + ASSERT_EQ(RET_OK, code); + ASSERT_NE(nullptr, tablet); + for (uint32_t row = 0; row < 4; row++) { + ASSERT_EQ(RET_OK, tablet_add_timestamp(tablet, row, row)); + ASSERT_EQ(RET_OK, tablet_add_value_by_name_string_with_len( + tablet, row, names[0], "d0", 2)); + ASSERT_EQ(RET_OK, tablet_add_value_by_name_int64_t(tablet, row, + names[1], row * 10)); + } + ASSERT_EQ(RET_OK, tsfile_writer_write(writer, tablet)); + ASSERT_EQ(RET_OK, tsfile_writer_close(writer)); + free_tablet(&tablet); + + uint8_t* buffer = nullptr; + uint32_t length = 0; + ASSERT_EQ(RET_OK, write_file_get_buffer(file, &buffer, &length)); + ASSERT_NE(nullptr, buffer); + ASSERT_GT(length, 0u); + +#ifdef _WIN32 + const auto process_id = static_cast(_getpid()); +#else + const auto process_id = static_cast(getpid()); +#endif + static std::atomic round_trip_counter{0}; + const std::string path = + "cwrapper_memory_write_round_trip_" + std::to_string(process_id) + "_" + + std::to_string(round_trip_counter.fetch_add(1)) + ".tsfile"; + { + FILE* out = fopen(path.c_str(), "wb"); + ASSERT_NE(nullptr, out); + ASSERT_EQ(length, fwrite(buffer, 1, length, out)); + fclose(out); + } + tsfile_free_buffer(buffer); + free_write_file(&file); + + TsFileReader reader = tsfile_reader_new(path.c_str(), &code); + ASSERT_EQ(RET_OK, code); + ASSERT_NE(nullptr, reader); + ResultSet result = + tsfile_query_table(reader, "memtable", names, 2, 0, 100, &code); + ASSERT_EQ(RET_OK, code); + int rows = 0; + while (tsfile_result_set_next(result, &code) && code == RET_OK) { + EXPECT_EQ(static_cast(rows * 10), + tsfile_result_set_get_value_by_name_int64_t(result, "value")); + rows++; + } + EXPECT_EQ(RET_OK, code); + EXPECT_EQ(4, rows); + free_tsfile_result_set(&result); + tsfile_reader_close(reader); + remove(path.c_str()); +} + } // namespace cwrapper diff --git a/python/MANIFEST.in b/python/MANIFEST.in index 1981d9212..e90457964 100644 --- a/python/MANIFEST.in +++ b/python/MANIFEST.in @@ -19,4 +19,5 @@ include _build_support.py recursive-include tsfile *.pyx *.pxd *.pxi include tsfile/python_random_access_read_file.h +include tsfile/python_random_access_write_file.h prune tsfile/include diff --git a/python/README.md b/python/README.md index 7980e91db..7713f119a 100644 --- a/python/README.md +++ b/python/README.md @@ -219,3 +219,25 @@ The object must provide `seek()`, `tell()`, and explicitly sized binary `read(size)` operations. The caller owns the object: `TsFileReader` keeps it alive while in use, preserves its cursor position, and does not close it. The local `FileReadBackend` setting does not apply to file objects. + +`TsFileWriter` accepts a writable binary file object as well, which lets a +TsFile be written straight to an object store: + +```python +import fsspec +from tsfile import TsFileWriter + +with fsspec.open("s3://bucket/output.tsfile", "wb") as target: + with TsFileWriter(target) as writer: + writer.register_table(table_schema) + writer.write_table(tablet) +``` + +`TsFileTableWriter` (the schema-bound table writer) accepts a writable file +object in the same way. + +The target must provide `write()` and `tell()`. When it is seekable its +existing content is discarded; a forward-only stream (for example a pipe) is +accepted only while it is positioned at 0. A failing `flush()` is reported as +`FileSyncError` instead of being ignored. As with the reader, the caller owns +the object and closes it. diff --git a/python/pom.xml b/python/pom.xml index 33d3338cb..7be934308 100644 --- a/python/pom.xml +++ b/python/pom.xml @@ -235,16 +235,20 @@ ${basedir}/tsfile + *.so* *.dylib *.dll - *.h *.cpp + + tsfile_py_cpp.h + tsfile_py_cpp_api.h - - python_random_access_read_file.h - tsfile.egg-info diff --git a/python/setup.py b/python/setup.py index 2672565bf..a983a66f0 100644 --- a/python/setup.py +++ b/python/setup.py @@ -408,7 +408,12 @@ def finalize_options(self): depends=["tsfile/python_random_access_read_file.h"], **common, ), - Extension("tsfile.tsfile_writer", ["tsfile/tsfile_writer.pyx"], **common), + Extension( + "tsfile.tsfile_writer", + ["tsfile/tsfile_writer.pyx", "tsfile/python_random_access_write_file.cc"], + depends=["tsfile/python_random_access_write_file.h"], + **common, + ), ] setup( diff --git a/python/tests/test_write_sources.py b/python/tests/test_write_sources.py new file mode 100644 index 000000000..d47a387f4 --- /dev/null +++ b/python/tests/test_write_sources.py @@ -0,0 +1,300 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import io + +import pytest + +from tsfile import ( + ColumnCategory, + ColumnSchema, + TableSchema, + Tablet, + TSDataType, + TsFileReader, + TsFileWriter, +) + + +def _write_table(writer): + column1 = ColumnSchema("item_id", TSDataType.STRING, ColumnCategory.TAG) + column2 = ColumnSchema("value", TSDataType.DOUBLE) + table = TableSchema("test_table", [column1, column2]) + writer.register_table(table) + + tablet = Tablet(["item_id", "value"], [TSDataType.STRING, TSDataType.DOUBLE], 10) + tablet.set_table_name("test_table") + for i in range(10): + tablet.add_timestamp(i, i) + tablet.add_value_by_name("item_id", i, f"id{i}") + tablet.add_value_by_name("value", i, i * 10.0) + writer.write_table(tablet) + + +def test_write_to_bytesio_and_read_back(): + buf = io.BytesIO() + with TsFileWriter(buf) as writer: + _write_table(writer) + + data = buf.getvalue() + assert len(data) > 0 + + source = io.BytesIO(data) + with TsFileReader(source) as reader: + result = reader.query_table("test_table", ["item_id", "value"]) + try: + seen = [] + while result.next(): + seen.append(result.get_value_by_name("item_id")) + assert seen == [f"id{i}" for i in range(10)] + finally: + result.close() + + +def test_write_target_must_be_writable(): + class NotWritable: + pass + + with pytest.raises(TypeError): + TsFileWriter(NotWritable()) + + +def test_path_and_file_object_produce_same_bytes(tmp_path): + path = tmp_path / "path_write.tsfile" + with TsFileWriter(str(path)) as writer: + _write_table(writer) + path_bytes = path.read_bytes() + + buf = io.BytesIO() + with TsFileWriter(buf) as writer: + _write_table(writer) + + assert buf.getvalue() == path_bytes + + +def test_pathname_keyword_and_bytes_path(tmp_path): + keyword_path = tmp_path / "keyword.tsfile" + with TsFileWriter(pathname=str(keyword_path)) as writer: + _write_table(writer) + keyword_bytes = keyword_path.read_bytes() + assert len(keyword_bytes) > 0 + + bytes_path = str(tmp_path / "bytes.tsfile").encode() + with TsFileWriter(bytes_path) as writer: + _write_table(writer) + assert (tmp_path / "bytes.tsfile").read_bytes() == keyword_bytes + + +def test_file_object_with_existing_content_is_reset(): + buf = io.BytesIO(b"stale trailing bytes" * 100) + buf.seek(1000) + with TsFileWriter(buf) as writer: + _write_table(writer) + + data = buf.getvalue() + with TsFileReader(io.BytesIO(data)) as reader: + result = reader.query_table("test_table", ["item_id", "value"]) + try: + seen = [] + while result.next(): + seen.append(result.get_value_by_name("item_id")) + assert seen == [f"id{i}" for i in range(10)] + finally: + result.close() + + +def test_flush_failure_is_reported(): + from tsfile import FileSyncError + + class FailingFlush(io.BytesIO): + def flush(self): + raise OSError("disk full") + + target = FailingFlush() + with pytest.raises(FileSyncError) as excinfo: + with TsFileWriter(target) as writer: + _write_table(writer) + # The original OSError text is kept, not replaced by a generic message. + assert "disk full" in str(excinfo.value) + + +def test_memory_threshold_is_applied(): + from tsfile import get_tsfile_config + + from tsfile import set_tsfile_config + + original = get_tsfile_config()["chunk_group_size_threshold_"] + try: + with TsFileWriter(io.BytesIO(), memory_threshold=4096) as writer: + _write_table(writer) + assert get_tsfile_config()["chunk_group_size_threshold_"] == 4096 + finally: + set_tsfile_config({"chunk_group_size_threshold_": original}) + + +def test_position_does_not_depend_on_tell_after_start(): + class FlakyTell(io.BytesIO): + def __init__(self, *args): + super().__init__(*args) + self._calls = 0 + + def tell(self): + self._calls += 1 + if self._calls > 1: + raise OSError("tell is not supported while writing") + return super().tell() + + target = FlakyTell() + with TsFileWriter(target) as writer: + _write_table(writer) + with TsFileReader(io.BytesIO(target.getvalue())) as reader: + result = reader.query_table("test_table", ["item_id", "value"]) + try: + seen = [] + while result.next(): + seen.append(result.get_value_by_name("item_id")) + assert seen == [f"id{i}" for i in range(10)] + finally: + result.close() + + +class _ForwardOnlyStream(io.RawIOBase): + """A write-only sink: seek/truncate exist but raise, like a pipe.""" + + def __init__(self): + self._buffer = bytearray() + + def writable(self): + return True + + def write(self, data): + self._buffer += data + return len(data) + + def tell(self): + return len(self._buffer) + + +def test_forward_only_stream_is_accepted(): + target = _ForwardOnlyStream() + with TsFileWriter(target) as writer: + _write_table(writer) + + with TsFileReader(io.BytesIO(bytes(target._buffer))) as reader: + result = reader.query_table("test_table", ["item_id", "value"]) + try: + seen = [] + while result.next(): + seen.append(result.get_value_by_name("item_id")) + assert seen == [f"id{i}" for i in range(10)] + finally: + result.close() + + +def test_forward_only_stream_at_nonzero_position_is_rejected(): + target = _ForwardOnlyStream() + target._buffer += b"prefix" + with pytest.raises(ValueError): + TsFileWriter(target) + + +class _SeekableWithoutTruncate(io.BytesIO): + def truncate(self, size=None): + raise io.UnsupportedOperation("truncate") + + +def test_seekable_target_without_truncate_must_be_empty(): + with pytest.raises(ValueError): + TsFileWriter(_SeekableWithoutTruncate(b"stale")) + + target = _SeekableWithoutTruncate() + with TsFileWriter(target) as writer: + _write_table(writer) + assert len(target.getvalue()) > 0 + + +def test_tree_model_write_to_file_object(): + from tsfile import Field, RowRecord, TimeseriesSchema + + buf = io.BytesIO() + with TsFileWriter(buf) as writer: + writer.register_timeseries( + "root.device", TimeseriesSchema("temperature", TSDataType.DOUBLE) + ) + for second in range(5): + fields = [Field("temperature", second * 0.5, TSDataType.DOUBLE)] + writer.write_row_record(RowRecord("root.device", second, fields)) + + with TsFileReader(io.BytesIO(buf.getvalue())) as reader: + with reader.query_timeseries("root.device", ["temperature"], 0, 4) as result: + seen = [] + while result.next(): + seen.append(result.get_value_by_index(2)) + assert seen == [second * 0.5 for second in range(5)] + + +def test_explicit_flush_keeps_writing(): + buf = io.BytesIO() + writer = TsFileWriter(buf) + try: + _write_table(writer) + writer.flush() + assert len(buf.getvalue()) > 0 + finally: + writer.close() + + with TsFileReader(io.BytesIO(buf.getvalue())) as reader: + result = reader.query_table("test_table", ["item_id", "value"]) + try: + assert result.next() + finally: + result.close() + + +def test_text_mode_file_is_rejected(tmp_path): + path = tmp_path / "text_mode.tsfile" + with open(path, "w", encoding="utf-8") as target: + with pytest.raises(ValueError, match="binary"): + TsFileWriter(target) + + +def test_table_writer_accepts_file_object(): + from tsfile import TsFileTableWriter + + buf = io.BytesIO() + schema = TableSchema( + "table_writer_table", + [ + ColumnSchema("item_id", TSDataType.STRING, ColumnCategory.TAG), + ColumnSchema("value", TSDataType.DOUBLE), + ], + ) + with TsFileTableWriter(buf, schema) as writer: + tablet = Tablet(["item_id", "value"], [TSDataType.STRING, TSDataType.DOUBLE], 4) + tablet.set_table_name("table_writer_table") + for row in range(4): + tablet.add_timestamp(row, row) + tablet.add_value_by_name("item_id", row, f"id{row}") + tablet.add_value_by_name("value", row, row * 1.5) + writer.write_table(tablet) + + with TsFileReader(io.BytesIO(buf.getvalue())) as reader: + with reader.query_table("table_writer_table", ["item_id", "value"]) as result: + seen = [] + while result.next(): + seen.append(result.get_value_by_name("value")) + assert seen == [row * 1.5 for row in range(4)] diff --git a/python/tsfile/python_random_access_write_file.cc b/python/tsfile/python_random_access_write_file.cc new file mode 100644 index 000000000..e66c23d8f --- /dev/null +++ b/python/tsfile/python_random_access_write_file.cc @@ -0,0 +1,409 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "python_random_access_write_file.h" + +#include +#include +#include +#include +#include + +#include "common/global.h" +#include "file/random_access_write_file.h" +#include "utils/errno_define.h" +#include "writer/tsfile_writer.h" + +namespace { + +// Every access to the target happens with the GIL held, so the GIL itself +// serializes them; mixing it with a separate mutex (and dropping the GIL while +// waiting for that mutex) is both unnecessary and easy to get wrong. +class PythonGILGuard { + public: + PythonGILGuard() : state_(PyGILState_Ensure()) {} + ~PythonGILGuard() { PyGILState_Release(state_); } + + private: + PyGILState_STATE state_; +}; + +// Preserve the underlying Python error text for the caller: the C API can only +// return an errno, but dropping the exception entirely would hide e.g. the +// OSError message behind a generic FileSyncError. +thread_local std::string g_last_python_error; + +std::string take_python_error_message() { + PyObject* type = nullptr; + PyObject* value = nullptr; + PyObject* traceback = nullptr; + PyErr_Fetch(&type, &value, &traceback); + PyErr_NormalizeException(&type, &value, &traceback); + std::string message; + if (value != nullptr) { + PyObject* text = PyObject_Str(value); + if (text != nullptr) { + const char* utf8 = PyUnicode_AsUTF8(text); + if (utf8 != nullptr) { + message.assign(utf8); + } + Py_DECREF(text); + } + } + Py_XDECREF(type); + Py_XDECREF(value); + Py_XDECREF(traceback); + return message; +} + +bool has_callable_attribute(PyObject* target, const char* name) { + PyObject* attribute = PyObject_GetAttrString(target, name); + if (attribute == nullptr) { + PyErr_Clear(); + return false; + } + const bool callable = PyCallable_Check(attribute) != 0; + Py_DECREF(attribute); + return callable; +} + +bool seek_to(PyObject* target, int64_t offset, int whence) { + PyObject* result = PyObject_CallMethod( + target, "seek", "Li", static_cast(offset), whence); + if (result == nullptr) { + return false; + } + Py_DECREF(result); + return true; +} + +bool tell_position(PyObject* target, int64_t& position) { + PyObject* result = + PyObject_CallMethod(target, const_cast("tell"), nullptr); + if (result == nullptr) { + return false; + } + const long long value = PyLong_AsLongLong(result); + Py_DECREF(result); + if (value == -1 && PyErr_Occurred()) { + return false; + } + position = static_cast(value); + return true; +} + +int64_t write_bytes(PyObject* target, const char* buf, Py_ssize_t len) { + PyObject* result = + PyObject_CallMethod(target, const_cast("write"), "y#", buf, len); + if (result == nullptr) { + return -1; + } + const long long written = PyLong_AsLongLong(result); + Py_DECREF(result); + if (written < 0 && PyErr_Occurred()) { + return -1; + } + return static_cast(written); +} + +std::string target_name(PyObject* target) { + std::string name(""); + PyObject* value = PyObject_GetAttrString(target, "name"); + if (value == nullptr) { + PyErr_Clear(); + return name; + } + if (PyUnicode_Check(value)) { + const char* text = PyUnicode_AsUTF8(value); + if (text != nullptr) { + name.assign(text); + } else { + PyErr_Clear(); + } + } else if (PyBytes_Check(value)) { + char* text = nullptr; + Py_ssize_t length = 0; + if (PyBytes_AsStringAndSize(value, &text, &length) == 0) { + name.assign(text, static_cast(length)); + } else { + PyErr_Clear(); + } + } + Py_DECREF(value); + return name; +} + +class PythonRandomAccessWriteFile : public storage::RandomAccessWriteFile { + public: + PythonRandomAccessWriteFile(PyObject* target, std::string name) + : target_(target), name_(std::move(name)), position_(0) { + Py_INCREF(target_); + } + + ~PythonRandomAccessWriteFile() override { + // A writer may outlive the interpreter (for example a module-level + // object); touching the Python API then is undefined. Leaking the + // reference during shutdown is the only safe option. + if (Py_IsInitialized()) { + close(); + } + } + + bool is_opened() const override { + PythonGILGuard gil; + return target_ != nullptr; + } + + const std::string& file_path() const override { return name_; } + + int write(const char* buf, uint32_t len) override { + if (len == 0) { + return common::E_OK; + } + if (buf == nullptr) { + return common::E_INVALID_ARG; + } + PythonGILGuard gil; + g_last_python_error.clear(); + if (target_ == nullptr) { + return common::E_FILE_WRITE_ERR; + } + uint32_t written = 0; + while (written < len) { + const int64_t cur = write_bytes( + target_, buf + written, static_cast(len - written)); + if (cur <= 0) { + g_last_python_error = take_python_error_message(); + return common::E_FILE_WRITE_ERR; + } + written += static_cast(cur); + } + position_ += written; + return common::E_OK; + } + + int sync() override { + PythonGILGuard gil; + g_last_python_error.clear(); + if (target_ == nullptr) { + return common::E_FILE_WRITE_ERR; + } + // flush() is optional for a non-durable sink, but if the target has + // one, a failure means the data may not be committed. + if (!has_callable_attribute(target_, "flush")) { + return common::E_OK; + } + PyObject* result = PyObject_CallMethod(target_, "flush", nullptr); + if (result == nullptr) { + g_last_python_error = take_python_error_message(); + return common::E_FILE_SYNC_ERR; + } + Py_DECREF(result); + return common::E_OK; + } + + int truncate(int64_t size) override { + if (size < 0) { + return common::E_INVALID_ARG; + } + PythonGILGuard gil; + if (target_ == nullptr) { + return common::E_FILE_WRITE_ERR; + } + g_last_python_error.clear(); + PyObject* result = PyObject_CallMethod(target_, "truncate", "L", + static_cast(size)); + if (result == nullptr) { + g_last_python_error = take_python_error_message(); + return common::E_NOT_SUPPORT; + } + Py_DECREF(result); + return common::E_OK; + } + + int seek_to_end() override { + PythonGILGuard gil; + if (target_ == nullptr) { + return common::E_FILE_WRITE_ERR; + } + g_last_python_error.clear(); + const int64_t previous_position = position_; + if (!seek_to(target_, 0, SEEK_END)) { + g_last_python_error = take_python_error_message(); + return common::E_NOT_SUPPORT; + } + int64_t end_position = 0; + if (!tell_position(target_, end_position)) { + g_last_python_error = take_python_error_message(); + // The cursor already moved; put it back so a caller that falls + // back to appending does not land at the wrong offset. + if (seek_to(target_, previous_position, SEEK_SET)) { + PyErr_Clear(); + } + return common::E_NOT_SUPPORT; + } + position_ = end_position; + return common::E_OK; + } + + // The writer asks for the position after every flush to record chunk + // offsets; tell() may fail on streams that only support it while idle, so + // track the bytes written instead of asking the target each time. + int64_t get_position() override { + PythonGILGuard gil; + return position_; + } + + int close() override { + PythonGILGuard gil; + if (target_ == nullptr) { + return common::E_OK; + } + Py_DECREF(target_); + target_ = nullptr; + return common::E_OK; + } + + private: + PyObject* target_; + std::string name_; + int64_t position_; +}; + +} // namespace + +const char* python_random_access_write_file_last_error() { + return g_last_python_error.empty() ? nullptr : g_last_python_error.c_str(); +} + +void* create_tsfile_writer_from_python_file(PyObject* target, + uint64_t memory_threshold, + int32_t* error_code) { + if (error_code == nullptr) { + PyErr_SetString(PyExc_ValueError, "error_code must not be null"); + return nullptr; + } + *error_code = common::E_OK; + if (target == nullptr || !has_callable_attribute(target, "write") || + !has_callable_attribute(target, "tell")) { + *error_code = common::E_INVALID_ARG; + PyErr_SetString(PyExc_TypeError, + "target must be a writable binary file object"); + return nullptr; + } + + *error_code = storage::libtsfile_init(); + if (*error_code != common::E_OK) { + return nullptr; + } + + // A text-mode stream would reject the first bytes with a TypeError; fail + // early with a message that says what is actually wrong. + PyObject* mode = PyObject_GetAttrString(target, "mode"); + if (mode == nullptr) { + PyErr_Clear(); + } else { + if (PyUnicode_Check(mode)) { + const char* text = PyUnicode_AsUTF8(mode); + if (text != nullptr && std::strchr(text, 'b') == nullptr) { + Py_DECREF(mode); + *error_code = common::E_INVALID_ARG; + PyErr_SetString(PyExc_ValueError, + "target must be opened in binary mode"); + return nullptr; + } + if (text == nullptr) { + PyErr_Clear(); + } + } + Py_DECREF(mode); + } + + // The file must start empty: a header written at a nonzero cursor is not + // a standalone TsFile, and bytes left over from a longer previous file + // would trail the new one. + int64_t initial_position = 0; + if (!tell_position(target, initial_position)) { + PyErr_Clear(); + *error_code = common::E_INVALID_ARG; + PyErr_SetString(PyExc_TypeError, "target must support tell()"); + return nullptr; + } + // Rewind when the target allows it. A forward-only stream inherits + // callable seek()/truncate() from io.RawIOBase that raise + // io.UnsupportedOperation, so treating a callable method as support would + // reject freshly opened streams; only the position check matters then. + bool rewound = false; + if (has_callable_attribute(target, "seek")) { + if (seek_to(target, 0, SEEK_SET)) { + rewound = true; + } else { + PyErr_Clear(); + } + } + if (!rewound && initial_position != 0) { + *error_code = common::E_INVALID_ARG; + PyErr_SetString(PyExc_ValueError, + "target must be positioned at 0 before writing"); + return nullptr; + } + if (rewound && has_callable_attribute(target, "truncate")) { + PyObject* truncated = PyObject_CallMethod(target, "truncate", "i", 0); + if (truncated == nullptr) { + PyErr_Clear(); + // Truncation is unsupported: tolerate it only if the target is + // provably empty, otherwise a shorter write would leave stale + // trailing bytes behind a valid-looking file. + int64_t end_position = 0; + const bool sized = seek_to(target, 0, SEEK_END) && + tell_position(target, end_position); + if (!sized) { + PyErr_Clear(); + } + if (!sized || end_position != 0 || !seek_to(target, 0, SEEK_SET)) { + PyErr_Clear(); + *error_code = common::E_INVALID_ARG; + PyErr_SetString(PyExc_ValueError, + "target has existing content and cannot be " + "truncated"); + return nullptr; + } + } else { + Py_DECREF(truncated); + } + } + + try { + common::g_config_value_.chunk_group_size_threshold_ = memory_threshold; + std::unique_ptr write_file( + new PythonRandomAccessWriteFile(target, target_name(target))); + std::unique_ptr writer( + new storage::TsFileWriter()); + *error_code = writer->init(std::move(write_file)); + if (*error_code != common::E_OK) { + return nullptr; + } + return writer.release(); + } catch (const std::bad_alloc&) { + *error_code = common::E_OOM; + } catch (...) { + *error_code = common::E_FILE_OPEN_ERR; + } + return nullptr; +} diff --git a/python/tsfile/python_random_access_write_file.h b/python/tsfile/python_random_access_write_file.h new file mode 100644 index 000000000..70c36c71c --- /dev/null +++ b/python/tsfile/python_random_access_write_file.h @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef PYTHON_TSFILE_PYTHON_RANDOM_ACCESS_WRITE_FILE_H +#define PYTHON_TSFILE_PYTHON_RANDOM_ACCESS_WRITE_FILE_H + +#define PY_SSIZE_T_CLEAN +#include +#include + +void* create_tsfile_writer_from_python_file(PyObject* target, + uint64_t memory_threshold, + int32_t* error_code); + +/** + * Returns the text of the last sink failure (for example the OSError raised by + * a target's flush()), or NULL when the last operation set no Python error. + * The pointer stays valid until the next sink operation on this thread. + */ +const char* python_random_access_write_file_last_error(void); + +#endif // PYTHON_TSFILE_PYTHON_RANDOM_ACCESS_WRITE_FILE_H diff --git a/python/tsfile/tsfile_py_cpp.pyx b/python/tsfile/tsfile_py_cpp.pyx index b295a1ba9..92816f4b9 100644 --- a/python/tsfile/tsfile_py_cpp.pyx +++ b/python/tsfile/tsfile_py_cpp.pyx @@ -821,7 +821,7 @@ cdef void free_c_row_record(TsRecord record): cdef TsFileWriter tsfile_writer_new_c(object pathname, uint64_t memory_threshold) except NULL: cdef ErrorCode errno = 0 cdef TsFileWriter writer = NULL - cdef bytes encoded_path = PyUnicode_AsUTF8String(pathname) + cdef bytes encoded_path = os.fsencode(pathname) cdef const char * c_path = encoded_path writer = _tsfile_writer_new(c_path, memory_threshold, &errno) check_error(errno) diff --git a/python/tsfile/tsfile_table_writer.py b/python/tsfile/tsfile_table_writer.py index 6e525ff98..560e87604 100644 --- a/python/tsfile/tsfile_table_writer.py +++ b/python/tsfile/tsfile_table_writer.py @@ -99,13 +99,15 @@ class TsFileTableWriter: def __init__( self, - path: str, + path: object, table_schema: TableSchema, memory_threshold=128 * 1024 * 1024, allow_non_decreasing_timestamps=False, ): """ - :param path: The path of tsfile, will create if it doesn't exist. + :param path: The path of tsfile, will create if it doesn't exist; a + writable binary file object (for example one opened through + ``fsspec``) is accepted as well. :param table_schema: describes the schema of the tables they want to write. :param memory_threshold(Byte): memory usage threshold for flushing data. :param allow_non_decreasing_timestamps: when True, a timestamp equal to diff --git a/python/tsfile/tsfile_writer.pyx b/python/tsfile/tsfile_writer.pyx index 7f0bd57ff..18882e7e3 100644 --- a/python/tsfile/tsfile_writer.pyx +++ b/python/tsfile/tsfile_writer.pyx @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. # +import os + import pandas import pyarrow as pa @@ -25,20 +27,42 @@ from tsfile.tablet import Tablet as TabletPy from libc.string cimport memset from libc.stdint cimport uint32_t, uint8_t, uintptr_t from cpython.bytes cimport PyBytes_AsStringAndSize +from cpython.ref cimport PyObject from .tsfile_cpp cimport * from .tsfile_py_cpp cimport * +cdef extern from "python_random_access_write_file.h": + TsFileWriter create_tsfile_writer_from_python_file( + PyObject* target, uint64_t memory_threshold, + int32_t* error_code) except? NULL + const char* python_random_access_write_file_last_error() + #cython: language_level=3 cdef class TsFileWriterPy: cdef TsFileWriter writer + cdef bint from_file_object - def __init__(self, pathname: str, memory_threshold: int = 128 * 1024 * 1024, + def __init__(self, pathname, memory_threshold: int = 128 * 1024 * 1024, allow_non_decreasing_timestamps: bool = False): + """ + Initialize from a str/bytes/PathLike path, or a writable binary file + object for fsspec-style remote targets. File objects must support + write/tell; existing content is discarded when the target is seekable + (a fresh file object simply starts empty). + """ self.writer = NULL + self.from_file_object = False set_allow_non_decreasing_timestamps( allow_non_decreasing_timestamps) - self.writer = tsfile_writer_new_c(pathname, memory_threshold) + cdef ErrorCode error_code = 0 + if isinstance(pathname, (str, bytes, os.PathLike)): + self.writer = tsfile_writer_new_c(pathname, memory_threshold) + return + self.from_file_object = True + self.writer = create_tsfile_writer_from_python_file( + pathname, memory_threshold, &error_code) + check_error(error_code) def register_timeseries(self, device_name : str, timeseries_schema : TimeseriesSchemaPy): """ @@ -208,9 +232,13 @@ cdef class TsFileWriterPy: cdef ErrorCode errno if self.writer == NULL: return - errno = _tsfile_writer_close(self.writer) - check_error(errno) + cdef TsFileWriter closing = self.writer self.writer = NULL + errno = _tsfile_writer_close(closing) + cdef const char* closing_context = NULL + if self.from_file_object: + closing_context = python_random_access_write_file_last_error() + check_error(errno, closing_context) cpdef flush(self): """ @@ -221,7 +249,10 @@ cdef class TsFileWriterPy: if self.writer == NULL: return errno = _tsfile_writer_flush(self.writer) - check_error(errno) + cdef const char* flush_context = NULL + if self.from_file_object: + flush_context = python_random_access_write_file_last_error() + check_error(errno, flush_context) def __dealloc__(self): try: