Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ python/tsfile/*dll*
python/tsfile/*dylib*
python/tsfile/*.h
!python/tsfile/python_random_access_read_file.h
!python/tsfile/python_random_access_write_file.h
python/tsfile/*.cpp
python/tsfile/**/*.cpp
python/tsfile/**/*so*
Expand Down
65 changes: 58 additions & 7 deletions cpp/src/cwrapper/tsfile_cwrapper.cc
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#include "cwrapper/tsfile_cwrapper.h"

#include <fcntl.h>
#include <file/memory_random_access_write_file.h>
#include <file/write_file.h>
#include <reader/qds_without_timegenerator.h>
#include <writer/tsfile_table_writer.h>
Expand Down Expand Up @@ -226,6 +227,56 @@ WriteFile write_file_new(const char* pathname, ERRNO* err_code) {
return file;
}

WriteFile write_file_new_memory(ERRNO* err_code) {
if (err_code == nullptr) {
return nullptr;
}
// Same lazy initialization as write_file_new(): a writer built on this
// handle must not see the zero-initialized global config of a process
// that has not touched any other API yet.
init_tsfile_config();
storage::MemoryRandomAccessWriteFile* file =
new (std::nothrow) storage::MemoryRandomAccessWriteFile();
if (file == nullptr) {
*err_code = common::E_OOM;
return nullptr;
}
*err_code = common::E_OK;
return file;
}

ERRNO write_file_get_buffer(WriteFile write_file, uint8_t** buffer,
uint32_t* length) {
if (buffer == nullptr || length == nullptr) {
return common::E_INVALID_ARG;
}
*buffer = nullptr;
*length = 0;
if (write_file == nullptr) {
return common::E_INVALID_ARG;
}
auto* sink = static_cast<storage::RandomAccessWriteFile*>(write_file);
auto* mem = dynamic_cast<storage::MemoryRandomAccessWriteFile*>(sink);
if (mem == nullptr) {
return common::E_NOT_SUPPORT;
}
const std::vector<char>& data = mem->data();
if (data.empty()) {
return common::E_OK;
}
if (data.size() > 0xFFFFFFFFULL) {
return common::E_OVERFLOW;
}
void* copy = malloc(data.size());
if (copy == nullptr) {
return common::E_OOM;
}
std::memcpy(copy, data.data(), data.size());
*buffer = static_cast<uint8_t*>(copy);
*length = static_cast<uint32_t>(data.size());
return common::E_OK;
}

TsFileWriter tsfile_writer_new(WriteFile file, TableSchema* schema,
ERRNO* err_code) {
// C API: every public entry must defend against null callers — a null
Expand Down Expand Up @@ -259,7 +310,8 @@ TsFileWriter tsfile_writer_new(WriteFile file, TableSchema* schema,
new storage::TableSchema(schema->table_name, column_schemas));
std::unique_ptr<storage::TsFileTableWriter> table_writer(
new storage::TsFileTableWriter(
static_cast<storage::WriteFile*>(file), table_schema.get()));
static_cast<storage::RandomAccessWriteFile*>(file),
table_schema.get()));
*err_code = table_writer->get_error();
if (*err_code != common::E_OK) {
return nullptr;
Expand Down Expand Up @@ -306,8 +358,8 @@ TsFileWriter tsfile_writer_new_with_memory_threshold(WriteFile file,
new storage::TableSchema(schema->table_name, column_schemas));
std::unique_ptr<storage::TsFileTableWriter> table_writer(
new storage::TsFileTableWriter(
static_cast<storage::WriteFile*>(file), table_schema.get(),
memory_threshold));
static_cast<storage::RandomAccessWriteFile*>(file),
table_schema.get(), memory_threshold));
*err_code = table_writer->get_error();
if (*err_code != common::E_OK) {
return nullptr;
Expand Down Expand Up @@ -2341,7 +2393,7 @@ void free_column_schema(ColumnSchema schema) { free(schema.column_name); }
void tsfile_free_buffer(void* buffer) { free(buffer); }

void free_write_file(WriteFile* write_file) {
auto f = static_cast<storage::WriteFile*>(*write_file);
auto f = static_cast<storage::RandomAccessWriteFile*>(*write_file);
delete f;
*write_file = nullptr;
}
Expand Down Expand Up @@ -2481,9 +2533,8 @@ ERRNO _tsfile_writer_write_ts_record(TsFileWriter writer, TsRecord data) {
ERRNO _tsfile_writer_close(TsFileWriter writer) {
auto* w = static_cast<storage::TsFileWriter*>(writer);
int ret = w->close();
if (ret != common::E_OK) {
return ret;
}
// Release the writer even when close() failed: the handle must not leak,
// and destroy() is the documented cleanup path for a failed close.
delete w;
return ret;
}
Expand Down
27 changes: 27 additions & 0 deletions cpp/src/cwrapper/tsfile_cwrapper.h
Original file line number Diff line number Diff line change
Expand Up @@ -517,6 +517,33 @@ int set_allow_non_decreasing_timestamps(bool enabled);

WriteFile write_file_new(const char* pathname, ERRNO* err_code);

/**
* @brief Creates an in-memory WriteFile that accumulates bytes instead of
* touching the filesystem.
*
* @param err_code [out] E_OK(0), or check error code in errno_define_c.h.
* @return WriteFile Valid handle on success.
* @note Use write_file_get_buffer() to obtain the serialized bytes (for
* example, to upload to object storage). Free the buffer with
* tsfile_free_buffer(). Free the handle with free_write_file() after the
* TsFileWriter has been closed.
*/
WriteFile write_file_new_memory(ERRNO* err_code);

/**
* @brief Copies the bytes written so far to a caller-owned buffer.
*
* @param write_file Valid in-memory WriteFile handle.
* @param buffer [out] malloc'd copy of the bytes; free with
* tsfile_free_buffer().
* @param length [out] Number of bytes in @p buffer.
* @return E_OK(0), RET_NOT_SUPPORT for a non-memory handle, or RET_OVERFLOW
* when the file exceeds UINT32_MAX bytes (the length is 32-bit);
* other errors as listed in errno_define_c.h.
*/
ERRNO write_file_get_buffer(WriteFile write_file, uint8_t** buffer,
uint32_t* length);

/**
* @brief Creates a TsFileWriter for writing a TsFile.
*
Expand Down
25 changes: 25 additions & 0 deletions cpp/src/writer/tsfile_writer.cc
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,31 @@ int TsFileWriter::init(RandomAccessWriteFile* write_file) {
// table_schema_map_ in do_check_schema_table when a new device appears.
// All new MeasurementSchemaGroup/MeasurementSchema are freed in destroy().
// -----------------------------------------------------------------------------
int TsFileWriter::init(std::unique_ptr<RandomAccessWriteFile> write_file) {
if (!write_file || !write_file->is_opened()) {
return E_INVALID_ARG;
}
// A previously adopted sink would be overwritten by init(); release it
// once the new one is in place (a writer can be re-initialized).
RandomAccessWriteFile* previous =
write_file_created_ ? write_file_ : nullptr;
// Keep the unique_ptr owning the sink until init() succeeds; if it throws
// (e.g. while allocating TsFileIOWriter) the sink is still released.
RandomAccessWriteFile* raw = write_file.get();
const int ret = init(raw);
if (ret != E_OK) {
return ret;
}
write_file.release();
if (previous != nullptr && previous != raw) {
delete previous;
}
// init(raw) stored the pointer without taking ownership; transfer it now
// so destroy() deletes it exactly once.
write_file_created_ = true;
return E_OK;
}

int TsFileWriter::init(RestorableTsFileIOWriter* rw) {
if (rw == nullptr || !rw->can_write()) {
return E_INVALID_ARG;
Expand Down
3 changes: 3 additions & 0 deletions cpp/src/writer/tsfile_writer.h
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,9 @@ class TsFileWriter {
int open(const std::string& file_path, int flags, mode_t mode);
int open(const std::string& file_path);
int init(storage::RandomAccessWriteFile* write_file);
/** Like init(RandomAccessWriteFile*) but the writer takes ownership of
* @p write_file and deletes it on destroy(). */
int init(std::unique_ptr<storage::RandomAccessWriteFile> write_file);
int init(storage::RestorableTsFileIOWriter* rw);

void set_generate_table_schema(bool generate_table_schema);
Expand Down
104 changes: 104 additions & 0 deletions cpp/test/cwrapper/cwrapper_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,13 @@
#include <gtest/gtest.h>
#ifdef _WIN32
#include <io.h>
#include <process.h>
#else
#include <unistd.h>
#endif
#include <utils/db_utils.h>

#include <atomic>
#include <cstring>
#include <limits>
#include <string>
Expand Down Expand Up @@ -762,4 +764,106 @@ TEST(CApiCompatibilityTest, KeepsLegacySchemaAndTagFilterEntryPoints) {
nullptr);
}

TEST_F(CWrapperTest, WriteFileGetBufferRejectsPathHandle) {
#ifdef _WIN32
const auto process_id = static_cast<uint64_t>(_getpid());
#else
const auto process_id = static_cast<uint64_t>(getpid());
#endif
static std::atomic<uint64_t> path_counter{0};
const std::string path =
"cwrapper_get_buffer_path_" + std::to_string(process_id) + "_" +
std::to_string(path_counter.fetch_add(1)) + ".tsfile";
ERRNO code = RET_OK;
WriteFile file = write_file_new(path.c_str(), &code);
ASSERT_EQ(RET_OK, code);
ASSERT_NE(nullptr, file);
uint8_t* buffer = reinterpret_cast<uint8_t*>(0x1);
uint32_t length = 7;
EXPECT_EQ(RET_NOT_SUPPORT, write_file_get_buffer(file, &buffer, &length));
EXPECT_EQ(nullptr, buffer);
EXPECT_EQ(0u, length);
free_write_file(&file);
remove(path.c_str());
}

// The in-memory write sink lets a C caller serialize a TsFile without a
// filesystem and upload the bytes themselves (write_file_get_buffer).
TEST_F(CWrapperTest, MemoryWriteFileRoundTrip) {
ERRNO code = RET_OK;
WriteFile file = write_file_new_memory(&code);
ASSERT_EQ(RET_OK, code);
ASSERT_NE(nullptr, file);

std::vector<ColumnSchema> columns;
columns.push_back(
ColumnSchema{const_cast<char*>("device"), TS_DATATYPE_STRING, TAG});
columns.push_back(
ColumnSchema{const_cast<char*>("value"), TS_DATATYPE_INT64, FIELD});
TableSchema schema = {const_cast<char*>("memtable"), columns.data(),
static_cast<int>(columns.size())};

TsFileWriter writer = tsfile_writer_new(file, &schema, &code);
ASSERT_EQ(RET_OK, code);
ASSERT_NE(nullptr, writer);

char* names[] = {const_cast<char*>("device"), const_cast<char*>("value")};
TSDataType types[] = {TS_DATATYPE_STRING, TS_DATATYPE_INT64};
Tablet tablet = tablet_new(names, types, 2, 4, &code);
ASSERT_EQ(RET_OK, code);
ASSERT_NE(nullptr, tablet);
for (uint32_t row = 0; row < 4; row++) {
ASSERT_EQ(RET_OK, tablet_add_timestamp(tablet, row, row));
ASSERT_EQ(RET_OK, tablet_add_value_by_name_string_with_len(
tablet, row, names[0], "d0", 2));
ASSERT_EQ(RET_OK, tablet_add_value_by_name_int64_t(tablet, row,
names[1], row * 10));
}
ASSERT_EQ(RET_OK, tsfile_writer_write(writer, tablet));
ASSERT_EQ(RET_OK, tsfile_writer_close(writer));
free_tablet(&tablet);

uint8_t* buffer = nullptr;
uint32_t length = 0;
ASSERT_EQ(RET_OK, write_file_get_buffer(file, &buffer, &length));
ASSERT_NE(nullptr, buffer);
ASSERT_GT(length, 0u);

#ifdef _WIN32
const auto process_id = static_cast<uint64_t>(_getpid());
#else
const auto process_id = static_cast<uint64_t>(getpid());
#endif
static std::atomic<uint64_t> round_trip_counter{0};
const std::string path =
"cwrapper_memory_write_round_trip_" + std::to_string(process_id) + "_" +
std::to_string(round_trip_counter.fetch_add(1)) + ".tsfile";
{
FILE* out = fopen(path.c_str(), "wb");
ASSERT_NE(nullptr, out);
ASSERT_EQ(length, fwrite(buffer, 1, length, out));
fclose(out);
}
tsfile_free_buffer(buffer);
free_write_file(&file);

TsFileReader reader = tsfile_reader_new(path.c_str(), &code);
ASSERT_EQ(RET_OK, code);
ASSERT_NE(nullptr, reader);
ResultSet result =
tsfile_query_table(reader, "memtable", names, 2, 0, 100, &code);
ASSERT_EQ(RET_OK, code);
int rows = 0;
while (tsfile_result_set_next(result, &code) && code == RET_OK) {
EXPECT_EQ(static_cast<int64_t>(rows * 10),
tsfile_result_set_get_value_by_name_int64_t(result, "value"));
rows++;
}
EXPECT_EQ(RET_OK, code);
EXPECT_EQ(4, rows);
free_tsfile_result_set(&result);
tsfile_reader_close(reader);
remove(path.c_str());
}

} // namespace cwrapper
1 change: 1 addition & 0 deletions python/MANIFEST.in
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,5 @@
include _build_support.py
recursive-include tsfile *.pyx *.pxd *.pxi
include tsfile/python_random_access_read_file.h
include tsfile/python_random_access_write_file.h
prune tsfile/include
22 changes: 22 additions & 0 deletions python/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -219,3 +219,25 @@ The object must provide `seek()`, `tell()`, and explicitly sized binary
`read(size)` operations. The caller owns the object: `TsFileReader` keeps it
alive while in use, preserves its cursor position, and does not close it.
The local `FileReadBackend` setting does not apply to file objects.

`TsFileWriter` accepts a writable binary file object as well, which lets a
TsFile be written straight to an object store:

```python
import fsspec
from tsfile import TsFileWriter

with fsspec.open("s3://bucket/output.tsfile", "wb") as target:
with TsFileWriter(target) as writer:
writer.register_table(table_schema)
writer.write_table(tablet)
```

`TsFileTableWriter` (the schema-bound table writer) accepts a writable file
object in the same way.

The target must provide `write()` and `tell()`. When it is seekable its
existing content is discarded; a forward-only stream (for example a pipe) is
accepted only while it is positioned at 0. A failing `flush()` is reported as
`FileSyncError` instead of being ignored. As with the reader, the caller owns
the object and closes it.
12 changes: 8 additions & 4 deletions python/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -235,16 +235,20 @@
<filesets>
<fileset>
<directory>${basedir}/tsfile</directory>
<!-- Generated artifacts only. Hand-written
bridge sources (.h) live here too and must
survive clean. -->
<includes>
<include>*.so*</include>
<include>*.dylib</include>
<include>*.dll</include>
<include>*.h</include>
<include>*.cpp</include>
<!-- Cython-generated public API headers;
the hand-written bridge headers are left
alone. -->
<include>tsfile_py_cpp.h</include>
<include>tsfile_py_cpp_api.h</include>
</includes>
<excludes>
<exclude>python_random_access_read_file.h</exclude>
</excludes>
</fileset>
<fileset>
<directory>tsfile.egg-info</directory>
Expand Down
7 changes: 6 additions & 1 deletion python/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -408,7 +408,12 @@ def finalize_options(self):
depends=["tsfile/python_random_access_read_file.h"],
**common,
),
Extension("tsfile.tsfile_writer", ["tsfile/tsfile_writer.pyx"], **common),
Extension(
"tsfile.tsfile_writer",
["tsfile/tsfile_writer.pyx", "tsfile/python_random_access_write_file.cc"],
depends=["tsfile/python_random_access_write_file.h"],
**common,
),
]

setup(
Expand Down
Loading
Loading