diff --git a/CLAUDE.md b/CLAUDE.md index 9a157d97c..5908f7e27 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -57,8 +57,9 @@ The root `pom.xml` orchestrates all three via Maven profiles: - The on-disk TsFile format must also remain consistent with Java for the same file-format version, including encoded/compressed payload structure, page/chunk headers, and metadata. Do not introduce language-specific file-format changes. - For changes affecting codecs or the on-disk format, add or update regression tests covering codec behavior, boundary values, and block/page boundaries. Verify cross-language compatibility in both directions: Java must read files produced by the changed implementation, and the changed implementation must read Java-generated files. -## Skill Documentation +## CLI and Skill Synchronization +- When SDK functionality is added or changed, review the corresponding CLI integration (`cpp/tools/` and `java/tools/`, as applicable) and synchronize commands, options, defaults, supported types/codecs, and errors in the same change. Reuse SDK capability information where available and add regression tests. If no CLI change is applicable, explain why in the change description. - When a code change alters behavior, update the affected skill documentation in the same change. This includes API and CLI behavior, defaults, errors, build workflows, and encoding/compression or file-format behavior. - Review `skills/tsfile/` for SDK and format changes, and `skills/tsfile-cli/` plus `cpp/tools/skills/tsfile-cli/` for CLI changes. Update the relevant `SKILL.md`, references, examples, and helper scripts as needed, keeping duplicated CLI documentation consistent. - A behavior change is incomplete until the skill instructions and examples describe the resulting behavior accurately. diff --git a/cpp/src/reader/table_result_set.cc b/cpp/src/reader/table_result_set.cc index ce54c98f8..63c598c61 100644 --- a/cpp/src/reader/table_result_set.cc +++ b/cpp/src/reader/table_result_set.cc @@ -111,6 +111,8 @@ void TableResultSet::materialize_current_row() { uint32_t len = 0; bool null = false; row_record_->reset(); + // RowRecord owns only the current row; release its previous string copies. + pa_.reset(); for (uint32_t i = 0; i < row_iterator_->get_column_count(); ++i) { const auto value = row_iterator_->read(i, &len, &null); if (!null) { diff --git a/cpp/test/reader/block/reusable_tsblock_test.cc b/cpp/test/reader/block/reusable_tsblock_test.cc index 8abfe09fc..c46f48d67 100644 --- a/cpp/test/reader/block/reusable_tsblock_test.cc +++ b/cpp/test/reader/block/reusable_tsblock_test.cc @@ -18,6 +18,7 @@ */ #include +#include #include #include #include @@ -31,6 +32,7 @@ #include "file/tsfile_io_reader.h" #include "file/write_file.h" #include "reader/block/single_device_tsblock_reader.h" +#include "reader/table_result_set.h" #include "reader/task/device_query_task.h" #include "writer/tsfile_table_writer.h" @@ -48,6 +50,78 @@ TupleDesc MakeDesc(const char* second_name, TSDataType second_type) { return desc; } +class SingleBlockReader : public TsBlockReader { + public: + explicit SingleBlockReader(TsBlock* block) : block_(block) {} + int has_next(bool& has_next) override { + has_next = !delivered_; + return E_OK; + } + int next(TsBlock*& block) override { + block = block_; + delivered_ = true; + return E_OK; + } + void close() override {} + + private: + TsBlock* block_; + bool delivered_ = false; +}; + +TEST(TableResultSetMemoryTest, MaterializedStringsReuseRowStorage) { + libtsfile_init(); + TupleDesc desc = MakeDesc("s", STRING); + desc.push_back(ColumnSchema("t", TEXT, UNCOMPRESSED, PLAIN)); + desc.push_back(ColumnSchema("b", BLOB, UNCOMPRESSED, PLAIN)); + const int row_count = 2048; + ReusableTsBlock reusable; + TsBlock* block = reusable.acquire(desc, row_count); + ASSERT_NE(block, nullptr); + RowAppender rows(block); + for (int64_t row = 0; row < row_count; ++row) { + ASSERT_TRUE(rows.add_row()); + rows.append(0, reinterpret_cast(&row), sizeof(row)); + const std::string value(row % 4 == 0 ? 2048 : 128, + static_cast('a' + row % 26)); + for (uint32_t col = 1; col <= 3; ++col) { + if (row % 7 == 1) + rows.append_null(col); + else + rows.append(col, value.data(), value.size()); + } + } + TableResultSet result( + std::unique_ptr(new SingleBlockReader(block)), + {"s", "t", "b"}, {STRING, TEXT, BLOB}); + auto& stats = ModStat::get_instance(); + const int64_t baseline = stats.get_stat(MOD_TSFILE_READER); + int64_t peak = baseline; + for (int row = 0; row < row_count; ++row) { + bool has_next = false; + ASSERT_EQ(result.next(has_next), E_OK); + ASSERT_TRUE(has_next); + EXPECT_EQ(result.get_value(1), row); + for (uint32_t col = 2; col <= 4; ++col) { + ASSERT_EQ(result.is_null(col), row % 7 == 1); + if (!result.is_null(col)) { + EXPECT_EQ(result.get_value(col)->to_std_string(), + std::string(row % 4 == 0 ? 2048 : 128, + static_cast('a' + row % 26))); + } + } + peak = std::max(peak, stats.get_stat(MOD_TSFILE_READER)); + } +#ifdef ENABLE_MEM_STAT + // Only the current row's three strings should occupy the materialization + // arena, including oversized allocations; preceding rows must be released. + EXPECT_LE(peak - baseline, 8 * 1024); +#endif + result.close(); + result.close(); + EXPECT_EQ(stats.get_stat(MOD_TSFILE_READER), baseline); +} + TEST(ReusableTsBlockTest, ReusesTheBlockForTheSameLayout) { ReusableTsBlock reusable; TupleDesc desc = MakeDesc("v", INT64); diff --git a/cpp/test/tools/codec_round_trip_test.cc b/cpp/test/tools/codec_round_trip_test.cc new file mode 100644 index 000000000..cb0d4e3ea --- /dev/null +++ b/cpp/test/tools/codec_round_trip_test.cc @@ -0,0 +1,425 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "cli/exit_codes.h" +#include "cli/run_cli.h" +#include "cli_test_util.h" +#include "common/config/config.h" +#include "encoding/decoder_factory.h" +#include "encoding/encoder_factory.h" +#include "format/result_set_format.h" + +namespace { + +const common::TSDataType kTypes[] = { + common::BOOLEAN, common::INT32, common::INT64, common::FLOAT, + common::DOUBLE, common::DATE, common::TIMESTAMP, common::STRING, + common::TEXT, common::BLOB}; +const char* const kEncodings[] = { + "PLAIN", "DICTIONARY", "RLE", "DIFF", "TS_2DIFF", "BITMAP", + "GORILLA_V1", "REGULAR", "GORILLA", "ZIGZAG", "FREQ", "CHIMP", + "SPRINTZ", "RLBE", "CAMEL", "ALP"}; + +struct TempPath { + explicit TempPath(const std::string& ext) + : path(tsfile_cli_test::unique_temp_path("cli_codec", ext)) {} + ~TempPath() { std::remove(path.c_str()); } + std::string path; +}; + +std::string value_for(common::TSDataType type, int row) { + switch (type) { + case common::BOOLEAN: + return row % 2 ? "true" : "false"; + case common::DATE: + return "2024-02-29"; + case common::FLOAT: + case common::DOUBLE: + return std::to_string(row % 17 + 1) + ".5"; + case common::STRING: + case common::TEXT: + return "value_" + std::to_string(row % 17); + case common::BLOB: + return "abc"; + default: + return std::to_string(row % 17 - 8); + } +} + +using CodecCase = std::pair; + +std::vector codec_cases() { + std::vector cases; + for (auto type : kTypes) { + for (int encoding = common::PLAIN; encoding <= common::ALP; + ++encoding) { + cases.emplace_back(type, encoding); + } + } + return cases; +} + +class CliCodec : public ::testing::TestWithParam { + protected: + void SetUp() override { + storage::libtsfile_init(); + saved_page_limit_ = common::g_config_value_.page_writer_max_point_num_; + common::config_set_page_max_point_count(2048); + } + void TearDown() override { + common::config_set_page_max_point_count(saved_page_limit_); + } + + private: + uint32_t saved_page_limit_; +}; + +TEST_P(CliCodec, WriteMatchesSdkFactoriesAndRoundTrips) { + const auto type = GetParam().first; + const auto encoding = static_cast(GetParam().second); + const std::string type_name = tsfile_cli::tsdatatype_name(type); + const std::string encoding_name = kEncodings[GetParam().second]; + auto* encoder = + storage::EncoderFactory::alloc_value_encoder(encoding, type); + auto* decoder = + storage::DecoderFactory::alloc_value_decoder(encoding, type); + const bool supported = encoder != nullptr && decoder != nullptr; + storage::EncoderFactory::free(encoder); + storage::DecoderFactory::free(decoder); + + // Cross ALP blocks and the configured page boundary, retaining null rows. + const int row_count = 2051; + TempPath csv(".csv"); + TempPath file(".tsfile"); + std::ostringstream expected; + std::ostringstream expected_csv; + expected << "time,value\n"; + expected_csv << "time,value\n"; + for (int row = 0; row < row_count; ++row) { + const std::string value = + row % 7 == 1 + ? "\\N" + : value_for(type, encoding == common::ALP ? row - 1024 : row); + expected << row << ',' << value << '\n'; + expected_csv << row << ',' + << (type == common::BLOB && row % 7 != 1 ? "0x616263" + : value) + << '\n'; + } + { + std::ofstream input(csv.path.c_str()); + input << expected.str(); + ASSERT_TRUE(input.good()); + } + std::ostringstream out; + std::ostringstream err; + const int code = tsfile_cli::run_cli( + {"write", "--table", "t", "--field", "value", type_name, "--encoding", + type_name, encoding_name, "--compression", type_name, "UNCOMPRESSED", + "-i", csv.path, "-o", file.path, "-v"}, + out, err); + ASSERT_EQ(code, supported ? 0 : 1) << err.str(); + if (!supported) { + EXPECT_FALSE(std::ifstream(file.path.c_str()).good()); + return; + } + EXPECT_NE(err.str().find("encoding=" + encoding_name), std::string::npos) + << err.str(); + + storage::TsFileReader reader; + ASSERT_EQ(reader.open(file.path), common::E_OK); + storage::ResultSet* result = nullptr; + ASSERT_EQ(reader.query("t", {"value"}, INT64_MIN, INT64_MAX, result), + common::E_OK); + ASSERT_NE(result, nullptr); + std::ostringstream sdk_rows; + sdk_rows << "time,value\n"; + int row = 0; + bool has_next = false; + int ret = common::E_OK; + while ((ret = result->next(has_next)) == common::E_OK && has_next) { + EXPECT_EQ(result->get_value(1), row); + EXPECT_EQ(result->is_null(2), row % 7 == 1); + sdk_rows << row << ',' + << (result->is_null(2) + ? "\\N" + : tsfile_cli::cell_to_string(result, 2, type)) + << '\n'; + ++row; + } + reader.destroy_query_data_set(result); + EXPECT_EQ(ret, common::E_OK); + EXPECT_EQ(row, row_count); + EXPECT_EQ(sdk_rows.str(), expected.str()); + + std::ostringstream schema_out; + std::ostringstream schema_err; + EXPECT_EQ(tsfile_cli::run_cli({"schema", "-f", "csv", file.path}, + schema_out, schema_err), + 0) + << schema_err.str(); + EXPECT_NE(schema_out.str().find(",FIELD," + type_name + "," + + encoding_name + ",UNCOMPRESSED\n"), + std::string::npos) + << schema_out.str(); + std::ostringstream cat_out; + std::ostringstream cat_err; + EXPECT_EQ( + tsfile_cli::run_cli({"cat", "-f", "csv", file.path}, cat_out, cat_err), + 0) + << cat_err.str(); + EXPECT_EQ(cat_out.str(), expected_csv.str()); +} + +INSTANTIATE_TEST_SUITE_P(SdkMatrix, CliCodec, + ::testing::ValuesIn(codec_cases()), + [](const ::testing::TestParamInfo& info) { + return std::string(tsfile_cli::tsdatatype_name( + info.param.first)) + + "_" + kEncodings[info.param.second]; + }); + +class CliGorillaV1Input : public ::testing::TestWithParam { + protected: + void SetUp() override { + storage::libtsfile_init(); + saved_float_encoding_ = common::g_config_value_.float_encoding_type_; + saved_double_encoding_ = common::g_config_value_.double_encoding_type_; + } + void TearDown() override { + common::g_config_value_.float_encoding_type_ = saved_float_encoding_; + common::g_config_value_.double_encoding_type_ = saved_double_encoding_; + } + + void expect_nan_rejected(const std::string& token, int nan_row, + bool override_encoding) { + SCOPED_TRACE(token + " at row " + std::to_string(nan_row)); + TempPath csv(".csv"); + TempPath file(".tsfile"); + { + std::ofstream input(csv.path.c_str()); + input << "time,value\n"; + const int row_count = nan_row == 3 ? 4 : nan_row + 3; + for (int row = 0; row < row_count; ++row) { + input << row << ',' << (row == nan_row ? token : "1.5") << '\n'; + } + ASSERT_TRUE(input.good()); + } + const std::string type_name = tsfile_cli::tsdatatype_name(GetParam()); + std::vector args = {"write", + "--table", + "t", + "--field", + "value", + type_name, + "--compression", + type_name, + "UNCOMPRESSED", + "-i", + csv.path, + "-o", + file.path}; + if (override_encoding) { + args.insert(args.end(), {"--encoding", type_name, "GORILLA_V1"}); + } + std::ostringstream out; + std::ostringstream err; + EXPECT_EQ(tsfile_cli::run_cli(args, out, err), tsfile_cli::kExitFile) + << err.str(); + EXPECT_TRUE(out.str().empty()); + EXPECT_NE(err.str().find("NaN"), std::string::npos) << err.str(); + EXPECT_NE(err.str().find("GORILLA_V1"), std::string::npos) << err.str(); + EXPECT_NE(err.str().find("value"), std::string::npos) << err.str(); + EXPECT_NE(err.str().find("line " + std::to_string(nan_row + 2)), + std::string::npos) + << err.str(); + EXPECT_FALSE(std::ifstream(file.path.c_str()).good()); + } + + private: + common::TSEncoding saved_float_encoding_; + common::TSEncoding saved_double_encoding_; +}; + +TEST_P(CliGorillaV1Input, RejectsNaNIncludingAfterWrittenBatch) { + for (const char* token : + {"nan", "NAN", "+nan", "-NaN", "nan(123)", "\"NaN\""}) { + for (int row : {0, 1, 3, 1025}) { + expect_nan_rejected(token, row, true); + } + } +} + +TEST_P(CliGorillaV1Input, RejectsNaNWithSdkDefaultEncoding) { + common::g_config_value_.float_encoding_type_ = common::GORILLA_V1; + common::g_config_value_.double_encoding_type_ = common::GORILLA_V1; + expect_nan_rejected("nan", 1, false); +} + +INSTANTIATE_TEST_SUITE_P(FloatingTypes, CliGorillaV1Input, + ::testing::Values(common::FLOAT, common::DOUBLE)); + +class CliFloatingSpecial : public ::testing::TestWithParam {}; + +TEST_P(CliFloatingSpecial, PreservesSupportedSpecialValues) { + storage::libtsfile_init(); + const auto type = GetParam().first; + const auto encoding = static_cast(GetParam().second); + const std::string type_name = tsfile_cli::tsdatatype_name(type); + const bool scaled = encoding == common::RLE || encoding == common::TS_2DIFF; + const std::vector values = { + "1.5", encoding == common::GORILLA_V1 ? "2.5" : "nan", + "3.5", "\\N", + "inf", "-inf", + "0", "-0", + "4.5"}; + TempPath csv(".csv"); + TempPath file(".tsfile"); + std::ostringstream expected_csv; + expected_csv << "time,value\n"; + { + std::ofstream input(csv.path.c_str()); + input << "time,value\n"; + for (size_t row = 0; row < values.size(); ++row) { + input << row << ',' << values[row] << '\n'; + expected_csv << row << ',' + << (scaled && values[row] == "-0" ? "0" : values[row]) + << '\n'; + } + ASSERT_TRUE(input.good()); + } + std::ostringstream out; + std::ostringstream err; + ASSERT_EQ( + tsfile_cli::run_cli( + {"write", "--table", "t", "--field", "value", type_name, + "--encoding", type_name, kEncodings[encoding], "--compression", + type_name, "UNCOMPRESSED", "-i", csv.path, "-o", file.path}, + out, err), + 0) + << err.str(); + storage::TsFileReader reader; + ASSERT_EQ(reader.open(file.path), common::E_OK); + storage::ResultSet* result = nullptr; + ASSERT_EQ(reader.query("t", {"value"}, INT64_MIN, INT64_MAX, result), + common::E_OK); + ASSERT_NE(result, nullptr); + bool has_next = false; + int ret = common::E_OK; + size_t row = 0; + while ((ret = result->next(has_next)) == common::E_OK && has_next) { + ASSERT_LT(row, values.size()); + EXPECT_EQ(result->get_value(1), static_cast(row)); + EXPECT_EQ(result->is_null(2), values[row] == "\\N"); + if (!result->is_null(2)) { + const double actual = type == common::FLOAT + ? result->get_value(2) + : result->get_value(2); + if (values[row] == "nan") { + EXPECT_TRUE(std::isnan(actual)); + } else { + const double expected = std::stod(values[row]); + EXPECT_EQ(actual, expected); + if (!scaled) + EXPECT_EQ(std::signbit(actual), std::signbit(expected)); + } + } + ++row; + } + reader.destroy_query_data_set(result); + EXPECT_EQ(ret, common::E_OK); + EXPECT_EQ(row, values.size()); + std::ostringstream cat_out; + std::ostringstream cat_err; + EXPECT_EQ( + tsfile_cli::run_cli({"cat", "-f", "csv", file.path}, cat_out, cat_err), + 0) + << cat_err.str(); + EXPECT_EQ(cat_out.str(), expected_csv.str()); +} + +INSTANTIATE_TEST_SUITE_P( + FloatingCodecs, CliFloatingSpecial, ::testing::ValuesIn([] { + std::vector cases; + for (auto type : {common::FLOAT, common::DOUBLE}) { + for (auto encoding : {common::PLAIN, common::RLE, common::TS_2DIFF, + common::GORILLA_V1, common::GORILLA, + common::CHIMP, common::RLBE, common::ALP}) { + cases.emplace_back(type, encoding); + } + } + return cases; + }())); + +TEST(CliCodecSpecialValues, PreservesIntegerEndMarkers) { + storage::libtsfile_init(); + for (auto type : {common::INT32, common::INT64, common::TIMESTAMP}) { + for (const char* encoding : {"GORILLA", "CHIMP"}) { + const std::string type_name = tsfile_cli::tsdatatype_name(type); + SCOPED_TRACE(type_name + " " + encoding); + const std::string minimum = + std::to_string(type == common::INT32 ? INT32_MIN : INT64_MIN); + const std::string maximum = + std::to_string(type == common::INT32 ? INT32_MAX : INT64_MAX); + const std::string rows = "time,value\n0,1\n1," + minimum + + "\n2,2\n3," + maximum + "\n4,3\n"; + TempPath csv(".csv"); + TempPath file(".tsfile"); + { + std::ofstream input(csv.path.c_str()); + input << rows; + ASSERT_TRUE(input.good()); + } + std::ostringstream out; + std::ostringstream err; + ASSERT_EQ(tsfile_cli::run_cli( + {"write", "--table", "t", "--field", "value", + type_name, "--encoding", type_name, encoding, + "--compression", type_name, "UNCOMPRESSED", "-i", + csv.path, "-o", file.path}, + out, err), + 0) + << err.str(); + EXPECT_EQ( + tsfile_cli::run_cli({"cat", "-f", "csv", file.path}, out, err), + 0) + << err.str(); + EXPECT_EQ(out.str(), rows); + out.str(""); + EXPECT_EQ(tsfile_cli::run_cli({"cat", "-f", "csv", "--offset", "2", + "-n", "2", file.path}, + out, err), + 0) + << err.str(); + EXPECT_EQ(out.str(), "time,value\n2,2\n3," + maximum + "\n"); + } + } +} + +} // namespace diff --git a/cpp/test/tools/command_e2e_test.cc b/cpp/test/tools/command_e2e_test.cc index e3af86ab1..71c437889 100644 --- a/cpp/test/tools/command_e2e_test.cc +++ b/cpp/test/tools/command_e2e_test.cc @@ -69,6 +69,23 @@ class FlushFailingStreamBuf : public std::stringbuf { int sync() override { return -1; } }; +class CountingStreamBuf : public std::streambuf { + public: + std::streamsize max_write = 0; + size_t bytes = 0; + size_t lines = 0; + + protected: + std::streamsize xsputn(const char* data, std::streamsize size) override { + if (size > max_write) max_write = size; + bytes += static_cast(size); + for (std::streamsize i = 0; i < size; ++i) { + if (data[i] == '\n') ++lines; + } + return size; + } +}; + size_t count_lines(const std::string& s) { size_t n = 0; for (char c : s) { @@ -81,6 +98,50 @@ size_t count_lines(const std::string& s) { } // namespace +TEST(CliE2E, LargeRowOutputPublishesBoundedBlocks) { + const std::string csv = + tsfile_cli_test::unique_temp_path("cli_large_output", ".csv"); + const std::string file = + tsfile_cli_test::unique_temp_path("cli_large_output", ".tsfile"); + const int row_count = 4096; + size_t csv_bytes = 0; + { + std::ofstream input(csv.c_str(), std::ios::binary); + input << "time,value\n"; + for (int row = 0; row < row_count; ++row) { + input << row << ',' << std::string(512, 'x') << '\n'; + } + csv_bytes = static_cast(input.tellp()); + } + std::ostringstream write_out; + std::ostringstream write_err; + const int write_code = + tsfile_cli::run_cli({"write", "--table", "t", "--field", "value", + "STRING", "-i", csv, "-o", file}, + write_out, write_err); + EXPECT_EQ(write_code, 0) << write_err.str(); + if (write_code == 0) { + for (const std::string format : {"csv", "ndjson", "table"}) { + CountingStreamBuf sink; + std::ostream out(&sink); + std::ostringstream err; + EXPECT_EQ( + tsfile_cli::run_cli({"cat", "-f", format, file}, out, err), 0) + << err.str(); + EXPECT_GT(sink.bytes, 2 * 1024 * 1024u); + EXPECT_LE(sink.max_write, 64 * 1024); + if (format == "csv") { + EXPECT_EQ(sink.bytes, csv_bytes); + EXPECT_EQ(sink.lines, row_count + 1u); + } else if (format == "ndjson") { + EXPECT_EQ(sink.lines, row_count); + } + } + } + std::remove(csv.c_str()); + std::remove(file.c_str()); +} + TEST(CliE2E, LsListsTableNameTsv) { Fixture f; std::ostringstream out; diff --git a/cpp/test/tools/output_format_test.cc b/cpp/test/tools/output_format_test.cc index aaf910691..d1284ebca 100644 --- a/cpp/test/tools/output_format_test.cc +++ b/cpp/test/tools/output_format_test.cc @@ -27,6 +27,7 @@ #include #include "common/db_common.h" +#include "format/staged_output.h" #include "utils/errno_define.h" using tsfile_cli::OutputFormat; @@ -48,6 +49,48 @@ class FlushFailingStreamBuf : public std::stringbuf { } // namespace +TEST(StagedOutputTest, PreservesBinaryDataAcrossBufferBoundaries) { + tsfile_cli::StagedOutput staged; + ASSERT_TRUE(staged.is_open()); + std::string expected; + for (size_t i = 0; i < 4 * 64 * 1024 + 317; ++i) { + expected += static_cast(i % 256); + } + staged.stream().write(expected.data(), expected.size()); + ASSERT_TRUE(staged.stream().good()); + std::ostringstream out; + EXPECT_TRUE(out.str().empty()); + EXPECT_TRUE(staged.publish(out)); + EXPECT_EQ(out.str(), expected); +} + +TEST(StagedOutputTest, PublishesEmptyOutput) { + tsfile_cli::StagedOutput staged; + std::ostringstream out; + EXPECT_TRUE(staged.publish(out)); + EXPECT_TRUE(out.str().empty()); +} + +TEST(StagedOutputTest, DoesNotPublishFailedStaging) { + tsfile_cli::StagedOutput staged; + staged.stream() << "partial rows\n"; + staged.stream().setstate(std::ios::badbit); + std::ostringstream out; + EXPECT_FALSE(staged.publish(out)); + EXPECT_TRUE(out.str().empty()); +} + +TEST(StagedOutputTest, ReportsWriteAndFlushFailures) { + tsfile_cli::StagedOutput staged; + staged.stream() << "rows\n"; + FailingStreamBuf failing_buffer; + std::ostream failing_out(&failing_buffer); + EXPECT_FALSE(staged.publish(failing_out)); + FlushFailingStreamBuf flush_buffer; + std::ostream flush_out(&flush_buffer); + EXPECT_FALSE(staged.publish(flush_out)); +} + TEST(ErrorCodeMessageTest, KnownCodesMapToReadablePhrases) { EXPECT_STREQ(tsfile_cli::error_code_message(common::E_TABLE_NOT_EXIST), "table does not exist"); @@ -185,6 +228,12 @@ TEST(EncodingNameTest, KnownEncodings) { EXPECT_STREQ(tsfile_cli::tsencoding_name(common::CHIMP), "CHIMP"); EXPECT_STREQ(tsfile_cli::tsencoding_name(common::RLBE), "RLBE"); EXPECT_STREQ(tsfile_cli::tsencoding_name(common::CAMEL), "CAMEL"); + EXPECT_STREQ(tsfile_cli::tsencoding_name(common::ALP), "ALP"); + EXPECT_STREQ(tsfile_cli::tsencoding_name(common::INVALID_ENCODING), + "UNKNOWN"); + EXPECT_STREQ(tsfile_cli::tsencoding_name( + static_cast(common::ALP + 1)), + "UNKNOWN"); } TEST(CompressionNameTest, KnownCompressors) { diff --git a/cpp/tools/README.md b/cpp/tools/README.md index ab09adc78..abb0b2a9c 100644 --- a/cpp/tools/README.md +++ b/cpp/tools/README.md @@ -96,7 +96,7 @@ Exit codes: `0` success, `1` usage/argument error, `2` file open/corrupt, | `count` | Object/column counts; no synthetic summary row | | `sketch` | Print the physical file sketch, optionally to `-o` | | `head` | First N rows (default 10; use `-n`) | -| `cat` | All matching rows, buffered before publishing | +| `cat` | All matching rows, disk-staged before publishing | | `export` | Export one object to `-o`, or multiple objects to `--output-dir`, using `--type` | Use `ls`, `schema`, and `meta` for metadata-only discovery. `stats` can scan @@ -119,12 +119,12 @@ strings, BOOLEAN/INT32/FLOAT/DOUBLE are bare values, nulls are `null`, and non-finite floats become `null`. CSV output follows RFC 4180. CSV nulls are unquoted `\N`; empty strings are quoted as `""`. Non-null STRING/TEXT values beginning with `\` gain one extra leading `\`, using the same encoding rule -as CSV input described below. Timestamps are raw int64 values. Row commands -currently stage the complete formatted result in memory before publishing it, -including CSV/NDJSON and `export`. The `table` formatter additionally uses a -temporary spool to align columns; this does not bound memory for the complete -output path. Use row, time, and TAG bounds for large reads. `sketch` does not -accept `--format`. +as CSV input described below. Timestamps are raw int64 values. Row commands use +bounded output buffers: stdout is staged on disk until reading succeeds, then +published in 64 KiB blocks. `export` streams directly to its atomic temporary +target. Reserve temporary disk space for the result; aligned `table` output +uses an additional spool. Row, time, and TAG bounds reduce the work required; +piping to `head` does not stop the initial read. `sketch` does not accept `--format`. Text output preserves well-formed UTF-8 and replaces malformed UTF-8 in names and text with U+FFFD (`�`). This also applies to sketch output and export @@ -168,10 +168,14 @@ By default each column uses the engine's default encoding and compression for it `--encoding` and `--compression` override by canonical data type, applying to every declared TAG/FIELD of that type; they do not target individual columns and do not affect `time`. -The CLI currently rejects explicit ALP, GORILLA_V1, and BOOLEAN/FLOAT/DOUBLE RLE -overrides even though the SDK supports them. ALP files can be read, but `schema` -currently displays their encoding as `UNKNOWN`. Native build options also -control compression availability; LZMA2 is opt-in. +Encoding overrides are validated against the SDK encoder/decoder factories, +including ALP and GORILLA_V1 for FLOAT/DOUBLE, and RLE for BOOLEAN/FLOAT/DOUBLE. +GORILLA_V1 reserves NaN as its end marker: CSV NaN values are rejected with the +column and line number, without committing output; NULL and infinities remain valid. +FLOAT/DOUBLE RLE follows the SDK's two-decimal scaling and may be lossy. +`schema` and verbose write summaries report ALP by name. ALP is C++/Python-only; +Java cannot query ALP series. Native build options control compression +availability; LZMA2 is opt-in. | Option | Meaning | |---|---| diff --git a/cpp/tools/commands/cmd_export.cc b/cpp/tools/commands/cmd_export.cc index 0646550e9..d1e61725f 100644 --- a/cpp/tools/commands/cmd_export.cc +++ b/cpp/tools/commands/cmd_export.cc @@ -194,7 +194,7 @@ int stream_query_to_file(const ParsedArgs& args, storage::TsFileReader& reader, return kExitRuntime; } code = run_row_query(args, reader, fmt, output, err, args.offset, - args.limit, &rows); + args.limit, &rows, /*stage_output=*/false); output.flush(); if (code == kExitOk && !output.good()) { err << "Error: failed to flush output target '" << path << "'\n"; diff --git a/cpp/tools/commands/cmd_write.cc b/cpp/tools/commands/cmd_write.cc index 5c7a804cc..09682eeba 100644 --- a/cpp/tools/commands/cmd_write.cc +++ b/cpp/tools/commands/cmd_write.cc @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -42,6 +43,8 @@ #include "common/datatype/date_converter.h" #include "common/schema.h" #include "common/tablet.h" +#include "encoding/decoder_factory.h" +#include "encoding/encoder_factory.h" #include "file/write_file.h" #include "format/atomic_output.h" #include "format/input_format.h" @@ -158,26 +161,11 @@ std::string lower_ascii(const std::string& s) { } bool parse_encoding_name(const std::string& name, common::TSEncoding& out) { - static const std::pair kEncodings[] = { - {"PLAIN", common::PLAIN}, - {"DICTIONARY", common::DICTIONARY}, - {"RLE", common::RLE}, - {"DIFF", common::DIFF}, - {"TS_2DIFF", common::TS_2DIFF}, - {"BITMAP", common::BITMAP}, - {"GORILLA_V1", common::GORILLA_V1}, - {"REGULAR", common::REGULAR}, - {"GORILLA", common::GORILLA}, - {"ZIGZAG", common::ZIGZAG}, - {"FREQ", common::FREQ}, - {"CHIMP", common::CHIMP}, - {"SPRINTZ", common::SPRINTZ}, - {"RLBE", common::RLBE}, - {"CAMEL", common::CAMEL}, - }; - for (const auto& e : kEncodings) { - if (name == e.first) { - out = e.second; + for (size_t i = 0; i < sizeof(common::s_encoding_names) / + sizeof(common::s_encoding_names[0]); + ++i) { + if (name == common::s_encoding_names[i]) { + out = static_cast(i); return true; } } @@ -207,33 +195,14 @@ bool parse_compression_name(const std::string& name, } bool encoding_supported(common::TSDataType type, common::TSEncoding encoding) { - switch (type) { - case common::BOOLEAN: - return encoding == common::PLAIN; - case common::INT32: - case common::DATE: - case common::INT64: - case common::TIMESTAMP: - return encoding == common::PLAIN || encoding == common::TS_2DIFF || - encoding == common::GORILLA || encoding == common::ZIGZAG || - encoding == common::RLE || encoding == common::SPRINTZ || - encoding == common::CHIMP || encoding == common::RLBE; - case common::FLOAT: - return encoding == common::PLAIN || encoding == common::TS_2DIFF || - encoding == common::GORILLA || encoding == common::SPRINTZ || - encoding == common::CHIMP || encoding == common::RLBE; - case common::DOUBLE: - return encoding == common::PLAIN || encoding == common::TS_2DIFF || - encoding == common::GORILLA || encoding == common::SPRINTZ || - encoding == common::CHIMP || encoding == common::RLBE || - encoding == common::CAMEL; - case common::STRING: - case common::TEXT: - case common::BLOB: - return encoding == common::PLAIN || encoding == common::DICTIONARY; - default: - return false; - } + auto* encoder = + storage::EncoderFactory::alloc_value_encoder(encoding, type); + auto* decoder = + storage::DecoderFactory::alloc_value_decoder(encoding, type); + const bool supported = encoder != nullptr && decoder != nullptr; + storage::EncoderFactory::free(encoder); + storage::DecoderFactory::free(decoder); + return supported; } bool compression_supported(common::CompressionType compression) { @@ -412,7 +381,8 @@ void unescape_leading_backslash_prefix(CsvCell& cell) { bool add_typed_value(storage::Tablet& tablet, uint32_t row, uint32_t schema_index, const ColumnDef& def, - const CsvCell& csv_cell, std::string& error) { + common::TSEncoding encoding, const CsvCell& csv_cell, + std::string& error) { if (is_csv_null(csv_cell)) { return true; // null } @@ -490,6 +460,13 @@ bool add_typed_value(storage::Tablet& tablet, uint32_t row, error = "FLOAT out of range '" + cell + "'"; return false; } + if (encoding == common::GORILLA_V1 && std::isnan(v)) { + error = + "NaN is reserved as the GORILLA_V1 end marker " + "(column '" + + def.name + "')"; + return false; + } tablet.add_value(row, schema_index, v); return true; } @@ -504,6 +481,13 @@ bool add_typed_value(storage::Tablet& tablet, uint32_t row, error = "DOUBLE out of range '" + cell + "'"; return false; } + if (encoding == common::GORILLA_V1 && std::isnan(v)) { + error = + "NaN is reserved as the GORILLA_V1 end marker " + "(column '" + + def.name + "')"; + return false; + } tablet.add_value(row, schema_index, v); return true; } @@ -676,7 +660,8 @@ int cmd_write(const ParsedArgs& args, std::ostream& /*out*/, for (size_t j = 0; j < columns.size(); ++j) { std::string cell_err; if (!add_typed_value(tablet, r, static_cast(j), - columns[j], batch[i].cells[j], cell_err)) { + columns[j], col_schemas[j].encoding_, + batch[i].cells[j], cell_err)) { err << "Error: " << cell_err << " (line " << batch[i].line_no << ")\n"; return kExitFile; diff --git a/cpp/tools/commands/commands.h b/cpp/tools/commands/commands.h index d20ae6a34..d7cd018c0 100644 --- a/cpp/tools/commands/commands.h +++ b/cpp/tools/commands/commands.h @@ -63,7 +63,7 @@ int validate_row_query_names(const ParsedArgs& args, int run_row_query(const ParsedArgs& args, storage::TsFileReader& reader, OutputFormat fmt, std::ostream& out, std::ostream& err, long long offset, long long limit, - long long* emitted_rows = nullptr); + long long* emitted_rows = nullptr, bool stage_output = true); int cmd_ls(const ParsedArgs& args, storage::TsFileReader& reader, OutputFormat fmt, std::ostream& out, std::ostream& err); diff --git a/cpp/tools/commands/row_query.cc b/cpp/tools/commands/row_query.cc index d35ae3121..04908a8df 100644 --- a/cpp/tools/commands/row_query.cc +++ b/cpp/tools/commands/row_query.cc @@ -32,6 +32,7 @@ #include "common/device_id.h" #include "common/schema.h" #include "format/result_set_format.h" +#include "format/staged_output.h" #include "reader/filter/tag_filter.h" #include "reader/tsfile_reader.h" @@ -374,7 +375,8 @@ int validate_row_query_names(const ParsedArgs& args, int run_row_query(const ParsedArgs& args, storage::TsFileReader& reader, OutputFormat fmt, std::ostream& out, std::ostream& err, - long long offset, long long limit, long long* emitted_rows) { + long long offset, long long limit, long long* emitted_rows, + bool stage_output) { const int64_t start = args.has_start ? static_cast(args.start) : std::numeric_limits::min(); const int64_t end = args.has_end ? static_cast(args.end) @@ -419,28 +421,36 @@ int run_row_query(const ParsedArgs& args, storage::TsFileReader& reader, return kExitFile; } - // Stage the complete result before publishing it. A decode/read failure - // can occur after the result header or earlier rows have been rendered; - // input failures must not leave a partial machine-readable stdout stream - // that callers could mistake for a complete result. The final write is - // still checked separately so stdout errors remain runtime failures. - std::ostringstream staged; + // Stdout is staged on disk so late input failures publish nothing without + // retaining the complete result in memory. Export already owns a temp file. + std::unique_ptr staged; + if (stage_output) { + staged.reset(new StagedOutput()); + if (!staged->is_open()) { + reader.destroy_query_data_set(rs); + err << "Error: cannot create temporary row output\n"; + return kExitRuntime; + } + } + std::ostream& destination = staged ? staged->stream() : out; std::string output_error; - int wret = - push_down - ? emit_result_set(rs, fmt, args.no_header, staged, 0, -1, - emitted_rows, &output_error, tree_device_prefix) - : emit_result_set(rs, fmt, args.no_header, staged, offset, limit, - emitted_rows, &output_error, tree_device_prefix); + int wret = push_down ? emit_result_set(rs, fmt, args.no_header, destination, + 0, -1, emitted_rows, &output_error, + tree_device_prefix) + : emit_result_set(rs, fmt, args.no_header, destination, + offset, limit, emitted_rows, + &output_error, tree_device_prefix); reader.destroy_query_data_set(rs); if (!output_error.empty()) { err << "Error: " << output_error << "\n"; return kExitFile; } if (wret == common::E_OK) { - const std::string bytes = staged.str(); - out.write(bytes.data(), static_cast(bytes.size())); - out.flush(); + if (staged) { + if (!staged->publish(out)) wret = common::E_FILE_WRITE_ERR; + } else { + out.flush(); + } if (!out.good()) { wret = common::E_FILE_WRITE_ERR; } diff --git a/cpp/tools/format/output_format.cc b/cpp/tools/format/output_format.cc index 696511456..ecaae2e45 100644 --- a/cpp/tools/format/output_format.cc +++ b/cpp/tools/format/output_format.cc @@ -128,41 +128,11 @@ const char* tsdatatype_name(common::TSDataType t) { } const char* tsencoding_name(common::TSEncoding e) { - switch (e) { - case common::PLAIN: - return "PLAIN"; - case common::DICTIONARY: - return "DICTIONARY"; - case common::RLE: - return "RLE"; - case common::DIFF: - return "DIFF"; - case common::TS_2DIFF: - return "TS_2DIFF"; - case common::BITMAP: - return "BITMAP"; - case common::GORILLA_V1: - return "GORILLA_V1"; - case common::REGULAR: - return "REGULAR"; - case common::GORILLA: - return "GORILLA"; - case common::ZIGZAG: - return "ZIGZAG"; - case common::FREQ: - return "FREQ"; - case common::CHIMP: - return "CHIMP"; - case common::SPRINTZ: - return "SPRINTZ"; - case common::RLBE: - return "RLBE"; - case common::CAMEL: - return "CAMEL"; - case common::INVALID_ENCODING: - default: - return "UNKNOWN"; - } + const size_t index = static_cast(e); + return index < sizeof(common::s_encoding_names) / + sizeof(common::s_encoding_names[0]) + ? common::s_encoding_names[index] + : "UNKNOWN"; } const char* compression_name(common::CompressionType c) { diff --git a/cpp/tools/format/staged_output.cc b/cpp/tools/format/staged_output.cc new file mode 100644 index 000000000..7fdf5f9d1 --- /dev/null +++ b/cpp/tools/format/staged_output.cc @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "format/staged_output.h" + +namespace tsfile_cli { + +StagedOutput::StagedOutput() : file_(std::tmpfile()), stream_(this) { + setp(buffer_, buffer_ + sizeof(buffer_)); + if (file_ == nullptr) stream_.setstate(std::ios::badbit); +} + +StagedOutput::~StagedOutput() { + if (file_ != nullptr) std::fclose(file_); +} + +int StagedOutput::sync() { + if (file_ == nullptr) return -1; + const size_t size = static_cast(pptr() - pbase()); + if (std::fwrite(buffer_, 1, size, file_) != size) return -1; + setp(buffer_, buffer_ + sizeof(buffer_)); + return 0; +} + +StagedOutput::int_type StagedOutput::overflow(int_type ch) { + if (sync() != 0) return traits_type::eof(); + if (!traits_type::eq_int_type(ch, traits_type::eof())) { + *pptr() = traits_type::to_char_type(ch); + pbump(1); + } + return traits_type::not_eof(ch); +} + +bool StagedOutput::publish(std::ostream& out) { + stream_.flush(); + if (!stream_.good() || std::fflush(file_) != 0 || + std::fseek(file_, 0, SEEK_SET) != 0) { + return false; + } + size_t size = 0; + while ((size = std::fread(buffer_, 1, sizeof(buffer_), file_)) != 0) { + out.write(buffer_, static_cast(size)); + if (!out.good()) return false; + } + if (std::ferror(file_)) return false; + out.flush(); + return out.good(); +} + +} // namespace tsfile_cli diff --git a/cpp/tools/format/staged_output.h b/cpp/tools/format/staged_output.h new file mode 100644 index 000000000..c23c445bb --- /dev/null +++ b/cpp/tools/format/staged_output.h @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef TSFILE_CLI_STAGED_OUTPUT_H +#define TSFILE_CLI_STAGED_OUTPUT_H + +#include +#include +#include + +namespace tsfile_cli { + +// Disk-backed output transaction: publish only after the input is fully read. +class StagedOutput : private std::streambuf { + public: + StagedOutput(); + ~StagedOutput() override; + StagedOutput(const StagedOutput&) = delete; + StagedOutput& operator=(const StagedOutput&) = delete; + + bool is_open() const { return file_ != nullptr; } + std::ostream& stream() { return stream_; } + bool publish(std::ostream& out); + + private: + int sync() override; + int_type overflow(int_type ch) override; + + std::FILE* file_; + char buffer_[64 * 1024]; + std::ostream stream_; +}; + +} // namespace tsfile_cli + +#endif // TSFILE_CLI_STAGED_OUTPUT_H diff --git a/cpp/tools/skills/tsfile-cli/SKILL.md b/cpp/tools/skills/tsfile-cli/SKILL.md index f81013f4b..0bebbe565 100644 --- a/cpp/tools/skills/tsfile-cli/SKILL.md +++ b/cpp/tools/skills/tsfile-cli/SKILL.md @@ -65,7 +65,7 @@ does not replace the Java batch and format-aware import tools. | `stats` | FIELD statistics with null counts and source | maybe | | `count` | object/column counts; no summary row | maybe | | `head` | first N rows (default 10, `-n`) | yes | -| `cat` | all matching rows (buffered) | yes | +| `cat` | all matching rows (disk-staged) | yes | | `sketch` | physical layout text from the bound `printSketch` behavior | no FIELD decode | | `export` | writes one object file or a numbered multi-object directory | yes | @@ -112,8 +112,11 @@ COMP ∈ UNCOMPRESSED, SNAPPY, GZIP, LZO, LZ4, ZSTD, LZMA2 - CSV must contain a unique header with `time` and exactly the declared TAG/FIELD names; rows are mapped by header name, not physical column order. - `--encoding` and `--compression` are keyed by declared data type and apply to all declared columns of that type. -- CLI rejects explicit ALP, GORILLA_V1, and BOOLEAN/FLOAT/DOUBLE RLE overrides; - ALP reads work but `schema` shows `UNKNOWN`. LZMA2 requires an enabled build. +- Encoding overrides follow the SDK encoder/decoder factories, including ALP + and GORILLA_V1 for FLOAT/DOUBLE, and RLE for BOOLEAN/FLOAT/DOUBLE. + GORILLA_V1 rejects NaN (reserved end marker), but accepts NULL and infinities. + FLOAT/DOUBLE RLE uses the SDK's two-decimal scaling and may be lossy. + ALP is C++/Python-only, not Java-compatible. LZMA2 requires an enabled build. - CSV null and backslash rules: see `references/commands.md` before preparing input. - `DATE` cells are `YYYY-MM-DD`; `TIMESTAMP` and row `time` use strict decimal int64. - **timestamps must be strictly increasing per device** (device = tag-column values); @@ -134,7 +137,8 @@ Tree-model / JSON / programmatic writes → C++ SDK `cpp/examples/cpp_examples/d ## Caveats -- `head`/`cat`/`export` buffer complete output in memory, including CSV/NDJSON; - use row/time/TAG bounds for large files. Nonzero exit means incomplete results. +- Row output uses bounded buffers: stdout is disk-staged until reads succeed; + `export` streams to an atomic temporary target. Allow disk space for the result. + Aligned `table` output also spools. Nonzero exit means incomplete results. - Table results always include all TAG columns before selected FIELD columns. - `sketch` follows the bound TsFile `printSketch` text and does not support `-f`. diff --git a/skills/tsfile-cli/SKILL.md b/skills/tsfile-cli/SKILL.md index 7abe9fb47..7f73e8ce9 100644 --- a/skills/tsfile-cli/SKILL.md +++ b/skills/tsfile-cli/SKILL.md @@ -60,7 +60,7 @@ does not replace the Java batch and format-aware import tools. | `stats` | FIELD statistics, null counts, and `stats_source` | maybe | | `count` | per-column row/entity/value counts; no summary row | maybe | | `head` | first N rows (default 10, `-n`) | yes | -| `cat` | all matching rows (buffered) | yes | +| `cat` | all matching rows (disk-staged) | yes | | `sketch` | physical layout text with offsets and chunk/page details | no FIELD decode | | `export` | writes one object file or a numbered multi-object directory | yes | @@ -91,9 +91,9 @@ opts: -f table|ndjson|csv (default table, including pipes) - `ndjson` emits one JSON object per line. INT64/TIMESTAMP values are decimal strings; BOOLEAN/INT32/FLOAT/DOUBLE are bare values, and NULL/NaN/Inf become JSON `null`. CSV includes a header and quotes cells using RFC 4180 rules. -- Row commands buffer complete output in memory, including CSV/NDJSON/export; - bound large reads with `-n` and time/TAG filters. Aligned `table` output also - uses a temporary spool and renders control characters as visible escapes. +- Row output uses bounded buffers: stdout is disk-staged until reads succeed; + `export` streams to an atomic temporary target. Allow disk space for the result. + Aligned `table` output also spools and renders control characters visibly. ## Sketch and export @@ -128,8 +128,11 @@ TYPE ∈ { BOOLEAN, INT32, INT64, FLOAT, DOUBLE, STRING, TEXT, TIMESTAMP, DATE, - The CSV header must contain `time` and exactly the declared TAG/FIELD names; rows are mapped by header name rather than physical column order. - `--encoding` and `--compression` may override the bound defaults by data type. -- CLI rejects explicit ALP, GORILLA_V1, and BOOLEAN/FLOAT/DOUBLE RLE overrides; - ALP reads work but `schema` shows `UNKNOWN`. LZMA2 requires an enabled build. +- Encoding overrides follow the SDK encoder/decoder factories, including ALP + and GORILLA_V1 for FLOAT/DOUBLE, and RLE for BOOLEAN/FLOAT/DOUBLE. + GORILLA_V1 rejects NaN (reserved end marker), but accepts NULL and infinities. + FLOAT/DOUBLE RLE uses the SDK's two-decimal scaling and may be lossy. + ALP is C++/Python-only, not Java-compatible. LZMA2 requires an enabled build. - `--stdin` or `-i ` is required; TSV, `--columns`, `--no-header`, and `--header-match` are not supported. - Unquoted `\N` denotes NULL. Empty STRING/TEXT cells are empty strings; diff --git a/skills/tsfile/references/cpp.md b/skills/tsfile/references/cpp.md index 28756373e..dd2227698 100644 --- a/skills/tsfile/references/cpp.md +++ b/skills/tsfile/references/cpp.md @@ -74,7 +74,8 @@ reader.close(); - Current readers use `query(..., ResultSet*&)`; do not reuse old `get_table_names()` or `read_table()` helpers without version verification. - Prefer stack/RAII ownership where supported, but still follow explicit - close/free requirements in the public API and examples. + close/free requirements. Table result rows reuse string storage; copy values + needed after advancing the result set. - Use `cpp/examples/cpp_examples/` as executable API documentation. - Verify with `./mvnw -P with-cpp clean verify` or the checkout's documented `cpp/build.sh` flow. diff --git a/skills/tsfile/references/encoding-compression.md b/skills/tsfile/references/encoding-compression.md index 010f83944..08edfa76e 100644 --- a/skills/tsfile/references/encoding-compression.md +++ b/skills/tsfile/references/encoding-compression.md @@ -71,8 +71,10 @@ allocations with `CompressorFactory::free`. Python exposes schema encoding/compression enums through the native binding but does not expose these Java/C++ low-level factories as an equivalent Python API. -ALP is C++/Python-only; Java cannot query ALP series. CLI write support is -narrower than this SDK matrix; use the `tsfile-cli` skill for its restrictions. +ALP is C++/Python-only; Java cannot query ALP series. CLI encoding overrides +follow the C++ encoder/decoder factory support matrix. +GORILLA_V1 reserves NaN as its end marker; the CLI rejects NaN input for this codec. +FLOAT/DOUBLE RLE uses two-decimal scaling by default and may be lossy. ## Source Anchors