diff --git a/cpp/src/reader/expression.cc b/cpp/src/reader/expression.cc index 08ffac98e..3623b390e 100644 --- a/cpp/src/reader/expression.cc +++ b/cpp/src/reader/expression.cc @@ -178,23 +178,17 @@ Expression* QueryExpression::optimize(Expression* expression, if (regular_left == nullptr || regular_right == nullptr) { return nullptr; } - Expression* mid_ret = nullptr; - if (type == AND_EXPR) { - mid_ret = new Expression(AND_EXPR, regular_left, regular_right); - my_exprs_.push_back(mid_ret); - } else if (type == OR_EXPR) { - mid_ret = new Expression(OR_EXPR, regular_left, regular_right); - my_exprs_.push_back(mid_ret); + // Reuse the node being optimized instead of allocating a second + // node wrapping the same children: the result becomes + // QueryExpression::expression_ and must be owned exactly once. + expression->left_ = regular_left; + expression->right_ = regular_right; + + if (regular_left->type_ == GLOBALTIME_EXPR || + regular_right->type_ == GLOBALTIME_EXPR) { + return optimize(expression, series_paths); } else { - std::cout << "unsupported Expression type:" << type - << std::endl; - } - - if (mid_ret->left_->type_ == GLOBALTIME_EXPR || - mid_ret->right_->type_ == GLOBALTIME_EXPR) { - return optimize(mid_ret, series_paths); - } else { - return mid_ret; + return expression; } } } else { diff --git a/cpp/src/reader/qds_with_timegenerator.cc b/cpp/src/reader/qds_with_timegenerator.cc index 3df3defbc..d5398bbc9 100644 --- a/cpp/src/reader/qds_with_timegenerator.cc +++ b/cpp/src/reader/qds_with_timegenerator.cc @@ -56,12 +56,12 @@ int64_t SeriesScanStream::front() { } else { if (col_iter_->end()) { int ret = E_OK; // cppcheck-suppress unreadVariable + delete col_iter_; + col_iter_ = nullptr; + tsblock_->reset(); if (RET_FAIL(ssi_->get_next(tsblock_, false))) { - delete col_iter_; - col_iter_ = nullptr; return INVALID_NEXT_TIMESTAMP; } else { - delete col_iter_; col_iter_ = new ColIterator(0, tsblock_); ASSERT(!col_iter_->end()); } @@ -136,15 +136,23 @@ void* ValueAt::at(int64_t target_timestamp) { } } tf_->reset_value(target_timestamp); - if (RET_FAIL(ssi_->get_next(tsblock_, (tsblock_ == nullptr), tf_))) { + delete time_col_iter_; + time_col_iter_ = nullptr; + delete value_col_iter_; + value_col_iter_ = nullptr; + // The iterators above only exist after the first lookup allocated + // tsblock_; keep the original allocate-if-null contract and only reset + // an already materialized block before it is reused. + const bool alloc_tsblock = (tsblock_ == nullptr); + if (tsblock_ != nullptr) { + tsblock_->reset(); + } + if (RET_FAIL(ssi_->get_next(tsblock_, alloc_tsblock, tf_))) { cur_time_ = INT64_MAX; return nullptr; - } else { - delete time_col_iter_; - delete value_col_iter_; - time_col_iter_ = new ColIterator(0, tsblock_); - value_col_iter_ = new ColIterator(1, tsblock_); } + time_col_iter_ = new ColIterator(0, tsblock_); + value_col_iter_ = new ColIterator(1, tsblock_); } } @@ -306,7 +314,7 @@ int QDSWithTimeGenerator::init(TsFileIOReader* io_reader, QueryExpression* qe) { return ret; } else { va.io_reader_ = io_reader_; - data_types.push_back(va.value_col_iter_->get_data_type()); + data_types.push_back(va.ssi_->get_data_type()); value_at_vec_.push_back(va); } } @@ -412,16 +420,27 @@ int QDSWithTimeGenerator::construct_node_tree(Expression* expr, Node*& node) { if (RET_FAIL(construct_node_tree(expr->left_, node->left_))) { } else if (RET_FAIL(construct_node_tree(expr->right_, node->right_))) { } + if (ret != E_OK) { + destroy_node(node); + node = nullptr; + } } else if (expr->type_ == SERIES_EXPR) { Node* leaf = new Node(LEAF_NODE); Path& path = expr->series_path_; - int ret = io_reader_->alloc_ssi(path.device_id_, path.measurement_, - leaf->sss_.ssi_, pa_, expr->filter_); + ret = io_reader_->alloc_ssi(path.device_id_, path.measurement_, + leaf->sss_.ssi_, pa_, expr->filter_); if (E_OK == ret) { leaf->sss_.init(); + } else if (ret == E_NO_MORE_DATA) { + // The filter pruned every chunk of this series. Keep the leaf so + // parent AND/OR nodes still have a child; its stream reports + // INVALID_NEXT_TIMESTAMP for every lookup. + ret = E_OK; + } + if (ret == E_OK) { node = leaf; } else { - // do nothing, this leaf node will return no data at all. + delete leaf; } } return ret; diff --git a/cpp/test/reader/qds_with_timegenerator_test.cc b/cpp/test/reader/qds_with_timegenerator_test.cc new file mode 100644 index 000000000..8bc756ff0 --- /dev/null +++ b/cpp/test/reader/qds_with_timegenerator_test.cc @@ -0,0 +1,508 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "reader/qds_with_timegenerator.h" + +#include + +#include +#include +#include +#include + +#include "common/global.h" +#include "common/record.h" +#include "common/schema.h" +#include "reader/expression.h" +#include "reader/filter/time_filter.h" +#include "reader/filter/time_operator.h" +#include "reader/tsfile_reader.h" +#include "writer/tsfile_writer.h" + +#if defined(_WIN32) +#include +#define QDS_GETPID _getpid +#else +#include +#define QDS_GETPID getpid +#endif + +namespace storage { +namespace { + +using namespace common; + +std::string unique_file_name(const char* tag) { + static std::atomic counter{0}; + return std::string(tag) + "_" + std::to_string(QDS_GETPID()) + "_" + + std::to_string(counter.fetch_add(1)) + ".tsfile"; +} + +Path series_path(const std::string& device, const std::string& measurement) { + return Path(device + "." + measurement); +} + +Expression* series_expr(const std::string& device, + const std::string& measurement, Filter* filter) { + return new Expression(SERIES_EXPR, series_path(device, measurement), + filter); +} + +struct RowSnapshot { + int64_t timestamp = 0; + bool value1_is_null = false; + int64_t value1 = 0; + bool value2_is_null = false; + int64_t value2 = 0; +}; + +// Collects the value columns present in the result (2 and 3 overall when both +// series are selected). +void collect_rows(ResultSet* result, std::vector& rows) { + const uint32_t column_count = result->get_metadata()->get_column_count(); + bool has_next = false; + while (true) { + ASSERT_EQ(result->next(has_next), E_OK); + if (!has_next) { + break; + } + RowRecord* record = result->get_row_record(); + ASSERT_NE(record, nullptr); + RowSnapshot row; + row.timestamp = record->get_timestamp(); + Field* first = record->get_field(1); + ASSERT_NE(first, nullptr); + row.value1_is_null = first->type_ == NULL_TYPE; + if (!row.value1_is_null) { + ASSERT_EQ(first->type_, INT64); + row.value1 = first->value_.lval_; + } + if (column_count > 2) { + Field* second = record->get_field(2); + ASSERT_NE(second, nullptr); + row.value2_is_null = second->type_ == NULL_TYPE; + if (!row.value2_is_null) { + ASSERT_EQ(second->type_, INT64); + row.value2 = second->value_.lval_; + } + } + rows.push_back(row); + } +} + +std::vector timestamps(const std::vector& rows) { + std::vector result; + result.reserve(rows.size()); + for (const auto& row : rows) { + result.push_back(row.timestamp); + } + return result; +} + +class QDSWithTimeGeneratorTest : public ::testing::Test { + protected: + void SetUp() override { + ASSERT_EQ(libtsfile_init(), E_OK); + file_ = unique_file_name("qds_with_timegenerator_test"); + std::remove(file_.c_str()); + write_file(); + ASSERT_EQ(reader_.open(file_), E_OK); + } + + void TearDown() override { + reader_.close(); + std::remove(file_.c_str()); + libtsfile_destroy(); + } + + // s1: [1, 8] -> 10 * t + // s2: {2, 4, 6} -> 100 * t + void write_file() { + TsFileWriter writer; + ASSERT_EQ(writer.open(file_), E_OK); + MeasurementSchema s1_schema("s1", INT64); + MeasurementSchema s2_schema("s2", INT64); + ASSERT_EQ(writer.register_timeseries(device_, s1_schema), E_OK); + ASSERT_EQ(writer.register_timeseries(device_, s2_schema), E_OK); + for (int64_t t = 1; t <= 8; ++t) { + TsRecord record(t, device_); + record.add_point("s1", static_cast(10 * t)); + ASSERT_EQ(writer.write_record(record), E_OK); + } + for (int64_t t : {2, 4, 6}) { + TsRecord record(t, device_); + record.add_point("s2", static_cast(100 * t)); + ASSERT_EQ(writer.write_record(record), E_OK); + } + ASSERT_EQ(writer.flush(), E_OK); + ASSERT_EQ(writer.close(), E_OK); + } + + ResultSet* query(Expression* expression, + const std::vector& measurements) { + std::vector paths; + for (const auto& measurement : measurements) { + paths.push_back(series_path(device_, measurement)); + } + ResultSet* result = nullptr; + EXPECT_EQ( + reader_.query(QueryExpression::create(paths, expression), result), + E_OK); + return result; + } + + const std::string device_ = "root.qds_with_timegenerator"; + std::string file_; + TsFileReader reader_; +}; + +TEST_F(QDSWithTimeGeneratorTest, SingleSeriesWithTimeFilter) { + ResultSet* result = + query(series_expr(device_, "s1", TimeFilter::gt_eq(3)), {"s1"}); + ASSERT_NE(result, nullptr); + + std::shared_ptr metadata = result->get_metadata(); + ASSERT_NE(metadata, nullptr); + EXPECT_EQ(metadata->get_column_count(), 2U); + EXPECT_EQ(metadata->get_column_name(1), "time"); + EXPECT_EQ(metadata->get_column_name(2), device_ + ".s1"); + EXPECT_EQ(metadata->get_column_type(2), INT64); + + std::vector rows; + bool has_next = false; + while (true) { + ASSERT_EQ(result->next(has_next), E_OK); + if (!has_next) { + break; + } + EXPECT_FALSE(result->is_null("s1")); + EXPECT_FALSE(result->is_null(2)); + EXPECT_FALSE(result->is_null(1)); + RowRecord* record = result->get_row_record(); + RowSnapshot row; + row.timestamp = record->get_timestamp(); + row.value1 = record->get_field(1)->value_.lval_; + rows.push_back(row); + } + ASSERT_EQ(rows.size(), 6U); + for (size_t i = 0; i < rows.size(); ++i) { + const int64_t expected_time = static_cast(i) + 3; + EXPECT_EQ(rows[i].timestamp, expected_time); + EXPECT_EQ(rows[i].value1, 10 * expected_time); + } + EXPECT_TRUE(result->is_null("unknown_column")); + reader_.destroy_query_data_set(result); +} + +TEST_F(QDSWithTimeGeneratorTest, AndOfTwoSeriesIntersectsTimestamps) { + Expression* expression = new Expression( + AND_EXPR, series_expr(device_, "s1", TimeFilter::gt_eq(2)), + series_expr(device_, "s2", TimeFilter::lt_eq(5))); + ResultSet* result = query(expression, {"s1", "s2"}); + ASSERT_NE(result, nullptr); + + std::vector rows; + collect_rows(result, rows); + ASSERT_EQ(timestamps(rows), (std::vector{2, 4})); + EXPECT_EQ(rows[0].value1, 20); + EXPECT_EQ(rows[0].value2, 200); + EXPECT_EQ(rows[1].value1, 40); + EXPECT_EQ(rows[1].value2, 400); + reader_.destroy_query_data_set(result); +} + +TEST_F(QDSWithTimeGeneratorTest, OrOfTwoSeriesUnionsAndMarksMissingNull) { + Expression* expression = + new Expression(OR_EXPR, series_expr(device_, "s1", TimeFilter::eq(3)), + series_expr(device_, "s2", TimeFilter::eq(4))); + ResultSet* result = query(expression, {"s1", "s2"}); + ASSERT_NE(result, nullptr); + + std::vector rows; + collect_rows(result, rows); + ASSERT_EQ(timestamps(rows), (std::vector{3, 4})); + EXPECT_EQ(rows[0].value1, 30); + EXPECT_TRUE(rows[0].value2_is_null); + EXPECT_EQ(rows[1].value1, 40); + EXPECT_EQ(rows[1].value2, 400); + reader_.destroy_query_data_set(result); +} + +TEST_F(QDSWithTimeGeneratorTest, AndNodeAdvancesTheRightSeriesFirst) { + // The left stream starts after the right one, so the AND cursor must + // advance the right series before the streams can meet. + Expression* expression = new Expression( + AND_EXPR, series_expr(device_, "s1", TimeFilter::gt_eq(4)), + series_expr(device_, "s2", TimeFilter::gt_eq(0))); + ResultSet* result = query(expression, {"s1", "s2"}); + ASSERT_NE(result, nullptr); + + std::vector rows; + collect_rows(result, rows); + ASSERT_EQ(timestamps(rows), (std::vector{4, 6})); + EXPECT_EQ(rows[0].value1, 40); + EXPECT_EQ(rows[0].value2, 400); + EXPECT_EQ(rows[1].value1, 60); + EXPECT_EQ(rows[1].value2, 600); + reader_.destroy_query_data_set(result); +} + +TEST_F(QDSWithTimeGeneratorTest, OrNodeAdvancesLeftSeriesFirst) { + Expression* expression = new Expression( + OR_EXPR, series_expr(device_, "s1", TimeFilter::gt_eq(1)), + series_expr(device_, "s2", TimeFilter::eq(4))); + ResultSet* result = query(expression, {"s1", "s2"}); + ASSERT_NE(result, nullptr); + + std::vector rows; + collect_rows(result, rows); + ASSERT_EQ(timestamps(rows), (std::vector{1, 2, 3, 4, 5, 6, 7, 8})); + EXPECT_EQ(rows[0].value1, 10); + EXPECT_TRUE(rows[0].value2_is_null); + EXPECT_EQ(rows[1].value2, 200); + reader_.destroy_query_data_set(result); +} + +TEST_F(QDSWithTimeGeneratorTest, OrNodeAdvancesRightSeriesFirst) { + Expression* expression = + new Expression(OR_EXPR, series_expr(device_, "s1", TimeFilter::eq(4)), + series_expr(device_, "s2", TimeFilter::eq(2))); + ResultSet* result = query(expression, {"s1", "s2"}); + ASSERT_NE(result, nullptr); + + std::vector rows; + collect_rows(result, rows); + ASSERT_EQ(timestamps(rows), (std::vector{2, 4})); + EXPECT_EQ(rows[0].value1, 20); + EXPECT_EQ(rows[0].value2, 200); + EXPECT_EQ(rows[1].value1, 40); + EXPECT_EQ(rows[1].value2, 400); + reader_.destroy_query_data_set(result); +} + +TEST_F(QDSWithTimeGeneratorTest, ValueAtStopsAfterSeriesIsExhausted) { + // s1 keeps producing timestamps past s2's last value, so the value lookup + // for s2 runs past the end of its scan and then short-circuits. + Expression* expression = new Expression( + OR_EXPR, series_expr(device_, "s1", TimeFilter::gt_eq(7)), + series_expr(device_, "s2", TimeFilter::eq(2))); + ResultSet* result = query(expression, {"s1", "s2"}); + ASSERT_NE(result, nullptr); + + std::vector rows; + collect_rows(result, rows); + ASSERT_EQ(timestamps(rows), (std::vector{2, 7, 8})); + EXPECT_EQ(rows[0].value1, 20); + EXPECT_EQ(rows[0].value2, 200); + EXPECT_EQ(rows[1].value1, 70); + EXPECT_TRUE(rows[1].value2_is_null); + EXPECT_EQ(rows[2].value1, 80); + EXPECT_TRUE(rows[2].value2_is_null); + reader_.destroy_query_data_set(result); +} + +TEST_F(QDSWithTimeGeneratorTest, ValueAtBeginsAfterSeriesRange) { + // s2 is selected but does not appear in the expression: its value lookup + // starts at a timestamp past its last value, so the first scan attempt is + // already exhausted. + Expression* expression = series_expr(device_, "s1", TimeFilter::gt_eq(7)); + ResultSet* result = query(expression, {"s1", "s2"}); + ASSERT_NE(result, nullptr); + + std::vector rows; + collect_rows(result, rows); + ASSERT_EQ(timestamps(rows), (std::vector{7, 8})); + EXPECT_EQ(rows[0].value1, 70); + EXPECT_TRUE(rows[0].value2_is_null); + EXPECT_EQ(rows[1].value1, 80); + EXPECT_TRUE(rows[1].value2_is_null); + reader_.destroy_query_data_set(result); +} + +TEST_F(QDSWithTimeGeneratorTest, StatisticsPrunedFilterReturnsEmptyResult) { + ResultSet* result = + query(series_expr(device_, "s1", TimeFilter::gt(1000)), {"s1"}); + ASSERT_NE(result, nullptr); + bool has_next = true; + ASSERT_EQ(result->next(has_next), E_OK); + EXPECT_FALSE(has_next); + EXPECT_EQ(result->get_metadata()->get_column_count(), 2U); + reader_.destroy_query_data_set(result); +} + +TEST_F(QDSWithTimeGeneratorTest, OrWithPrunedSeriesKeepsOtherSeries) { + // s2 has no chunk that can satisfy t > 100, so its leaf is an empty + // stream; the OR must still return s1's rows and pad s2 with NULLs. + Expression* expression = + new Expression(OR_EXPR, series_expr(device_, "s1", TimeFilter::eq(2)), + series_expr(device_, "s2", TimeFilter::gt(100))); + ResultSet* result = query(expression, {"s1", "s2"}); + ASSERT_NE(result, nullptr); + + std::vector rows; + collect_rows(result, rows); + ASSERT_EQ(timestamps(rows), (std::vector{2})); + EXPECT_EQ(rows[0].value1, 20); + EXPECT_EQ(rows[0].value2, 200); + reader_.destroy_query_data_set(result); +} + +TEST_F(QDSWithTimeGeneratorTest, AndWithPrunedSeriesReturnsEmptyResult) { + Expression* expression = + new Expression(AND_EXPR, series_expr(device_, "s1", TimeFilter::eq(2)), + series_expr(device_, "s2", TimeFilter::gt(100))); + ResultSet* result = query(expression, {"s1", "s2"}); + ASSERT_NE(result, nullptr); + bool has_next = true; + ASSERT_EQ(result->next(has_next), E_OK); + EXPECT_FALSE(has_next); + reader_.destroy_query_data_set(result); +} + +TEST_F(QDSWithTimeGeneratorTest, RowFilteredSeriesReturnsEmptyResult) { + // 5 lies inside s2's [2, 6] statistics range, so chunk pruning keeps the + // series but no row satisfies the filter. + ResultSet* result = + query(series_expr(device_, "s2", TimeFilter::eq(5)), {"s2"}); + ASSERT_NE(result, nullptr); + bool has_next = true; + ASSERT_EQ(result->next(has_next), E_OK); + EXPECT_FALSE(has_next); + reader_.destroy_query_data_set(result); +} + +TEST_F(QDSWithTimeGeneratorTest, MissingSeriesFailsTheQuery) { + ResultSet* result = nullptr; + std::vector paths{series_path(device_, "missing")}; + EXPECT_NE(reader_.query(QueryExpression::create( + paths, series_expr(device_, "missing", + TimeFilter::gt_eq(0))), + result), + E_OK); + EXPECT_EQ(result, nullptr); +} + +class QDSWithTimeGeneratorMultiBlockTest : public ::testing::Test { + protected: + void SetUp() override { + saved_block_memory_ = g_config_value_.tsblock_max_memory_; + config_modified_ = false; + ASSERT_EQ(libtsfile_init(), E_OK); + g_config_value_.tsblock_max_memory_ = 2048; + config_modified_ = true; + file_ = unique_file_name("qds_with_timegenerator_blocks_test"); + std::remove(file_.c_str()); + write_file(); + ASSERT_EQ(reader_.open(file_), E_OK); + } + + void TearDown() override { + reader_.close(); + std::remove(file_.c_str()); + if (config_modified_) { + g_config_value_.tsblock_max_memory_ = saved_block_memory_; + config_modified_ = false; + } + libtsfile_destroy(); + } + + // 300 rows per series guarantee several TsBlocks per series at 2 KiB. + void write_file() { + TsFileWriter writer; + ASSERT_EQ(writer.open(file_), E_OK); + MeasurementSchema s1_schema("s1", INT64); + MeasurementSchema s2_schema("s2", INT64); + ASSERT_EQ(writer.register_timeseries(device_, s1_schema), E_OK); + ASSERT_EQ(writer.register_timeseries(device_, s2_schema), E_OK); + for (int64_t t = 1; t <= 300; ++t) { + TsRecord record(t, device_); + record.add_point("s1", static_cast(t)); + ASSERT_EQ(writer.write_record(record), E_OK); + } + for (int64_t t = 2; t <= 300; t += 2) { + TsRecord record(t, device_); + record.add_point("s2", static_cast(2 * t)); + ASSERT_EQ(writer.write_record(record), E_OK); + } + ASSERT_EQ(writer.flush(), E_OK); + ASSERT_EQ(writer.close(), E_OK); + } + + const std::string device_ = "root.qds_with_timegenerator_blocks"; + uint32_t saved_block_memory_ = 0; + bool config_modified_ = false; + std::string file_; + TsFileReader reader_; +}; + +TEST_F(QDSWithTimeGeneratorMultiBlockTest, ScansAcrossMultipleTsBlocks) { + std::vector paths{series_path(device_, "s1")}; + ResultSet* result = nullptr; + ASSERT_EQ(reader_.query( + QueryExpression::create( + paths, series_expr(device_, "s1", TimeFilter::gt_eq(1))), + result), + E_OK); + ASSERT_NE(result, nullptr); + + std::vector rows; + collect_rows(result, rows); + ASSERT_EQ(rows.size(), 300U); + for (size_t i = 0; i < rows.size(); ++i) { + const int64_t expected_time = static_cast(i) + 1; + EXPECT_EQ(rows[i].timestamp, expected_time); + EXPECT_EQ(rows[i].value1, expected_time); + } + reader_.destroy_query_data_set(result); +} + +TEST_F(QDSWithTimeGeneratorMultiBlockTest, + OrMergesSeriesAcrossBlockBoundaries) { + Expression* expression = new Expression( + OR_EXPR, series_expr(device_, "s1", TimeFilter::gt_eq(297)), + series_expr(device_, "s2", TimeFilter::gt_eq(290))); + std::vector paths{series_path(device_, "s1"), + series_path(device_, "s2")}; + ResultSet* result = nullptr; + ASSERT_EQ(reader_.query(QueryExpression::create(paths, expression), result), + E_OK); + ASSERT_NE(result, nullptr); + + std::vector rows; + collect_rows(result, rows); + // s2 contributes the even timestamps 290..300, s1 adds 297, 298, 299, 300. + std::vector expected; + for (int64_t t = 290; t <= 300; ++t) { + if (t % 2 == 0 || t >= 297) { + expected.push_back(t); + } + } + EXPECT_EQ(timestamps(rows), expected); + for (const auto& row : rows) { + if (row.timestamp % 2 == 0) { + EXPECT_EQ(row.value2, 2 * row.timestamp); + } else { + EXPECT_TRUE(row.value2_is_null); + } + } + reader_.destroy_query_data_set(result); +} + +} // namespace +} // namespace storage