diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java index aa635d8288db0..34205959845c6 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java @@ -133,6 +133,39 @@ public class IoTDBTableAggregationIT { "INSERT INTO stat_table(time, device_id, s1, s3, s4) VALUES (5, 'n1', 50, 50.0, 50.0)", "INSERT INTO stat_table(time, device_id, s1, s2, s3, s4) VALUES (6, 'n1', 40, 20, 40.0, 20.0)", "FLUSH", + // rate(), increase(), irate() and delta() integration-test fixtures + "CREATE TABLE rate_test(scenario STRING TAG, value_i32 INT32 FIELD, value_i64 INT64 FIELD, value_float FLOAT FIELD, value_double DOUBLE FIELD, gauge DOUBLE FIELD, sample_time INT64 FIELD, window_start INT64 FIELD, window_end INT64 FIELD, text_value STRING FIELD, bool_value BOOLEAN FIELD)", + "CREATE TABLE rate_int32_extreme(scenario STRING TAG, value_col INT32 FIELD, sample_time INT64 FIELD, window_start INT64 FIELD, window_end INT64 FIELD)", + "CREATE TABLE rate_int64_extreme(scenario STRING TAG, value_col INT64 FIELD, sample_time INT64 FIELD, window_start INT64 FIELD, window_end INT64 FIELD)", + "INSERT INTO rate_test(time,scenario,value_i32,value_i64,value_float,value_double,gauge,sample_time,window_start,window_end,text_value,bool_value) VALUES (0,'timestamp_normal',100,100,100.0,100.0,100.0,0,0,5000,'x',true),(1,'timestamp_normal',160,160,160.0,160.0,160.0,1000,0,5000,'x',true)", + "INSERT INTO rate_test(time,scenario,value_double,gauge,sample_time,window_start,window_end) VALUES (100,'reset',500.0,500.0,0,0,5000),(101,'reset',580.0,580.0,1000,0,5000)", + "INSERT INTO rate_test(time,scenario,value_double,gauge,sample_time,window_start,window_end) VALUES (1000,'distributed',500.0,100.0,1000,1000,5000),(2000,'distributed',580.0,60.0,2000,1000,5000)", + "FLUSH", + "INSERT INTO rate_test(time,scenario,value_i32,value_i64,value_float,value_double,gauge,sample_time,window_start,window_end,text_value,bool_value) VALUES (2,'timestamp_normal',220,220,220.0,220.0,220.0,2000,0,5000,'x',true),(3,'timestamp_normal',280,280,280.0,280.0,280.0,3000,0,5000,'x',true),(4,'timestamp_normal',340,340,340.0,340.0,340.0,4000,0,5000,'x',true)", + "INSERT INTO rate_test(time,scenario,value_double,gauge,sample_time,window_start,window_end) VALUES (102,'reset',20.0,20.0,2000,0,5000),(103,'reset',100.0,100.0,3000,0,5000),(104,'reset',180.0,180.0,4000,0,5000)", + "INSERT INTO rate_test(time,scenario,value_double,gauge,sample_time,window_start,window_end) VALUES (604800000,'distributed',20.0,80.0,3000,1000,5000),(604801000,'distributed',100.0,70.0,4000,1000,5000)", + "INSERT INTO rate_test(time,scenario,value_double,gauge,sample_time,window_start,window_end) VALUES (200,'unordered',30.0,30.0,3000,0,5000),(201,'unordered',10.0,10.0,1000,0,5000),(202,'unordered',40.0,40.0,4000,0,5000),(203,'unordered',20.0,20.0,2000,0,5000)", + "INSERT INTO rate_test(time,scenario,value_double,gauge,sample_time,window_start,window_end) VALUES (300,'two_reset',580.0,580.0,1000,0,5000),(301,'two_reset',20.0,20.0,2000,0,5000)", + "INSERT INTO rate_test(time,scenario,value_double,gauge,sample_time,window_start,window_end) VALUES (310,'single',10.0,10.0,1000,0,5000)", + "INSERT INTO rate_test(time,scenario,value_double,gauge,sample_time,window_start,window_end) VALUES (320,'nulls',10.0,10.0,0,0,4000),(321,'nulls',null,null,null,null,null),(322,'nulls',30.0,30.0,2000,0,4000),(323,'nulls',40.0,40.0,3000,0,4000)", + "INSERT INTO rate_test(time,scenario,value_double,gauge,sample_time,window_start,window_end) VALUES (330,'multiple_reset',100.0,100.0,0,0,6000),(331,'multiple_reset',150.0,150.0,1000,0,6000),(332,'multiple_reset',20.0,20.0,2000,0,6000),(333,'multiple_reset',70.0,70.0,3000,0,6000),(334,'multiple_reset',10.0,10.0,4000,0,6000),(335,'multiple_reset',40.0,40.0,5000,0,6000)", + "INSERT INTO rate_test(time,scenario,value_double,gauge,sample_time,window_start,window_end) VALUES (340,'zero',100.0,100.0,2000,0,10000),(341,'zero',100.0,100.0,4000,0,10000),(342,'zero',100.0,100.0,6000,0,10000),(343,'zero',100.0,100.0,8000,0,10000)", + "INSERT INTO rate_test(time,scenario,value_double,gauge,sample_time,window_start,window_end) VALUES (350,'zero_protection',10.0,10.0,2000,0,10000),(351,'zero_protection',100.0,100.0,4000,0,10000),(352,'zero_protection',10.0,10.0,6000,0,10000),(353,'zero_protection',30.0,30.0,8000,0,10000)", + "INSERT INTO rate_test(time,scenario,value_double,gauge,sample_time,window_start,window_end) VALUES (360,'threshold',11.0,11.0,1100,0,4100),(361,'threshold',21.0,21.0,2100,0,4100),(362,'threshold',31.0,31.0,3100,0,4100)", + "INSERT INTO rate_test(time,scenario,value_double,gauge,sample_time,window_start,window_end) VALUES (370,'far_boundary',20.0,20.0,2000,0,10000),(371,'far_boundary',30.0,30.0,3000,0,10000),(372,'far_boundary',40.0,40.0,4000,0,10000)", + "INSERT INTO rate_test(time,scenario,value_double,gauge,sample_time,window_start,window_end) VALUES (380,'boundary',0.0,0.0,0,0,2000),(381,'boundary',10.0,10.0,1000,0,2000),(382,'boundary',20.0,20.0,2000,0,2000)", + "INSERT INTO rate_test(time,scenario,value_double,gauge,sample_time,window_start,window_end) VALUES (11100,'gapfill',100.0,100.0,11100,11000,12000),(11900,'gapfill',108.0,108.0,11900,11000,12000),(13100,'gapfill',120.0,120.0,13100,13000,14000),(13900,'gapfill',128.0,128.0,13900,13000,14000)", + "INSERT INTO rate_test(time,scenario,value_double,gauge,sample_time,window_start,window_end) VALUES (390,'negative_delta',100.0,100.0,0,0,3000),(391,'negative_delta',-10.0,-10.0,1000,0,3000),(392,'negative_delta',20.0,20.0,2000,0,3000)", + "INSERT INTO rate_test(time,scenario,value_double,gauge,window_start,window_end) VALUES (400,'null_time',10.0,10.0,0,2000)", + "INSERT INTO rate_test(time,scenario,value_double,gauge,sample_time,window_end) VALUES (401,'null_start',10.0,10.0,0,2000)", + "INSERT INTO rate_test(time,scenario,value_double,gauge,sample_time,window_start) VALUES (402,'null_end',10.0,10.0,0,0)", + "INSERT INTO rate_test(time,scenario,value_double,gauge,sample_time,window_start,window_end) VALUES (410,'inconsistent_window',10.0,10.0,0,0,2000),(411,'inconsistent_window',20.0,20.0,1000,100,2000)", + "INSERT INTO rate_test(time,scenario,value_double,gauge,sample_time,window_start,window_end) VALUES (420,'duplicate_time',0.0,0.0,0,0,2000),(421,'duplicate_time',10.0,10.0,1000,0,2000),(422,'duplicate_time',20.0,20.0,1000,0,2000)", + "INSERT INTO rate_test(time,scenario,value_double,gauge,sample_time,window_start,window_end) VALUES (430,'outside_before',10.0,10.0,-1,0,2000)", + "INSERT INTO rate_test(time,scenario,value_double,gauge,sample_time,window_start,window_end) VALUES (440,'counter_a',100.0,100.0,0,0,2000),(441,'counter_a',120.0,120.0,1000,0,2000),(450,'counter_b',1000.0,1000.0,0,0,2000),(451,'counter_b',1030.0,1030.0,1000,0,2000)", + "INSERT INTO rate_int32_extreme(time,scenario,value_col,sample_time,window_start,window_end) VALUES (0,'extreme',-2147483648,0,0,2000),(1,'extreme',2147483647,1000,0,2000)", + "INSERT INTO rate_int64_extreme(time,scenario,value_col,sample_time,window_start,window_end) VALUES (0,'extreme',-9223372036854775808,0,0,2000),(1,'extreme',9223372036854775807,1000,0,2000)", + "FLUSH", "CLEAR ATTRIBUTE CACHE", }; @@ -5717,6 +5750,479 @@ public void orderByLimitTest() { DATABASE_NAME); } + // ================================================================== + // ================= Rate-Family Aggregation Tests ================== + // ================================================================== + + @Test + public void rateFunctionsNormalTest() { + tableResultSetEqualTest( + "SELECT scenario,date_bin(5ms,time) AS window_start," + + "rate(value_double,time,window_start,window_start + 5) AS rate_value," + + "increase(value_double,time,window_start,window_start + 5) AS increase_value," + + "irate(value_double,time) AS irate_value," + + "delta(value_double,time,window_start,window_start + 5) AS delta_value " + + "FROM (SELECT time,scenario,value_double FROM rate_test " + + "WHERE scenario='timestamp_normal') GROUP BY scenario,window_start", + new String[] { + "scenario", "window_start", "rate_value", "increase_value", "irate_value", "delta_value" + }, + new String[] {"timestamp_normal,1970-01-01T00:00:00.000Z,60000.0,300.0,60000.0,300.0,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT rate(value_double,sample_time,window_start,window_end) AS rate_value," + + "increase(value_double,sample_time,window_start,window_end) AS increase_value," + + "irate(value_double,sample_time) AS irate_value," + + "delta(value_double,sample_time,window_start,window_end) AS delta_value " + + "FROM rate_test WHERE scenario='reset'", + new String[] {"rate_value", "increase_value", "irate_value", "delta_value"}, + new String[] {"65.0,325.0,80.0,-400.0,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT scenario,rate(value_double,sample_time,window_start,window_end) AS rate_value," + + "increase(value_double,sample_time,window_start,window_end) AS increase_value," + + "irate(value_double,sample_time) AS irate_value," + + "delta(value_double,sample_time,window_start,window_end) AS delta_value " + + "FROM rate_test WHERE scenario IN ('timestamp_normal','reset') " + + "GROUP BY scenario ORDER BY scenario", + new String[] {"scenario", "rate_value", "increase_value", "irate_value", "delta_value"}, + new String[] {"reset,65.0,325.0,80.0,-400.0,", "timestamp_normal,60.0,300.0,60.0,300.0,"}, + DATABASE_NAME); + + // The rows span two flushed files and two physical time partitions. In cluster mode this also + // exercises partial/intermediate/final aggregation. + tableResultSetEqualTest( + "SELECT rate(value_double,sample_time,window_start,window_end) AS rate_value," + + "increase(value_double,sample_time,window_start,window_end) AS increase_value," + + "irate(value_double,sample_time) AS irate_value," + + "delta(gauge,sample_time,window_start,window_end) AS delta_value " + + "FROM rate_test WHERE scenario='distributed'", + new String[] {"rate_value", "increase_value", "irate_value", "delta_value"}, + new String[] {"60.0,240.0,80.0,-40.0,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT rate(value_double,sample_time,window_start,window_end) AS rate_value," + + "increase(value_double,sample_time,window_start,window_end) AS increase_value," + + "irate(value_double,sample_time) AS irate_value," + + "delta(value_double,sample_time,window_start,window_end) AS delta_value " + + "FROM rate_test WHERE scenario='unordered'", + new String[] {"rate_value", "increase_value", "irate_value", "delta_value"}, + new String[] {"10.0,50.0,10.0,50.0,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT window_start,window_end," + + "rate(value_double,time,window_start,window_end) AS rate_value," + + "increase(value_double,time,window_start,window_end) AS increase_value," + + "irate(value_double,time) AS irate_value," + + "delta(value_double,time,window_start,window_end) AS delta_value " + + "FROM TUMBLE(DATA => (SELECT time,value_double FROM rate_test " + + "WHERE scenario='timestamp_normal'),TIMECOL => 'time',SIZE => 5ms) " + + "GROUP BY window_start,window_end", + new String[] { + "window_start", "window_end", "rate_value", "increase_value", "irate_value", "delta_value" + }, + new String[] { + "1970-01-01T00:00:00.000Z,1970-01-01T00:00:00.005Z,60000.0,300.0,60000.0,300.0," + }, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT window_start,window_end," + + "rate(value_double,time,window_start,window_end) AS rate_value " + + "FROM HOP(DATA => (SELECT time,value_double FROM rate_test " + + "WHERE scenario='timestamp_normal'),TIMECOL => 'time',SLIDE => 5ms,SIZE => 5ms) " + + "GROUP BY window_start,window_end", + new String[] {"window_start", "window_end", "rate_value"}, + new String[] {"1970-01-01T00:00:00.000Z,1970-01-01T00:00:00.005Z,60000.0,"}, + DATABASE_NAME); + } + + @Test + public void rateFunctionsBoundaryTest() { + tableResultSetEqualTest( + "SELECT rate(value_double,sample_time,window_start,window_end) AS rate_value," + + "increase(value_double,sample_time,window_start,window_end) AS increase_value," + + "irate(value_double,sample_time) AS irate_value," + + "delta(value_double,sample_time,window_start,window_end) AS delta_value " + + "FROM rate_test WHERE scenario='single'", + new String[] {"rate_value", "increase_value", "irate_value", "delta_value"}, + new String[] {"null,null,null,null,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT rate(value_double,sample_time,window_start,window_end) AS rate_value," + + "increase(value_double,sample_time,window_start,window_end) AS increase_value," + + "irate(value_double,sample_time) AS irate_value," + + "delta(value_double,sample_time,window_start,window_end) AS delta_value " + + "FROM rate_test WHERE scenario='missing'", + new String[] {"rate_value", "increase_value", "irate_value", "delta_value"}, + new String[] {"null,null,null,null,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT rate(value_double,sample_time,window_start,window_end) AS rate_value," + + "increase(value_double,sample_time,window_start,window_end) AS increase_value," + + "irate(value_double,sample_time) AS irate_value," + + "delta(value_double,sample_time,window_start,window_end) AS delta_value " + + "FROM rate_test WHERE scenario='two_reset'", + new String[] {"rate_value", "increase_value", "irate_value", "delta_value"}, + new String[] {"10.0,50.0,20.0,-1400.0,"}, + DATABASE_NAME); + + // The NULL row has NULL time and boundary columns as well; it must be ignored with value_col. + tableResultSetEqualTest( + "SELECT rate(value_double,sample_time,window_start,window_end) AS rate_value," + + "increase(value_double,sample_time,window_start,window_end) AS increase_value," + + "irate(value_double,sample_time) AS irate_value," + + "delta(value_double,sample_time,window_start,window_end) AS delta_value " + + "FROM rate_test WHERE scenario='nulls'", + new String[] {"rate_value", "increase_value", "irate_value", "delta_value"}, + new String[] {"10.0,40.0,10.0,40.0,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT rate(value_double,sample_time,window_start,window_end) AS rate_value," + + "increase(value_double,sample_time,window_start,window_end) AS increase_value," + + "irate(value_double,sample_time) AS irate_value," + + "delta(value_double,sample_time,window_start,window_end) AS delta_value " + + "FROM rate_test WHERE scenario='multiple_reset'", + new String[] {"rate_value", "increase_value", "irate_value", "delta_value"}, + new String[] {"32.0,192.0,30.0,-72.0,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT rate(value_double,sample_time,window_start,window_end) AS rate_value," + + "increase(value_double,sample_time,window_start,window_end) AS increase_value," + + "irate(value_double,sample_time) AS irate_value," + + "delta(value_double,sample_time,window_start,window_end) AS delta_value " + + "FROM rate_test WHERE scenario='zero'", + new String[] {"rate_value", "increase_value", "irate_value", "delta_value"}, + new String[] {"0.0,0.0,0.0,0.0,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT rate(value_double,sample_time,window_start,window_end) AS rate_value," + + "increase(value_double,sample_time,window_start,window_end) AS increase_value " + + "FROM rate_test WHERE scenario='zero_protection'", + new String[] {"rate_value", "increase_value"}, + new String[] {"17.0,170.0,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT rate(value_double,sample_time,window_start,window_end) AS rate_value," + + "increase(value_double,sample_time,window_start,window_end) AS increase_value," + + "irate(value_double,sample_time) AS irate_value," + + "delta(value_double,sample_time,window_start,window_end) AS delta_value " + + "FROM rate_test WHERE scenario='threshold'", + new String[] {"rate_value", "increase_value", "irate_value", "delta_value"}, + new String[] {"8.536585365853659,35.0,10.0,35.0,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT rate(value_double,sample_time,window_start,window_end) AS rate_value," + + "increase(value_double,sample_time,window_start,window_end) AS increase_value," + + "irate(value_double,sample_time) AS irate_value," + + "delta(value_double,sample_time,window_start,window_end) AS delta_value " + + "FROM rate_test WHERE scenario='far_boundary'", + new String[] {"rate_value", "increase_value", "irate_value", "delta_value"}, + new String[] {"3.0,30.0,10.0,30.0,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT rate(value_double,sample_time,window_start,window_end) AS rate_value," + + "increase(value_double,sample_time,window_start,window_end) AS increase_value," + + "irate(value_double,sample_time) AS irate_value," + + "delta(value_double,sample_time,window_start,window_end) AS delta_value " + + "FROM rate_test WHERE scenario='boundary' " + + "AND sample_time >= window_start AND sample_time < window_end", + new String[] {"rate_value", "increase_value", "irate_value", "delta_value"}, + new String[] {"10.0,20.0,10.0,20.0,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT date_bin_gapfill(1s,time) AS gap_start," + + "rate(value_double,time,gap_start,gap_start + 1000) AS rate_value," + + "increase(value_double,time,gap_start,gap_start + 1000) AS increase_value," + + "irate(value_double,time) AS irate_value," + + "delta(value_double,time,gap_start,gap_start + 1000) AS delta_value " + + "FROM rate_test WHERE scenario='gapfill' AND time >= 11000 AND time < 14000 " + + "GROUP BY gap_start ORDER BY gap_start", + new String[] {"gap_start", "rate_value", "increase_value", "irate_value", "delta_value"}, + new String[] { + "1970-01-01T00:00:11.000Z,10.0,10.0,10.0,10.0,", + "1970-01-01T00:00:12.000Z,null,null,null,null,", + "1970-01-01T00:00:13.000Z,10.0,10.0,10.0,10.0," + }, + DATABASE_NAME); + } + + @Test + public void rateFunctionsTypeTest() { + tableResultSetEqualTest( + "SELECT rate(value_i32,sample_time,window_start,window_end) AS rate_i32," + + "rate(value_i64,sample_time,window_start,window_end) AS rate_i64," + + "rate(value_float,sample_time,window_start,window_end) AS rate_float," + + "rate(value_double,sample_time,window_start,window_end) AS rate_double," + + "increase(value_i32,sample_time,window_start,window_end) AS increase_i32," + + "increase(value_i64,sample_time,window_start,window_end) AS increase_i64," + + "increase(value_float,sample_time,window_start,window_end) AS increase_float," + + "increase(value_double,sample_time,window_start,window_end) AS increase_double," + + "irate(value_i32,sample_time) AS irate_i32," + + "irate(value_i64,sample_time) AS irate_i64," + + "irate(value_float,sample_time) AS irate_float," + + "irate(value_double,sample_time) AS irate_double," + + "delta(value_i32,sample_time,window_start,window_end) AS delta_i32," + + "delta(value_i64,sample_time,window_start,window_end) AS delta_i64," + + "delta(value_float,sample_time,window_start,window_end) AS delta_float," + + "delta(value_double,sample_time,window_start,window_end) AS delta_double " + + "FROM rate_test WHERE scenario='timestamp_normal'", + new String[] { + "rate_i32", + "rate_i64", + "rate_float", + "rate_double", + "increase_i32", + "increase_i64", + "increase_float", + "increase_double", + "irate_i32", + "irate_i64", + "irate_float", + "irate_double", + "delta_i32", + "delta_i64", + "delta_float", + "delta_double" + }, + new String[] { + "60.0,60.0,60.0,60.0,300.0,300.0,300.0,300.0," + + "60.0,60.0,60.0,60.0,300.0,300.0,300.0,300.0," + }, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT rate(value_double,time," + + "CAST('1970-01-01T00:00:00.000+00:00' AS TIMESTAMP)," + + "CAST('1970-01-01T00:00:00.005+00:00' AS TIMESTAMP)) AS rate_value," + + "irate(value_double,time) AS irate_value " + + "FROM rate_test WHERE scenario='timestamp_normal'", + new String[] {"rate_value", "irate_value"}, + new String[] {"60000.0,60000.0,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT delta(value_double,sample_time,window_start,window_end) AS delta_value " + + "FROM rate_test WHERE scenario='negative_delta'", + new String[] {"delta_value"}, + new String[] {"-120.0,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT delta(value_col,sample_time,window_start,window_end) AS delta_value " + + "FROM rate_int32_extreme", + new String[] {"delta_value"}, + new String[] {"8.58993459E9,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT delta(value_col,sample_time,window_start,window_end) AS delta_value " + + "FROM rate_int64_extreme", + new String[] {"delta_value"}, + new String[] {"3.6893488147419103E19,"}, + DATABASE_NAME); + } + + @Test + public void rateFunctionsExceptionTest() { + for (String function : new String[] {"rate", "increase", "delta"}) { + tableAssertTestFail( + "SELECT " + function + "(value_double,sample_time,window_start) FROM rate_test", + "Aggregate function [" + function + "] requires 4 arguments", + DATABASE_NAME); + tableAssertTestFail( + "SELECT " + function + "(text_value,sample_time,window_start,window_end) FROM rate_test", + "Aggregate function [" + + function + + "] only supports INT32, INT64, FLOAT and DOUBLE as the first argument", + DATABASE_NAME); + tableAssertTestFail( + "SELECT " + function + "(value_double,text_value,window_start,window_end) FROM rate_test", + "The time arguments of aggregate function [" + + function + + "] should be TIMESTAMP or INT64 type", + DATABASE_NAME); + } + + tableAssertTestFail( + "SELECT irate(value_double,sample_time,window_start) FROM rate_test", + "Aggregate function [irate] requires 2 arguments", + DATABASE_NAME); + tableAssertTestFail( + "SELECT irate(text_value,sample_time) FROM rate_test", + "Aggregate function [irate] only supports INT32, INT64, FLOAT and DOUBLE as the first argument", + DATABASE_NAME); + tableAssertTestFail( + "SELECT irate(value_double,bool_value) FROM rate_test", + "The time arguments of aggregate function [irate] should be TIMESTAMP or INT64 type", + DATABASE_NAME); + tableAssertTestFail( + "SELECT increase(value_double,sample_time,text_value,window_end) FROM rate_test", + "The time arguments of aggregate function [increase] should be TIMESTAMP or INT64 type", + DATABASE_NAME); + tableAssertTestFail( + "SELECT delta(value_double,sample_time,window_start,bool_value) FROM rate_test", + "The time arguments of aggregate function [delta] should be TIMESTAMP or INT64 type", + DATABASE_NAME); + + tableAssertTestFail( + "SELECT rate(from_ieee754_64(X'7FF8000000000000'),sample_time,window_start,window_end) " + + "FROM rate_test WHERE scenario='timestamp_normal'", + "Aggregate function [rate] does not support non-finite value_col: NaN", + DATABASE_NAME); + tableAssertTestFail( + "SELECT increase(from_ieee754_64(X'7FF0000000000000'),sample_time,window_start,window_end) " + + "FROM rate_test WHERE scenario='timestamp_normal'", + "Aggregate function [increase] does not support non-finite value_col: Infinity", + DATABASE_NAME); + tableAssertTestFail( + "SELECT irate(from_ieee754_64(X'FFF0000000000000'),sample_time) " + + "FROM rate_test WHERE scenario='timestamp_normal'", + "Aggregate function [irate] does not support non-finite value_col: -Infinity", + DATABASE_NAME); + tableAssertTestFail( + "SELECT delta(from_ieee754_64(X'7FF8000000000000'),sample_time,window_start,window_end) " + + "FROM rate_test WHERE scenario='timestamp_normal'", + "Aggregate function [delta] does not support non-finite value_col: NaN", + DATABASE_NAME); + + tableAssertTestFail( + "SELECT rate(value_double,sample_time,window_start,window_end) " + + "FROM rate_test WHERE scenario='negative_delta'", + "The value_col argument of aggregate function [rate] must be a non-negative number", + DATABASE_NAME); + tableAssertTestFail( + "SELECT increase(value_double,sample_time,window_start,window_end) " + + "FROM rate_test WHERE scenario='negative_delta'", + "The value_col argument of aggregate function [increase] must be a non-negative number", + DATABASE_NAME); + tableAssertTestFail( + "SELECT irate(value_double,sample_time) " + + "FROM rate_test WHERE scenario='negative_delta'", + "The value_col argument of aggregate function [irate] must be a non-negative number", + DATABASE_NAME); + + tableAssertTestFail( + "SELECT rate(value_double,sample_time,window_start,window_end) " + + "FROM rate_test WHERE scenario='null_time'", + "The argument 2 of aggregate function [rate] must not be NULL", + DATABASE_NAME); + tableAssertTestFail( + "SELECT irate(value_double,sample_time) FROM rate_test WHERE scenario='null_time'", + "The argument 2 of aggregate function [irate] must not be NULL", + DATABASE_NAME); + tableAssertTestFail( + "SELECT rate(value_double,sample_time,window_start,window_end) " + + "FROM rate_test WHERE scenario='null_start'", + "The argument 3 of aggregate function [rate] must not be NULL", + DATABASE_NAME); + tableAssertTestFail( + "SELECT rate(value_double,sample_time,window_start,window_end) " + + "FROM rate_test WHERE scenario='null_end'", + "The argument 4 of aggregate function [rate] must not be NULL", + DATABASE_NAME); + tableAssertTestFail( + "SELECT increase(value_double,sample_time,window_start,window_end) " + + "FROM rate_test WHERE scenario='null_start'", + "The argument 3 of aggregate function [increase] must not be NULL", + DATABASE_NAME); + tableAssertTestFail( + "SELECT delta(value_double,sample_time,window_start,window_end) " + + "FROM rate_test WHERE scenario='null_end'", + "The argument 4 of aggregate function [delta] must not be NULL", + DATABASE_NAME); + + tableAssertTestFail( + "SELECT rate(value_double,sample_time,CAST(5000 AS INT64),CAST(0 AS INT64)) " + + "FROM rate_test WHERE scenario='timestamp_normal'", + "The window_start argument of aggregate function [rate] must be less than window_end", + DATABASE_NAME); + tableAssertTestFail( + "SELECT rate(value_double,sample_time,CAST(0 AS INT64),CAST(0 AS INT64)) " + + "FROM rate_test WHERE scenario='timestamp_normal'", + "The window_start argument of aggregate function [rate] must be less than window_end", + DATABASE_NAME); + tableAssertTestFail( + "SELECT increase(value_double,sample_time,CAST(5000 AS INT64),CAST(0 AS INT64)) " + + "FROM rate_test WHERE scenario='timestamp_normal'", + "The window_start argument of aggregate function [increase] must be less than window_end", + DATABASE_NAME); + tableAssertTestFail( + "SELECT delta(value_double,sample_time,CAST(0 AS INT64),CAST(0 AS INT64)) " + + "FROM rate_test WHERE scenario='timestamp_normal'", + "The window_start argument of aggregate function [delta] must be less than window_end", + DATABASE_NAME); + + for (String function : new String[] {"rate", "increase", "delta"}) { + tableAssertTestFail( + "SELECT " + + function + + "(value_double,sample_time,window_start,window_end) " + + "FROM rate_test WHERE scenario='boundary'", + "The sample time of aggregate function [" + + function + + "] must satisfy window_start <= time_col < window_end", + DATABASE_NAME); + tableAssertTestFail( + "SELECT " + + function + + "(value_double,sample_time,window_start,window_end) " + + "FROM rate_test WHERE scenario='inconsistent_window'", + "Aggregate function [" + + function + + "] requires consistent window boundaries in the same aggregation group", + DATABASE_NAME); + } + tableAssertTestFail( + "SELECT rate(value_double,sample_time,window_start,window_end) " + + "FROM rate_test WHERE scenario='outside_before'", + "The sample time of aggregate function [rate] must satisfy window_start <= time_col < window_end", + DATABASE_NAME); + + for (String function : new String[] {"rate", "increase", "delta"}) { + tableAssertTestFail( + "SELECT " + + function + + "(value_double,sample_time,window_start,window_end) " + + "FROM rate_test WHERE scenario='duplicate_time'", + "Aggregate function [" + + function + + "] does not support duplicate time_col values in the same aggregation group: 1000", + DATABASE_NAME); + } + tableAssertTestFail( + "SELECT irate(value_double,sample_time) " + + "FROM rate_test WHERE scenario='duplicate_time'", + "Aggregate function [irate] does not support duplicate time_col values in the same aggregation group: 1000", + DATABASE_NAME); + + tableAssertTestFail( + "SELECT rate(value_double,sample_time,window_start,window_end) FROM rate_test " + + "WHERE scenario IN ('counter_a','counter_b')", + "Aggregate function [rate] does not support duplicate time_col values in the same aggregation group", + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT scenario,rate(value_double,sample_time,window_start,window_end) AS rate_value " + + "FROM rate_test WHERE scenario IN ('counter_a','counter_b') " + + "GROUP BY scenario ORDER BY scenario", + new String[] {"scenario", "rate_value"}, + new String[] {"counter_a,20.0,", "counter_b,30.0,"}, + DATABASE_NAME); + } + @Test public void statFunctionsSampleSizeAndFilterEdgeTest() { String[] expectedHeader = diff --git a/iotdb-core/calc-commons/src/main/i18n/en/org/apache/iotdb/calc/i18n/CalcMessages.java b/iotdb-core/calc-commons/src/main/i18n/en/org/apache/iotdb/calc/i18n/CalcMessages.java index d8ec4e0e6f9bb..5bce4eaca773b 100644 --- a/iotdb-core/calc-commons/src/main/i18n/en/org/apache/iotdb/calc/i18n/CalcMessages.java +++ b/iotdb-core/calc-commons/src/main/i18n/en/org/apache/iotdb/calc/i18n/CalcMessages.java @@ -468,5 +468,62 @@ private CalcMessages() {} "Unsupported data type in PERCENTILE Aggregation: %s"; public static final String EXCEPTION_PERCENTILEACCUMULATOR_DOES_NOT_SUPPORT_STATISTICS_66308C79 = "PercentileAccumulator does not support statistics"; + public static final String + EXCEPTION_RATE_FAMILY_AGGREGATE_FUNCTION_REQUIRES_AT_LEAST_TWO_VALID_SAMPLES_1A89901C = + "Rate-family aggregate function requires at least two valid samples"; + public static final String + EXCEPTION_THE_FIRST_SAMPLE_TIME_MUST_BE_LESS_THAN_THE_LAST_SAMPLE_TIME_FC4D3517 = + "The first sample time must be less than the last sample time"; + public static final String + EXCEPTION_THE_WINDOW_START_MUST_BE_LESS_THAN_THE_WINDOW_END_B1A38C98 = + "The window start must be less than the window end"; + public static final String + EXCEPTION_RATE_FAMILY_AGGREGATE_FUNCTION_REQUIRES_WINDOW_START_FIRST_TIME_LAST_TIME_WINDOW_END_60DE93C8 = + "Rate-family aggregate function requires window_start <= first_time < last_time < window_end"; + public static final String + EXCEPTION_RATE_FAMILY_AGGREGATE_FUNCTION_PRODUCED_A_NON_FINITE_INTERMEDIATE_RESULT_D46B30CD = + "Rate-family aggregate function produced a non-finite intermediate result"; + public static final String + EXCEPTION_RATE_FAMILY_AGGREGATE_FUNCTION_PRODUCED_A_NON_FINITE_TIME_INTERVAL_B26C4162 = + "Rate-family aggregate function produced a non-finite time interval"; + public static final String + EXCEPTION_RATE_FAMILY_AGGREGATE_FUNCTION_PRODUCED_A_NON_FINITE_EXTRAPOLATION_RESULT_6482CF1D = + "Rate-family aggregate function produced a non-finite extrapolation result"; + public static final String + EXCEPTION_AGGREGATE_FUNCTION_ARG_DOES_NOT_SUPPORT_DUPLICATE_TIME_COL_VALUES_IN_THE_SAME_AGGREGATION_GROUP_ARG_087A91BC = + "Aggregate function [%s] does not support duplicate time_col values in the same aggregation group: %d"; + public static final String + EXCEPTION_AGGREGATE_FUNCTION_ARG_DOES_NOT_SUPPORT_NON_FINITE_VALUE_COL_ARG_AC2AAC62 = + "Aggregate function [%s] does not support non-finite value_col: %s"; + public static final String + EXCEPTION_THE_VALUE_COL_ARGUMENT_OF_AGGREGATE_FUNCTION_ARG_MUST_BE_A_NON_NEGATIVE_NUMBER_BUT_GOT_ARG_4D5B7D74 = + "The value_col argument of aggregate function [%s] must be a non-negative number, but got: %s"; + public static final String + EXCEPTION_THE_ARGUMENT_ARG_OF_AGGREGATE_FUNCTION_ARG_MUST_NOT_BE_NULL_WHEN_VALUE_COL_IS_NOT_NULL_7F087E99 = + "The argument %d of aggregate function [%s] must not be NULL when value_col is not NULL"; + public static final String + EXCEPTION_THE_WINDOW_START_ARGUMENT_OF_AGGREGATE_FUNCTION_ARG_MUST_BE_LESS_THAN_WINDOW_END_17D2A79A = + "The window_start argument of aggregate function [%s] must be less than window_end"; + public static final String + EXCEPTION_THE_SAMPLE_TIME_OF_AGGREGATE_FUNCTION_ARG_MUST_SATISFY_WINDOW_START_TIME_COL_WINDOW_END_35014D15 = + "The sample time of aggregate function [%s] must satisfy window_start <= time_col < window_end"; + public static final String + EXCEPTION_AGGREGATE_FUNCTION_ARG_REQUIRES_CONSISTENT_WINDOW_BOUNDARIES_IN_THE_SAME_AGGREGATION_GROUP_EXPECTED_ARG_ARG_BUT_GOT_ARG_ARG_38631886 = + "Aggregate function [%s] requires consistent window boundaries in the same aggregation group: expected [%d, %d), but got [%d, %d)"; + public static final String + EXCEPTION_AGGREGATE_FUNCTION_ARG_EXPECTED_TIME_COL_IN_STRICTLY_ASCENDING_ORDER_BUT_GOT_ARG_AFTER_ARG_9289E0F9 = + "Aggregate function [%s] expected time_col in strictly ascending order, but got %d after %d"; + public static final String + EXCEPTION_ORDERED_AGGREGATE_FUNCTION_ARG_DOES_NOT_SUPPORT_INTERMEDIATE_STATE_6B4B2B1B = + "Ordered aggregate function [%s] does not support intermediate state"; + public static final String + EXCEPTION_INVALID_INTERMEDIATE_STATE_FOR_AGGREGATE_FUNCTION_ARG_2999C30B = + "Invalid intermediate state for aggregate function [%s]"; + public static final String + EXCEPTION_AGGREGATE_FUNCTION_ARG_DOES_NOT_SUPPORT_VALUE_TYPE_ARG_9DD7388D = + "Aggregate function [%s] does not support value type %s"; + public static final String + EXCEPTION_AGGREGATE_FUNCTION_ARG_REQUIRES_ARG_ARGUMENTS_BUT_GOT_ARG_D6249DD2 = + "Aggregate function [%s] requires %d arguments, but got %d"; } diff --git a/iotdb-core/calc-commons/src/main/i18n/zh/org/apache/iotdb/calc/i18n/CalcMessages.java b/iotdb-core/calc-commons/src/main/i18n/zh/org/apache/iotdb/calc/i18n/CalcMessages.java index 6d8eb2bca0d98..ab341754da0af 100644 --- a/iotdb-core/calc-commons/src/main/i18n/zh/org/apache/iotdb/calc/i18n/CalcMessages.java +++ b/iotdb-core/calc-commons/src/main/i18n/zh/org/apache/iotdb/calc/i18n/CalcMessages.java @@ -445,4 +445,61 @@ private CalcMessages() {} "PERCENTILE 聚合中不支持的数据类型：%s"; public static final String EXCEPTION_PERCENTILEACCUMULATOR_DOES_NOT_SUPPORT_STATISTICS_66308C79 = "PercentileAccumulator 不支持统计信息"; + public static final String + EXCEPTION_RATE_FAMILY_AGGREGATE_FUNCTION_REQUIRES_AT_LEAST_TWO_VALID_SAMPLES_1A89901C = + "rate 系列聚合函数至少需要两个有效样本"; + public static final String + EXCEPTION_THE_FIRST_SAMPLE_TIME_MUST_BE_LESS_THAN_THE_LAST_SAMPLE_TIME_FC4D3517 = + "第一个样本时间必须小于最后一个样本时间"; + public static final String + EXCEPTION_THE_WINDOW_START_MUST_BE_LESS_THAN_THE_WINDOW_END_B1A38C98 = + "window_start 必须小于 window_end"; + public static final String + EXCEPTION_RATE_FAMILY_AGGREGATE_FUNCTION_REQUIRES_WINDOW_START_FIRST_TIME_LAST_TIME_WINDOW_END_60DE93C8 = + "rate 系列聚合函数要求 window_start <= first_time < last_time < window_end"; + public static final String + EXCEPTION_RATE_FAMILY_AGGREGATE_FUNCTION_PRODUCED_A_NON_FINITE_INTERMEDIATE_RESULT_D46B30CD = + "rate 系列聚合函数产生了非有限中间结果"; + public static final String + EXCEPTION_RATE_FAMILY_AGGREGATE_FUNCTION_PRODUCED_A_NON_FINITE_TIME_INTERVAL_B26C4162 = + "rate 系列聚合函数产生了非有限时间间隔"; + public static final String + EXCEPTION_RATE_FAMILY_AGGREGATE_FUNCTION_PRODUCED_A_NON_FINITE_EXTRAPOLATION_RESULT_6482CF1D = + "rate 系列聚合函数产生了非有限外推结果"; + public static final String + EXCEPTION_AGGREGATE_FUNCTION_ARG_DOES_NOT_SUPPORT_DUPLICATE_TIME_COL_VALUES_IN_THE_SAME_AGGREGATION_GROUP_ARG_087A91BC = + "聚合函数 [%s] 不支持同一聚合分组内出现重复的 time_col 值：%d"; + public static final String + EXCEPTION_AGGREGATE_FUNCTION_ARG_DOES_NOT_SUPPORT_NON_FINITE_VALUE_COL_ARG_AC2AAC62 = + "聚合函数 [%s] 不支持非有限 value_col：%s"; + public static final String + EXCEPTION_THE_VALUE_COL_ARGUMENT_OF_AGGREGATE_FUNCTION_ARG_MUST_BE_A_NON_NEGATIVE_NUMBER_BUT_GOT_ARG_4D5B7D74 = + "聚合函数 [%s] 的 value_col 参数必须是非负数，实际为：%s"; + public static final String + EXCEPTION_THE_ARGUMENT_ARG_OF_AGGREGATE_FUNCTION_ARG_MUST_NOT_BE_NULL_WHEN_VALUE_COL_IS_NOT_NULL_7F087E99 = + "第 %d 个参数在聚合函数 [%s] 的 value_col 不为 NULL 时不得为 NULL"; + public static final String + EXCEPTION_THE_WINDOW_START_ARGUMENT_OF_AGGREGATE_FUNCTION_ARG_MUST_BE_LESS_THAN_WINDOW_END_17D2A79A = + "聚合函数 [%s] 的 window_start 参数必须小于 window_end"; + public static final String + EXCEPTION_THE_SAMPLE_TIME_OF_AGGREGATE_FUNCTION_ARG_MUST_SATISFY_WINDOW_START_TIME_COL_WINDOW_END_35014D15 = + "聚合函数 [%s] 的样本时间必须满足 window_start <= time_col < window_end"; + public static final String + EXCEPTION_AGGREGATE_FUNCTION_ARG_REQUIRES_CONSISTENT_WINDOW_BOUNDARIES_IN_THE_SAME_AGGREGATION_GROUP_EXPECTED_ARG_ARG_BUT_GOT_ARG_ARG_38631886 = + "聚合函数 [%s] 要求同一聚合分组内的窗口边界一致：期望 [%d, %d)，实际为 [%d, %d)"; + public static final String + EXCEPTION_AGGREGATE_FUNCTION_ARG_EXPECTED_TIME_COL_IN_STRICTLY_ASCENDING_ORDER_BUT_GOT_ARG_AFTER_ARG_9289E0F9 = + "聚合函数 [%s] 要求 time_col 严格升序，但在 %d 之后得到了 %d"; + public static final String + EXCEPTION_ORDERED_AGGREGATE_FUNCTION_ARG_DOES_NOT_SUPPORT_INTERMEDIATE_STATE_6B4B2B1B = + "有序聚合函数 [%s] 不支持 Intermediate State"; + public static final String + EXCEPTION_INVALID_INTERMEDIATE_STATE_FOR_AGGREGATE_FUNCTION_ARG_2999C30B = + "聚合函数 [%s] 的 Intermediate State 非法"; + public static final String + EXCEPTION_AGGREGATE_FUNCTION_ARG_DOES_NOT_SUPPORT_VALUE_TYPE_ARG_9DD7388D = + "聚合函数 [%s] 不支持值类型 %s"; + public static final String + EXCEPTION_AGGREGATE_FUNCTION_ARG_REQUIRES_ARG_ARGUMENTS_BUT_GOT_ARG_D6249DD2 = + "聚合函数 [%s] 需要 %d 个参数，但实际传入 %d 个"; } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/AccumulatorFactory.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/AccumulatorFactory.java index 73f74bc7c8177..1367df34e9002 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/AccumulatorFactory.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/AccumulatorFactory.java @@ -56,11 +56,28 @@ import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.GroupedVarianceAccumulator; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.IntGroupedApproxMostFrequentAccumulator; import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.LongGroupedApproxMostFrequentAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.rate.GroupedNaiveDeltaAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.rate.GroupedNaiveIncreaseAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.rate.GroupedNaiveIrateAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.rate.GroupedNaiveRateAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.rate.GroupedOrderedDeltaAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.rate.GroupedOrderedIncreaseAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.rate.GroupedOrderedIrateAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.rate.GroupedOrderedRateAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.NaiveDeltaAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.NaiveIncreaseAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.NaiveIrateAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.NaiveRateAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.OrderedDeltaAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.OrderedIncreaseAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.OrderedIrateAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.OrderedRateAccumulator; import org.apache.iotdb.calc.i18n.CalcMessages; import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; import org.apache.iotdb.common.rpc.thrift.TAggregationType; import org.apache.iotdb.commons.queryengine.execution.operator.source.relational.aggregation.grouped.UpdateMemory; import org.apache.iotdb.commons.queryengine.execution.operator.source.relational.aggregation.grouped.hash.MarkDistinctHash; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.AggregationNode; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.commons.queryengine.plan.relational.type.InternalTypeManager; import org.apache.iotdb.commons.queryengine.plan.udf.TableUDFUtils; @@ -142,6 +159,38 @@ public static TableAccumulator createAccumulator( boolean distinct, MemoryReservationManager memoryReservationManager, @Nullable IoTDBLocal ioTDBLocal) { + return createAccumulator( + functionName, + aggregationType, + inputDataTypes, + inputExpressions, + inputAttributes, + ascending, + isAggTableScan, + timeColumnName, + measurementColumnNames, + distinct, + AggregationNode.Step.SINGLE, + false, + memoryReservationManager, + ioTDBLocal); + } + + public static TableAccumulator createAccumulator( + String functionName, + TAggregationType aggregationType, + List inputDataTypes, + List inputExpressions, + Map inputAttributes, + boolean ascending, + boolean isAggTableScan, + String timeColumnName, + Set measurementColumnNames, + boolean distinct, + AggregationNode.Step step, + boolean inputOrderedByTimeAscending, + MemoryReservationManager memoryReservationManager, + @Nullable IoTDBLocal ioTDBLocal) { TableAccumulator result; // Input expression size of 1 indicates aggregation split has occurred and this is a final @@ -201,7 +250,13 @@ public static TableAccumulator createAccumulator( ? new FirstAccumulator(inputDataTypes.get(0), isAggTableScan) : new FirstDescAccumulator(inputDataTypes.get(0)); } else { - result = createBuiltinAccumulator(aggregationType, inputDataTypes, memoryReservationManager); + result = + createBuiltinAccumulator( + aggregationType, + inputDataTypes, + step, + inputOrderedByTimeAscending, + memoryReservationManager); } if (distinct) { @@ -247,6 +302,32 @@ public static GroupedAccumulator createGroupedAccumulator( boolean distinct, MemoryReservationManager memoryReservationManager, @Nullable IoTDBLocal ioTDBLocal) { + return createGroupedAccumulator( + functionName, + aggregationType, + inputDataTypes, + inputExpressions, + inputAttributes, + ascending, + distinct, + AggregationNode.Step.SINGLE, + false, + memoryReservationManager, + ioTDBLocal); + } + + public static GroupedAccumulator createGroupedAccumulator( + String functionName, + TAggregationType aggregationType, + List inputDataTypes, + List inputExpressions, + Map inputAttributes, + boolean ascending, + boolean distinct, + AggregationNode.Step step, + boolean inputOrderedByTimeAscending, + MemoryReservationManager memoryReservationManager, + @Nullable IoTDBLocal ioTDBLocal) { GroupedAccumulator result; if (aggregationType == TAggregationType.UDAF) { @@ -261,6 +342,8 @@ public static GroupedAccumulator createGroupedAccumulator( inputExpressions, inputAttributes, ascending, + step, + inputOrderedByTimeAscending, memoryReservationManager); } @@ -317,7 +400,11 @@ private static GroupedAccumulator createBuiltinGroupedAccumulator( List inputExpressions, Map inputAttributes, boolean ascending, + AggregationNode.Step step, + boolean inputOrderedByTimeAscending, MemoryReservationManager memoryReservationManager) { + boolean useOrderedImplementation = + step == AggregationNode.Step.SINGLE && inputOrderedByTimeAscending; switch (aggregationType) { case COUNT: return new GroupedCountAccumulator(); @@ -403,6 +490,22 @@ private static GroupedAccumulator createBuiltinGroupedAccumulator( inputDataTypes.get(0), CentralMomentAccumulator.MomentType.KURTOSIS); case PERCENTILE: return new GroupedPercentileAccumulator(inputDataTypes.get(0), memoryReservationManager); + case RATE: + return useOrderedImplementation + ? new GroupedOrderedRateAccumulator(inputDataTypes.get(0), memoryReservationManager) + : new GroupedNaiveRateAccumulator(inputDataTypes.get(0), memoryReservationManager); + case INCREASE: + return useOrderedImplementation + ? new GroupedOrderedIncreaseAccumulator(inputDataTypes.get(0), memoryReservationManager) + : new GroupedNaiveIncreaseAccumulator(inputDataTypes.get(0), memoryReservationManager); + case IRATE: + return useOrderedImplementation + ? new GroupedOrderedIrateAccumulator(inputDataTypes.get(0), memoryReservationManager) + : new GroupedNaiveIrateAccumulator(inputDataTypes.get(0), memoryReservationManager); + case DELTA: + return useOrderedImplementation + ? new GroupedOrderedDeltaAccumulator(inputDataTypes.get(0), memoryReservationManager) + : new GroupedNaiveDeltaAccumulator(inputDataTypes.get(0), memoryReservationManager); default: throw new IllegalArgumentException( CalcMessages.INVALID_AGGREGATION_FUNCTION + aggregationType); @@ -413,6 +516,22 @@ public static TableAccumulator createBuiltinAccumulator( TAggregationType aggregationType, List inputDataTypes, MemoryReservationManager memoryReservationManager) { + return createBuiltinAccumulator( + aggregationType, + inputDataTypes, + AggregationNode.Step.SINGLE, + false, + memoryReservationManager); + } + + public static TableAccumulator createBuiltinAccumulator( + TAggregationType aggregationType, + List inputDataTypes, + AggregationNode.Step step, + boolean inputOrderedByTimeAscending, + MemoryReservationManager memoryReservationManager) { + boolean useOrderedImplementation = + step == AggregationNode.Step.SINGLE && inputOrderedByTimeAscending; switch (aggregationType) { case COUNT: return new CountAccumulator(); @@ -499,6 +618,22 @@ public static TableAccumulator createBuiltinAccumulator( inputDataTypes.get(0), CentralMomentAccumulator.MomentType.KURTOSIS); case PERCENTILE: return new PercentileAccumulator(inputDataTypes.get(0), memoryReservationManager); + case RATE: + return useOrderedImplementation + ? new OrderedRateAccumulator(inputDataTypes.get(0)) + : new NaiveRateAccumulator(inputDataTypes.get(0), memoryReservationManager); + case INCREASE: + return useOrderedImplementation + ? new OrderedIncreaseAccumulator(inputDataTypes.get(0)) + : new NaiveIncreaseAccumulator(inputDataTypes.get(0), memoryReservationManager); + case IRATE: + return useOrderedImplementation + ? new OrderedIrateAccumulator(inputDataTypes.get(0)) + : new NaiveIrateAccumulator(inputDataTypes.get(0), memoryReservationManager); + case DELTA: + return useOrderedImplementation + ? new OrderedDeltaAccumulator(inputDataTypes.get(0)) + : new NaiveDeltaAccumulator(inputDataTypes.get(0), memoryReservationManager); default: throw new IllegalArgumentException( CalcMessages.INVALID_AGGREGATION_FUNCTION + aggregationType); @@ -573,6 +708,10 @@ public static boolean isMultiInputAggregation(TAggregationType aggregationType) case COVAR_SAMP: case REGR_SLOPE: case REGR_INTERCEPT: + case RATE: + case INCREASE: + case IRATE: + case DELTA: return true; default: return false; diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/AbstractGroupedDeltaAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/AbstractGroupedDeltaAccumulator.java new file mode 100644 index 0000000000000..5867474875148 --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/AbstractGroupedDeltaAccumulator.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.rate; + +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.GroupedAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.BooleanBigArray; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.LongBigArray; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.ExtrapolationUtil; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.RateFunctionType; +import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.commons.exception.SemanticException; + +import org.apache.tsfile.enums.TSDataType; + +abstract class AbstractGroupedDeltaAccumulator implements GroupedAccumulator { + + protected final TSDataType valueDataType; + private final BooleanBigArray windowInitialized = new BooleanBigArray(); + private final LongBigArray windowStarts = new LongBigArray(); + private final LongBigArray windowEnds = new LongBigArray(); + + protected AbstractGroupedDeltaAccumulator(TSDataType valueDataType) { + this.valueDataType = valueDataType; + } + + protected final void ensureWindowCapacity(long groupCount) { + windowInitialized.ensureCapacity(groupCount); + windowStarts.ensureCapacity(groupCount); + windowEnds.ensureCapacity(groupCount); + } + + protected final void initializeOrValidateWindow( + int groupId, long candidateWindowStart, long candidateWindowEnd) { + if (!windowInitialized.get(groupId)) { + windowInitialized.set(groupId, true); + windowStarts.set(groupId, candidateWindowStart); + windowEnds.set(groupId, candidateWindowEnd); + return; + } + long expectedStart = windowStarts.get(groupId); + long expectedEnd = windowEnds.get(groupId); + if (expectedStart != candidateWindowStart || expectedEnd != candidateWindowEnd) { + throw new SemanticException( + String.format( + CalcMessages + .EXCEPTION_AGGREGATE_FUNCTION_ARG_REQUIRES_CONSISTENT_WINDOW_BOUNDARIES_IN_THE_SAME_AGGREGATION_GROUP_EXPECTED_ARG_ARG_BUT_GOT_ARG_ARG_38631886, + RateFunctionType.DELTA.getFunctionName(), + expectedStart, + expectedEnd, + candidateWindowStart, + candidateWindowEnd)); + } + } + + protected final long windowStart(int groupId) { + return windowStarts.get(groupId); + } + + protected final long windowEnd(int groupId) { + return windowEnds.get(groupId); + } + + protected final long windowsSizeOf() { + return windowInitialized.sizeOf() + windowStarts.sizeOf() + windowEnds.sizeOf(); + } + + protected final void resetWindows() { + windowInitialized.reset(); + windowStarts.reset(); + windowEnds.reset(); + } + + protected final double calculateDelta( + int sampleCount, + long firstTime, + double firstValue, + long lastTime, + double lastValue, + long windowStart, + long windowEnd) { + return ExtrapolationUtil.extrapolate( + sampleCount, + firstTime, + firstValue, + lastTime, + windowStart, + windowEnd, + lastValue - firstValue, + false); + } + + protected final SemanticException duplicateTimestamp(long time) { + return new SemanticException( + String.format( + CalcMessages + .EXCEPTION_AGGREGATE_FUNCTION_ARG_DOES_NOT_SUPPORT_DUPLICATE_TIME_COL_VALUES_IN_THE_SAME_AGGREGATION_GROUP_ARG_087A91BC, + RateFunctionType.DELTA.getFunctionName(), + time)); + } + + protected final SemanticException orderedInputViolation(long time, long previousTime) { + return new SemanticException( + String.format( + CalcMessages + .EXCEPTION_AGGREGATE_FUNCTION_ARG_EXPECTED_TIME_COL_IN_STRICTLY_ASCENDING_ORDER_BUT_GOT_ARG_AFTER_ARG_9289E0F9, + RateFunctionType.DELTA.getFunctionName(), + time, + previousTime)); + } + + protected final UnsupportedOperationException unsupportedIntermediate() { + return new UnsupportedOperationException( + String.format( + CalcMessages + .EXCEPTION_ORDERED_AGGREGATE_FUNCTION_ARG_DOES_NOT_SUPPORT_INTERMEDIATE_STATE_6B4B2B1B, + RateFunctionType.DELTA.getFunctionName())); + } + + @Override + public final void prepareFinal() {} +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/AbstractGroupedIncreaseAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/AbstractGroupedIncreaseAccumulator.java new file mode 100644 index 0000000000000..9d7d81fce1907 --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/AbstractGroupedIncreaseAccumulator.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.rate; + +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.GroupedAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.BooleanBigArray; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.LongBigArray; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.ExtrapolationUtil; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.RateFunctionType; +import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.commons.exception.SemanticException; + +import org.apache.tsfile.enums.TSDataType; + +abstract class AbstractGroupedIncreaseAccumulator implements GroupedAccumulator { + + protected final TSDataType valueDataType; + private final BooleanBigArray windowInitialized = new BooleanBigArray(); + private final LongBigArray windowStarts = new LongBigArray(); + private final LongBigArray windowEnds = new LongBigArray(); + + protected AbstractGroupedIncreaseAccumulator(TSDataType valueDataType) { + this.valueDataType = valueDataType; + } + + protected final void ensureWindowCapacity(long groupCount) { + windowInitialized.ensureCapacity(groupCount); + windowStarts.ensureCapacity(groupCount); + windowEnds.ensureCapacity(groupCount); + } + + protected final void initializeOrValidateWindow( + int groupId, long candidateWindowStart, long candidateWindowEnd) { + if (!windowInitialized.get(groupId)) { + windowInitialized.set(groupId, true); + windowStarts.set(groupId, candidateWindowStart); + windowEnds.set(groupId, candidateWindowEnd); + return; + } + long expectedStart = windowStarts.get(groupId); + long expectedEnd = windowEnds.get(groupId); + if (expectedStart != candidateWindowStart || expectedEnd != candidateWindowEnd) { + throw new SemanticException( + String.format( + CalcMessages + .EXCEPTION_AGGREGATE_FUNCTION_ARG_REQUIRES_CONSISTENT_WINDOW_BOUNDARIES_IN_THE_SAME_AGGREGATION_GROUP_EXPECTED_ARG_ARG_BUT_GOT_ARG_ARG_38631886, + RateFunctionType.INCREASE.getFunctionName(), + expectedStart, + expectedEnd, + candidateWindowStart, + candidateWindowEnd)); + } + } + + protected final long windowStart(int groupId) { + return windowStarts.get(groupId); + } + + protected final long windowEnd(int groupId) { + return windowEnds.get(groupId); + } + + protected final long windowsSizeOf() { + return windowInitialized.sizeOf() + windowStarts.sizeOf() + windowEnds.sizeOf(); + } + + protected final void resetWindows() { + windowInitialized.reset(); + windowStarts.reset(); + windowEnds.reset(); + } + + protected final double calculateIncrease( + int sampleCount, + long firstTime, + double firstValue, + long lastTime, + double correctedIncrease, + long windowStart, + long windowEnd) { + return ExtrapolationUtil.extrapolate( + sampleCount, + firstTime, + firstValue, + lastTime, + windowStart, + windowEnd, + correctedIncrease, + true); + } + + protected final SemanticException duplicateTimestamp(long time) { + return new SemanticException( + String.format( + CalcMessages + .EXCEPTION_AGGREGATE_FUNCTION_ARG_DOES_NOT_SUPPORT_DUPLICATE_TIME_COL_VALUES_IN_THE_SAME_AGGREGATION_GROUP_ARG_087A91BC, + RateFunctionType.INCREASE.getFunctionName(), + time)); + } + + protected final SemanticException orderedInputViolation(long time, long previousTime) { + return new SemanticException( + String.format( + CalcMessages + .EXCEPTION_AGGREGATE_FUNCTION_ARG_EXPECTED_TIME_COL_IN_STRICTLY_ASCENDING_ORDER_BUT_GOT_ARG_AFTER_ARG_9289E0F9, + RateFunctionType.INCREASE.getFunctionName(), + time, + previousTime)); + } + + protected final UnsupportedOperationException unsupportedIntermediate() { + return new UnsupportedOperationException( + String.format( + CalcMessages + .EXCEPTION_ORDERED_AGGREGATE_FUNCTION_ARG_DOES_NOT_SUPPORT_INTERMEDIATE_STATE_6B4B2B1B, + RateFunctionType.INCREASE.getFunctionName())); + } + + protected final double validateFinite(double value) { + if (!Double.isFinite(value)) { + throw new SemanticException( + CalcMessages + .EXCEPTION_RATE_FAMILY_AGGREGATE_FUNCTION_PRODUCED_A_NON_FINITE_INTERMEDIATE_RESULT_D46B30CD); + } + return value; + } + + @Override + public final void prepareFinal() {} +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/AbstractGroupedIrateAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/AbstractGroupedIrateAccumulator.java new file mode 100644 index 0000000000000..2bb655679bfb5 --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/AbstractGroupedIrateAccumulator.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.rate; + +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.GroupedAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.ExtrapolationUtil; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.RateFunctionType; +import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.commons.exception.SemanticException; + +import org.apache.tsfile.enums.TSDataType; + +abstract class AbstractGroupedIrateAccumulator implements GroupedAccumulator { + + protected final TSDataType valueDataType; + + protected AbstractGroupedIrateAccumulator(TSDataType valueDataType) { + this.valueDataType = valueDataType; + } + + protected final double calculateIrate( + long previousTime, double previousValue, long lastTime, double lastValue) { + double increase = lastValue >= previousValue ? lastValue - previousValue : lastValue; + return validateFinite( + increase / ExtrapolationUtil.timestampDiffToSeconds(lastTime, previousTime)); + } + + protected final SemanticException duplicateTimestamp(long time) { + return new SemanticException( + String.format( + CalcMessages + .EXCEPTION_AGGREGATE_FUNCTION_ARG_DOES_NOT_SUPPORT_DUPLICATE_TIME_COL_VALUES_IN_THE_SAME_AGGREGATION_GROUP_ARG_087A91BC, + RateFunctionType.IRATE.getFunctionName(), + time)); + } + + protected final SemanticException orderedInputViolation(long time, long previousTime) { + return new SemanticException( + String.format( + CalcMessages + .EXCEPTION_AGGREGATE_FUNCTION_ARG_EXPECTED_TIME_COL_IN_STRICTLY_ASCENDING_ORDER_BUT_GOT_ARG_AFTER_ARG_9289E0F9, + RateFunctionType.IRATE.getFunctionName(), + time, + previousTime)); + } + + protected final UnsupportedOperationException unsupportedIntermediate() { + return new UnsupportedOperationException( + String.format( + CalcMessages + .EXCEPTION_ORDERED_AGGREGATE_FUNCTION_ARG_DOES_NOT_SUPPORT_INTERMEDIATE_STATE_6B4B2B1B, + RateFunctionType.IRATE.getFunctionName())); + } + + protected final double validateFinite(double value) { + if (!Double.isFinite(value)) { + throw new SemanticException( + CalcMessages + .EXCEPTION_RATE_FAMILY_AGGREGATE_FUNCTION_PRODUCED_A_NON_FINITE_INTERMEDIATE_RESULT_D46B30CD); + } + return value; + } + + @Override + public final void prepareFinal() {} +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/AbstractGroupedRateAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/AbstractGroupedRateAccumulator.java new file mode 100644 index 0000000000000..35379363b83b7 --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/AbstractGroupedRateAccumulator.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.rate; + +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.GroupedAccumulator; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.BooleanBigArray; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.LongBigArray; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.ExtrapolationUtil; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.RateFunctionType; +import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.commons.exception.SemanticException; + +import org.apache.tsfile.enums.TSDataType; + +abstract class AbstractGroupedRateAccumulator implements GroupedAccumulator { + + protected final TSDataType valueDataType; + private final BooleanBigArray windowInitialized = new BooleanBigArray(); + private final LongBigArray windowStarts = new LongBigArray(); + private final LongBigArray windowEnds = new LongBigArray(); + + protected AbstractGroupedRateAccumulator(TSDataType valueDataType) { + this.valueDataType = valueDataType; + } + + protected final void ensureWindowCapacity(long groupCount) { + windowInitialized.ensureCapacity(groupCount); + windowStarts.ensureCapacity(groupCount); + windowEnds.ensureCapacity(groupCount); + } + + protected final void initializeOrValidateWindow( + int groupId, long candidateWindowStart, long candidateWindowEnd) { + if (!windowInitialized.get(groupId)) { + windowInitialized.set(groupId, true); + windowStarts.set(groupId, candidateWindowStart); + windowEnds.set(groupId, candidateWindowEnd); + return; + } + long expectedStart = windowStarts.get(groupId); + long expectedEnd = windowEnds.get(groupId); + if (expectedStart != candidateWindowStart || expectedEnd != candidateWindowEnd) { + throw new SemanticException( + String.format( + CalcMessages + .EXCEPTION_AGGREGATE_FUNCTION_ARG_REQUIRES_CONSISTENT_WINDOW_BOUNDARIES_IN_THE_SAME_AGGREGATION_GROUP_EXPECTED_ARG_ARG_BUT_GOT_ARG_ARG_38631886, + RateFunctionType.RATE.getFunctionName(), + expectedStart, + expectedEnd, + candidateWindowStart, + candidateWindowEnd)); + } + } + + protected final long windowStart(int groupId) { + return windowStarts.get(groupId); + } + + protected final long windowEnd(int groupId) { + return windowEnds.get(groupId); + } + + protected final long windowsSizeOf() { + return windowInitialized.sizeOf() + windowStarts.sizeOf() + windowEnds.sizeOf(); + } + + protected final void resetWindows() { + windowInitialized.reset(); + windowStarts.reset(); + windowEnds.reset(); + } + + protected final double calculateRate( + int sampleCount, + long firstTime, + double firstValue, + long lastTime, + double correctedIncrease, + long windowStart, + long windowEnd) { + double extrapolated = + ExtrapolationUtil.extrapolate( + sampleCount, + firstTime, + firstValue, + lastTime, + windowStart, + windowEnd, + correctedIncrease, + true); + return validateFinite( + extrapolated / ExtrapolationUtil.timestampDiffToSeconds(windowEnd, windowStart)); + } + + protected final SemanticException duplicateTimestamp(long time) { + return new SemanticException( + String.format( + CalcMessages + .EXCEPTION_AGGREGATE_FUNCTION_ARG_DOES_NOT_SUPPORT_DUPLICATE_TIME_COL_VALUES_IN_THE_SAME_AGGREGATION_GROUP_ARG_087A91BC, + RateFunctionType.RATE.getFunctionName(), + time)); + } + + protected final SemanticException orderedInputViolation(long time, long previousTime) { + return new SemanticException( + String.format( + CalcMessages + .EXCEPTION_AGGREGATE_FUNCTION_ARG_EXPECTED_TIME_COL_IN_STRICTLY_ASCENDING_ORDER_BUT_GOT_ARG_AFTER_ARG_9289E0F9, + RateFunctionType.RATE.getFunctionName(), + time, + previousTime)); + } + + protected final UnsupportedOperationException unsupportedIntermediate() { + return new UnsupportedOperationException( + String.format( + CalcMessages + .EXCEPTION_ORDERED_AGGREGATE_FUNCTION_ARG_DOES_NOT_SUPPORT_INTERMEDIATE_STATE_6B4B2B1B, + RateFunctionType.RATE.getFunctionName())); + } + + protected final double validateFinite(double value) { + if (!Double.isFinite(value)) { + throw new SemanticException( + CalcMessages + .EXCEPTION_RATE_FAMILY_AGGREGATE_FUNCTION_PRODUCED_A_NON_FINITE_INTERMEDIATE_RESULT_D46B30CD); + } + return value; + } + + @Override + public final void prepareFinal() {} +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/GroupedNaiveDeltaAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/GroupedNaiveDeltaAccumulator.java new file mode 100644 index 0000000000000..e52b1f71a2c92 --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/GroupedNaiveDeltaAccumulator.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.rate; + +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.AggregationMask; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.RateFunctionIntermediateStateCodec; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.RateFunctionType; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.RateFunctionValidation; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.TimeValueBuffer; +import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.RamUsageEstimator; + +public final class GroupedNaiveDeltaAccumulator extends AbstractGroupedDeltaAccumulator { + + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(GroupedNaiveDeltaAccumulator.class); + + private final TimeValueBufferBigArray samples = new TimeValueBufferBigArray(); + private final MemoryReservationManager memoryReservationManager; + private long previousSize; + + public GroupedNaiveDeltaAccumulator( + TSDataType valueDataType, MemoryReservationManager memoryReservationManager) { + super(valueDataType); + this.memoryReservationManager = memoryReservationManager; + updateMemoryReservation(); + } + + @Override + public long getEstimatedSize() { + return INSTANCE_SIZE + windowsSizeOf() + samples.sizeOf(); + } + + @Override + public void setGroupCount(long groupCount) { + ensureWindowCapacity(groupCount); + samples.ensureCapacity(groupCount); + updateMemoryReservation(); + } + + @Override + public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { + RateFunctionValidation.validateArgumentCount(arguments, RateFunctionType.DELTA); + int selectedCount = mask.getSelectedPositionCount(); + int[] selectedPositions = mask.isSelectAll() ? null : mask.getSelectedPositions(); + for (int index = 0; index < selectedCount; index++) { + int position = mask.isSelectAll() ? index : selectedPositions[index]; + if (arguments[0].isNull(position)) { + continue; + } + int groupId = groupIds[position]; + double value = + RateFunctionValidation.readValue( + arguments[0], position, valueDataType, RateFunctionType.DELTA); + long time = + RateFunctionValidation.readRequiredTime( + arguments[1], position, RateFunctionType.DELTA, 2); + long currentWindowStart = + RateFunctionValidation.readRequiredTime( + arguments[2], position, RateFunctionType.DELTA, 3); + long currentWindowEnd = + RateFunctionValidation.readRequiredTime( + arguments[3], position, RateFunctionType.DELTA, 4); + RateFunctionValidation.validateWindow( + RateFunctionType.DELTA, time, currentWindowStart, currentWindowEnd); + initializeOrValidateWindow(groupId, currentWindowStart, currentWindowEnd); + samples.add(groupId, time, value); + } + updateMemoryReservation(); + } + + @Override + public void addIntermediate(int[] groupIds, Column argument) { + for (int position = 0; position < groupIds.length; position++) { + if (argument.isNull(position)) { + continue; + } + int groupId = groupIds[position]; + RateFunctionIntermediateStateCodec.DecodedState decoded = + RateFunctionIntermediateStateCodec.decode( + RateFunctionType.DELTA, argument.getBinary(position)); + initializeOrValidateWindow(groupId, decoded.getWindowStart(), decoded.getWindowEnd()); + samples.merge(groupId, decoded.getSamples()); + } + updateMemoryReservation(); + } + + @Override + public void evaluateIntermediate(int groupId, ColumnBuilder output) { + RateFunctionIntermediateStateCodec.encode( + RateFunctionType.DELTA, + windowStart(groupId), + windowEnd(groupId), + samples.get(groupId), + output); + } + + @Override + public void evaluateFinal(int groupId, ColumnBuilder output) { + TimeValueBuffer buffer = samples.get(groupId); + if (buffer == null || buffer.size() < 2) { + output.appendNull(); + return; + } + buffer.sortAndValidate(RateFunctionType.DELTA.getFunctionName()); + int lastIndex = buffer.size() - 1; + output.writeDouble( + calculateDelta( + buffer.size(), + buffer.getTime(0), + buffer.getValue(0), + buffer.getTime(lastIndex), + buffer.getValue(lastIndex), + windowStart(groupId), + windowEnd(groupId))); + } + + @Override + public void reset() { + resetWindows(); + samples.reset(); + updateMemoryReservation(); + } + + private void updateMemoryReservation() { + long currentSize = getEstimatedSize(); + long delta = currentSize - previousSize; + if (delta > 0) { + memoryReservationManager.reserveMemoryCumulatively(delta); + } else if (delta < 0) { + memoryReservationManager.releaseMemoryCumulatively(-delta); + } + previousSize = currentSize; + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/GroupedNaiveIncreaseAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/GroupedNaiveIncreaseAccumulator.java new file mode 100644 index 0000000000000..f2cdd5ae46074 --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/GroupedNaiveIncreaseAccumulator.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.rate; + +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.AggregationMask; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.RateFunctionIntermediateStateCodec; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.RateFunctionType; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.RateFunctionValidation; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.TimeValueBuffer; +import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.RamUsageEstimator; + +public final class GroupedNaiveIncreaseAccumulator extends AbstractGroupedIncreaseAccumulator { + + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(GroupedNaiveIncreaseAccumulator.class); + + private final TimeValueBufferBigArray samples = new TimeValueBufferBigArray(); + private final MemoryReservationManager memoryReservationManager; + private long previousSize; + + public GroupedNaiveIncreaseAccumulator( + TSDataType valueDataType, MemoryReservationManager memoryReservationManager) { + super(valueDataType); + this.memoryReservationManager = memoryReservationManager; + updateMemoryReservation(); + } + + @Override + public long getEstimatedSize() { + return INSTANCE_SIZE + windowsSizeOf() + samples.sizeOf(); + } + + @Override + public void setGroupCount(long groupCount) { + ensureWindowCapacity(groupCount); + samples.ensureCapacity(groupCount); + updateMemoryReservation(); + } + + @Override + public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { + RateFunctionValidation.validateArgumentCount(arguments, RateFunctionType.INCREASE); + int selectedCount = mask.getSelectedPositionCount(); + int[] selectedPositions = mask.isSelectAll() ? null : mask.getSelectedPositions(); + for (int index = 0; index < selectedCount; index++) { + int position = mask.isSelectAll() ? index : selectedPositions[index]; + if (arguments[0].isNull(position)) { + continue; + } + int groupId = groupIds[position]; + double value = + RateFunctionValidation.readValue( + arguments[0], position, valueDataType, RateFunctionType.INCREASE); + long time = + RateFunctionValidation.readRequiredTime( + arguments[1], position, RateFunctionType.INCREASE, 2); + long currentWindowStart = + RateFunctionValidation.readRequiredTime( + arguments[2], position, RateFunctionType.INCREASE, 3); + long currentWindowEnd = + RateFunctionValidation.readRequiredTime( + arguments[3], position, RateFunctionType.INCREASE, 4); + RateFunctionValidation.validateWindow( + RateFunctionType.INCREASE, time, currentWindowStart, currentWindowEnd); + initializeOrValidateWindow(groupId, currentWindowStart, currentWindowEnd); + samples.add(groupId, time, value); + } + updateMemoryReservation(); + } + + @Override + public void addIntermediate(int[] groupIds, Column argument) { + for (int position = 0; position < groupIds.length; position++) { + if (argument.isNull(position)) { + continue; + } + int groupId = groupIds[position]; + RateFunctionIntermediateStateCodec.DecodedState decoded = + RateFunctionIntermediateStateCodec.decode( + RateFunctionType.INCREASE, argument.getBinary(position)); + initializeOrValidateWindow(groupId, decoded.getWindowStart(), decoded.getWindowEnd()); + samples.merge(groupId, decoded.getSamples()); + } + updateMemoryReservation(); + } + + @Override + public void evaluateIntermediate(int groupId, ColumnBuilder output) { + RateFunctionIntermediateStateCodec.encode( + RateFunctionType.INCREASE, + windowStart(groupId), + windowEnd(groupId), + samples.get(groupId), + output); + } + + @Override + public void evaluateFinal(int groupId, ColumnBuilder output) { + TimeValueBuffer buffer = samples.get(groupId); + if (buffer == null || buffer.size() < 2) { + output.appendNull(); + return; + } + buffer.sortAndValidate(RateFunctionType.INCREASE.getFunctionName()); + int lastIndex = buffer.size() - 1; + output.writeDouble( + calculateIncrease( + buffer.size(), + buffer.getTime(0), + buffer.getValue(0), + buffer.getTime(lastIndex), + calculateCorrectedIncrease(buffer), + windowStart(groupId), + windowEnd(groupId))); + } + + @Override + public void reset() { + resetWindows(); + samples.reset(); + updateMemoryReservation(); + } + + private double calculateCorrectedIncrease(TimeValueBuffer buffer) { + double result = 0.0; + for (int index = 1; index < buffer.size(); index++) { + double previous = buffer.getValue(index - 1); + double current = buffer.getValue(index); + result = validateFinite(result + (current >= previous ? current - previous : current)); + } + return result; + } + + private void updateMemoryReservation() { + long currentSize = getEstimatedSize(); + long delta = currentSize - previousSize; + if (delta > 0) { + memoryReservationManager.reserveMemoryCumulatively(delta); + } else if (delta < 0) { + memoryReservationManager.releaseMemoryCumulatively(-delta); + } + previousSize = currentSize; + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/GroupedNaiveIrateAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/GroupedNaiveIrateAccumulator.java new file mode 100644 index 0000000000000..24126374b7282 --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/GroupedNaiveIrateAccumulator.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.rate; + +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.AggregationMask; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.RateFunctionIntermediateStateCodec; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.RateFunctionType; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.RateFunctionValidation; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.TimeValueBuffer; +import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.RamUsageEstimator; + +public final class GroupedNaiveIrateAccumulator extends AbstractGroupedIrateAccumulator { + + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(GroupedNaiveIrateAccumulator.class); + + private final TimeValueBufferBigArray samples = new TimeValueBufferBigArray(); + private final MemoryReservationManager memoryReservationManager; + private long previousSize; + + public GroupedNaiveIrateAccumulator( + TSDataType valueDataType, MemoryReservationManager memoryReservationManager) { + super(valueDataType); + this.memoryReservationManager = memoryReservationManager; + updateMemoryReservation(); + } + + @Override + public long getEstimatedSize() { + return INSTANCE_SIZE + samples.sizeOf(); + } + + @Override + public void setGroupCount(long groupCount) { + samples.ensureCapacity(groupCount); + updateMemoryReservation(); + } + + @Override + public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { + RateFunctionValidation.validateArgumentCount(arguments, RateFunctionType.IRATE); + int selectedCount = mask.getSelectedPositionCount(); + int[] selectedPositions = mask.isSelectAll() ? null : mask.getSelectedPositions(); + for (int index = 0; index < selectedCount; index++) { + int position = mask.isSelectAll() ? index : selectedPositions[index]; + if (arguments[0].isNull(position)) { + continue; + } + int groupId = groupIds[position]; + double value = + RateFunctionValidation.readValue( + arguments[0], position, valueDataType, RateFunctionType.IRATE); + long time = + RateFunctionValidation.readRequiredTime( + arguments[1], position, RateFunctionType.IRATE, 2); + samples.add(groupId, time, value); + } + updateMemoryReservation(); + } + + @Override + public void addIntermediate(int[] groupIds, Column argument) { + for (int position = 0; position < groupIds.length; position++) { + if (argument.isNull(position)) { + continue; + } + RateFunctionIntermediateStateCodec.DecodedState decoded = + RateFunctionIntermediateStateCodec.decode( + RateFunctionType.IRATE, argument.getBinary(position)); + samples.merge(groupIds[position], decoded.getSamples()); + } + updateMemoryReservation(); + } + + @Override + public void evaluateIntermediate(int groupId, ColumnBuilder output) { + RateFunctionIntermediateStateCodec.encode( + RateFunctionType.IRATE, 0, 0, samples.get(groupId), output); + } + + @Override + public void evaluateFinal(int groupId, ColumnBuilder output) { + TimeValueBuffer buffer = samples.get(groupId); + if (buffer == null || buffer.size() < 2) { + output.appendNull(); + return; + } + buffer.sortAndValidate(RateFunctionType.IRATE.getFunctionName()); + int lastIndex = buffer.size() - 1; + output.writeDouble( + calculateIrate( + buffer.getTime(lastIndex - 1), + buffer.getValue(lastIndex - 1), + buffer.getTime(lastIndex), + buffer.getValue(lastIndex))); + } + + @Override + public void reset() { + samples.reset(); + updateMemoryReservation(); + } + + private void updateMemoryReservation() { + long currentSize = getEstimatedSize(); + long delta = currentSize - previousSize; + if (delta > 0) { + memoryReservationManager.reserveMemoryCumulatively(delta); + } else if (delta < 0) { + memoryReservationManager.releaseMemoryCumulatively(-delta); + } + previousSize = currentSize; + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/GroupedNaiveRateAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/GroupedNaiveRateAccumulator.java new file mode 100644 index 0000000000000..8c282034c455a --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/GroupedNaiveRateAccumulator.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.rate; + +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.AggregationMask; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.RateFunctionIntermediateStateCodec; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.RateFunctionType; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.RateFunctionValidation; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.TimeValueBuffer; +import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.RamUsageEstimator; + +public final class GroupedNaiveRateAccumulator extends AbstractGroupedRateAccumulator { + + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(GroupedNaiveRateAccumulator.class); + + private final TimeValueBufferBigArray samples = new TimeValueBufferBigArray(); + private final MemoryReservationManager memoryReservationManager; + private long previousSize; + + public GroupedNaiveRateAccumulator( + TSDataType valueDataType, MemoryReservationManager memoryReservationManager) { + super(valueDataType); + this.memoryReservationManager = memoryReservationManager; + updateMemoryReservation(); + } + + @Override + public long getEstimatedSize() { + return INSTANCE_SIZE + windowsSizeOf() + samples.sizeOf(); + } + + @Override + public void setGroupCount(long groupCount) { + ensureWindowCapacity(groupCount); + samples.ensureCapacity(groupCount); + updateMemoryReservation(); + } + + @Override + public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { + RateFunctionValidation.validateArgumentCount(arguments, RateFunctionType.RATE); + int selectedCount = mask.getSelectedPositionCount(); + int[] selectedPositions = mask.isSelectAll() ? null : mask.getSelectedPositions(); + for (int index = 0; index < selectedCount; index++) { + int position = mask.isSelectAll() ? index : selectedPositions[index]; + if (arguments[0].isNull(position)) { + continue; + } + int groupId = groupIds[position]; + double value = + RateFunctionValidation.readValue( + arguments[0], position, valueDataType, RateFunctionType.RATE); + long time = + RateFunctionValidation.readRequiredTime(arguments[1], position, RateFunctionType.RATE, 2); + long currentWindowStart = + RateFunctionValidation.readRequiredTime(arguments[2], position, RateFunctionType.RATE, 3); + long currentWindowEnd = + RateFunctionValidation.readRequiredTime(arguments[3], position, RateFunctionType.RATE, 4); + RateFunctionValidation.validateWindow( + RateFunctionType.RATE, time, currentWindowStart, currentWindowEnd); + initializeOrValidateWindow(groupId, currentWindowStart, currentWindowEnd); + samples.add(groupId, time, value); + } + updateMemoryReservation(); + } + + @Override + public void addIntermediate(int[] groupIds, Column argument) { + for (int position = 0; position < groupIds.length; position++) { + if (argument.isNull(position)) { + continue; + } + int groupId = groupIds[position]; + RateFunctionIntermediateStateCodec.DecodedState decoded = + RateFunctionIntermediateStateCodec.decode( + RateFunctionType.RATE, argument.getBinary(position)); + initializeOrValidateWindow(groupId, decoded.getWindowStart(), decoded.getWindowEnd()); + samples.merge(groupId, decoded.getSamples()); + } + updateMemoryReservation(); + } + + @Override + public void evaluateIntermediate(int groupId, ColumnBuilder output) { + RateFunctionIntermediateStateCodec.encode( + RateFunctionType.RATE, + windowStart(groupId), + windowEnd(groupId), + samples.get(groupId), + output); + } + + @Override + public void evaluateFinal(int groupId, ColumnBuilder output) { + TimeValueBuffer buffer = samples.get(groupId); + if (buffer == null || buffer.size() < 2) { + output.appendNull(); + return; + } + buffer.sortAndValidate(RateFunctionType.RATE.getFunctionName()); + int lastIndex = buffer.size() - 1; + output.writeDouble( + calculateRate( + buffer.size(), + buffer.getTime(0), + buffer.getValue(0), + buffer.getTime(lastIndex), + calculateCorrectedIncrease(buffer), + windowStart(groupId), + windowEnd(groupId))); + } + + @Override + public void reset() { + resetWindows(); + samples.reset(); + updateMemoryReservation(); + } + + private double calculateCorrectedIncrease(TimeValueBuffer buffer) { + double result = 0.0; + for (int index = 1; index < buffer.size(); index++) { + double previous = buffer.getValue(index - 1); + double current = buffer.getValue(index); + result = validateFinite(result + (current >= previous ? current - previous : current)); + } + return result; + } + + private void updateMemoryReservation() { + long currentSize = getEstimatedSize(); + long delta = currentSize - previousSize; + if (delta > 0) { + memoryReservationManager.reserveMemoryCumulatively(delta); + } else if (delta < 0) { + memoryReservationManager.releaseMemoryCumulatively(-delta); + } + previousSize = currentSize; + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/GroupedOrderedDeltaAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/GroupedOrderedDeltaAccumulator.java new file mode 100644 index 0000000000000..2923fb7741b5e --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/GroupedOrderedDeltaAccumulator.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.rate; + +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.AggregationMask; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.DoubleBigArray; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.IntBigArray; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.LongBigArray; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.RateFunctionType; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.RateFunctionValidation; +import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.RamUsageEstimator; + +public final class GroupedOrderedDeltaAccumulator extends AbstractGroupedDeltaAccumulator { + + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(GroupedOrderedDeltaAccumulator.class); + + private final IntBigArray sampleCounts = new IntBigArray(); + private final LongBigArray firstTimes = new LongBigArray(); + private final DoubleBigArray firstValues = new DoubleBigArray(); + private final LongBigArray lastTimes = new LongBigArray(); + private final DoubleBigArray lastValues = new DoubleBigArray(); + private final MemoryReservationManager memoryReservationManager; + private long previousSize; + + public GroupedOrderedDeltaAccumulator( + TSDataType valueDataType, MemoryReservationManager memoryReservationManager) { + super(valueDataType); + this.memoryReservationManager = memoryReservationManager; + updateMemoryReservation(); + } + + @Override + public long getEstimatedSize() { + return INSTANCE_SIZE + + windowsSizeOf() + + sampleCounts.sizeOf() + + firstTimes.sizeOf() + + firstValues.sizeOf() + + lastTimes.sizeOf() + + lastValues.sizeOf(); + } + + @Override + public void setGroupCount(long groupCount) { + ensureWindowCapacity(groupCount); + sampleCounts.ensureCapacity(groupCount); + firstTimes.ensureCapacity(groupCount); + firstValues.ensureCapacity(groupCount); + lastTimes.ensureCapacity(groupCount); + lastValues.ensureCapacity(groupCount); + updateMemoryReservation(); + } + + @Override + public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { + RateFunctionValidation.validateArgumentCount(arguments, RateFunctionType.DELTA); + int selectedCount = mask.getSelectedPositionCount(); + int[] selectedPositions = mask.isSelectAll() ? null : mask.getSelectedPositions(); + for (int index = 0; index < selectedCount; index++) { + int position = mask.isSelectAll() ? index : selectedPositions[index]; + if (arguments[0].isNull(position)) { + continue; + } + int groupId = groupIds[position]; + double value = + RateFunctionValidation.readValue( + arguments[0], position, valueDataType, RateFunctionType.DELTA); + long time = + RateFunctionValidation.readRequiredTime( + arguments[1], position, RateFunctionType.DELTA, 2); + long currentWindowStart = + RateFunctionValidation.readRequiredTime( + arguments[2], position, RateFunctionType.DELTA, 3); + long currentWindowEnd = + RateFunctionValidation.readRequiredTime( + arguments[3], position, RateFunctionType.DELTA, 4); + RateFunctionValidation.validateWindow( + RateFunctionType.DELTA, time, currentWindowStart, currentWindowEnd); + initializeOrValidateWindow(groupId, currentWindowStart, currentWindowEnd); + update(groupId, time, value); + } + } + + @Override + public void addIntermediate(int[] groupIds, Column argument) { + throw unsupportedIntermediate(); + } + + @Override + public void evaluateIntermediate(int groupId, ColumnBuilder output) { + throw unsupportedIntermediate(); + } + + @Override + public void evaluateFinal(int groupId, ColumnBuilder output) { + int sampleCount = sampleCounts.get(groupId); + if (sampleCount < 2) { + output.appendNull(); + return; + } + output.writeDouble( + calculateDelta( + sampleCount, + firstTimes.get(groupId), + firstValues.get(groupId), + lastTimes.get(groupId), + lastValues.get(groupId), + windowStart(groupId), + windowEnd(groupId))); + } + + @Override + public void reset() { + resetWindows(); + sampleCounts.reset(); + firstTimes.reset(); + firstValues.reset(); + lastTimes.reset(); + lastValues.reset(); + updateMemoryReservation(); + } + + private void update(int groupId, long time, double value) { + int sampleCount = sampleCounts.get(groupId); + if (sampleCount == 0) { + sampleCounts.set(groupId, 1); + firstTimes.set(groupId, time); + firstValues.set(groupId, value); + lastTimes.set(groupId, time); + lastValues.set(groupId, value); + return; + } + long lastTime = lastTimes.get(groupId); + if (time == lastTime) { + throw duplicateTimestamp(time); + } + if (time < lastTime) { + throw orderedInputViolation(time, lastTime); + } + lastTimes.set(groupId, time); + lastValues.set(groupId, value); + sampleCounts.set(groupId, Math.incrementExact(sampleCount)); + } + + private void updateMemoryReservation() { + long currentSize = getEstimatedSize(); + long delta = currentSize - previousSize; + if (delta > 0) { + memoryReservationManager.reserveMemoryCumulatively(delta); + } else if (delta < 0) { + memoryReservationManager.releaseMemoryCumulatively(-delta); + } + previousSize = currentSize; + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/GroupedOrderedIncreaseAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/GroupedOrderedIncreaseAccumulator.java new file mode 100644 index 0000000000000..ebe29b6040b4e --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/GroupedOrderedIncreaseAccumulator.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.rate; + +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.AggregationMask; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.DoubleBigArray; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.IntBigArray; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.LongBigArray; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.RateFunctionType; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.RateFunctionValidation; +import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.RamUsageEstimator; + +public final class GroupedOrderedIncreaseAccumulator extends AbstractGroupedIncreaseAccumulator { + + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(GroupedOrderedIncreaseAccumulator.class); + + private final IntBigArray sampleCounts = new IntBigArray(); + private final LongBigArray firstTimes = new LongBigArray(); + private final DoubleBigArray firstValues = new DoubleBigArray(); + private final LongBigArray lastTimes = new LongBigArray(); + private final DoubleBigArray lastValues = new DoubleBigArray(); + private final DoubleBigArray correctedIncreases = new DoubleBigArray(); + private final MemoryReservationManager memoryReservationManager; + private long previousSize; + + public GroupedOrderedIncreaseAccumulator( + TSDataType valueDataType, MemoryReservationManager memoryReservationManager) { + super(valueDataType); + this.memoryReservationManager = memoryReservationManager; + updateMemoryReservation(); + } + + @Override + public long getEstimatedSize() { + return INSTANCE_SIZE + + windowsSizeOf() + + sampleCounts.sizeOf() + + firstTimes.sizeOf() + + firstValues.sizeOf() + + lastTimes.sizeOf() + + lastValues.sizeOf() + + correctedIncreases.sizeOf(); + } + + @Override + public void setGroupCount(long groupCount) { + ensureWindowCapacity(groupCount); + sampleCounts.ensureCapacity(groupCount); + firstTimes.ensureCapacity(groupCount); + firstValues.ensureCapacity(groupCount); + lastTimes.ensureCapacity(groupCount); + lastValues.ensureCapacity(groupCount); + correctedIncreases.ensureCapacity(groupCount); + updateMemoryReservation(); + } + + @Override + public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { + RateFunctionValidation.validateArgumentCount(arguments, RateFunctionType.INCREASE); + int selectedCount = mask.getSelectedPositionCount(); + int[] selectedPositions = mask.isSelectAll() ? null : mask.getSelectedPositions(); + for (int index = 0; index < selectedCount; index++) { + int position = mask.isSelectAll() ? index : selectedPositions[index]; + if (arguments[0].isNull(position)) { + continue; + } + int groupId = groupIds[position]; + double value = + RateFunctionValidation.readValue( + arguments[0], position, valueDataType, RateFunctionType.INCREASE); + long time = + RateFunctionValidation.readRequiredTime( + arguments[1], position, RateFunctionType.INCREASE, 2); + long currentWindowStart = + RateFunctionValidation.readRequiredTime( + arguments[2], position, RateFunctionType.INCREASE, 3); + long currentWindowEnd = + RateFunctionValidation.readRequiredTime( + arguments[3], position, RateFunctionType.INCREASE, 4); + RateFunctionValidation.validateWindow( + RateFunctionType.INCREASE, time, currentWindowStart, currentWindowEnd); + initializeOrValidateWindow(groupId, currentWindowStart, currentWindowEnd); + update(groupId, time, value); + } + } + + @Override + public void addIntermediate(int[] groupIds, Column argument) { + throw unsupportedIntermediate(); + } + + @Override + public void evaluateIntermediate(int groupId, ColumnBuilder output) { + throw unsupportedIntermediate(); + } + + @Override + public void evaluateFinal(int groupId, ColumnBuilder output) { + int sampleCount = sampleCounts.get(groupId); + if (sampleCount < 2) { + output.appendNull(); + return; + } + output.writeDouble( + calculateIncrease( + sampleCount, + firstTimes.get(groupId), + firstValues.get(groupId), + lastTimes.get(groupId), + correctedIncreases.get(groupId), + windowStart(groupId), + windowEnd(groupId))); + } + + @Override + public void reset() { + resetWindows(); + sampleCounts.reset(); + firstTimes.reset(); + firstValues.reset(); + lastTimes.reset(); + lastValues.reset(); + correctedIncreases.reset(); + updateMemoryReservation(); + } + + private void update(int groupId, long time, double value) { + int sampleCount = sampleCounts.get(groupId); + if (sampleCount == 0) { + sampleCounts.set(groupId, 1); + firstTimes.set(groupId, time); + firstValues.set(groupId, value); + lastTimes.set(groupId, time); + lastValues.set(groupId, value); + return; + } + long lastTime = lastTimes.get(groupId); + if (time == lastTime) { + throw duplicateTimestamp(time); + } + if (time < lastTime) { + throw orderedInputViolation(time, lastTime); + } + double lastValue = lastValues.get(groupId); + double increment = value >= lastValue ? value - lastValue : value; + double correctedIncrease = validateFinite(correctedIncreases.get(groupId) + increment); + lastTimes.set(groupId, time); + lastValues.set(groupId, value); + correctedIncreases.set(groupId, correctedIncrease); + sampleCounts.set(groupId, Math.incrementExact(sampleCount)); + } + + private void updateMemoryReservation() { + long currentSize = getEstimatedSize(); + long delta = currentSize - previousSize; + if (delta > 0) { + memoryReservationManager.reserveMemoryCumulatively(delta); + } else if (delta < 0) { + memoryReservationManager.releaseMemoryCumulatively(-delta); + } + previousSize = currentSize; + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/GroupedOrderedIrateAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/GroupedOrderedIrateAccumulator.java new file mode 100644 index 0000000000000..837c9d3dc8849 --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/GroupedOrderedIrateAccumulator.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.rate; + +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.AggregationMask; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.DoubleBigArray; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.IntBigArray; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.LongBigArray; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.RateFunctionType; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.RateFunctionValidation; +import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.RamUsageEstimator; + +public final class GroupedOrderedIrateAccumulator extends AbstractGroupedIrateAccumulator { + + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(GroupedOrderedIrateAccumulator.class); + + private final IntBigArray sampleCounts = new IntBigArray(); + private final LongBigArray previousTimes = new LongBigArray(); + private final DoubleBigArray previousValues = new DoubleBigArray(); + private final LongBigArray lastTimes = new LongBigArray(); + private final DoubleBigArray lastValues = new DoubleBigArray(); + private final MemoryReservationManager memoryReservationManager; + private long previousSize; + + public GroupedOrderedIrateAccumulator( + TSDataType valueDataType, MemoryReservationManager memoryReservationManager) { + super(valueDataType); + this.memoryReservationManager = memoryReservationManager; + updateMemoryReservation(); + } + + @Override + public long getEstimatedSize() { + return INSTANCE_SIZE + + sampleCounts.sizeOf() + + previousTimes.sizeOf() + + previousValues.sizeOf() + + lastTimes.sizeOf() + + lastValues.sizeOf(); + } + + @Override + public void setGroupCount(long groupCount) { + sampleCounts.ensureCapacity(groupCount); + previousTimes.ensureCapacity(groupCount); + previousValues.ensureCapacity(groupCount); + lastTimes.ensureCapacity(groupCount); + lastValues.ensureCapacity(groupCount); + updateMemoryReservation(); + } + + @Override + public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { + RateFunctionValidation.validateArgumentCount(arguments, RateFunctionType.IRATE); + int selectedCount = mask.getSelectedPositionCount(); + int[] selectedPositions = mask.isSelectAll() ? null : mask.getSelectedPositions(); + for (int index = 0; index < selectedCount; index++) { + int position = mask.isSelectAll() ? index : selectedPositions[index]; + if (arguments[0].isNull(position)) { + continue; + } + int groupId = groupIds[position]; + double value = + RateFunctionValidation.readValue( + arguments[0], position, valueDataType, RateFunctionType.IRATE); + long time = + RateFunctionValidation.readRequiredTime( + arguments[1], position, RateFunctionType.IRATE, 2); + update(groupId, time, value); + } + } + + @Override + public void addIntermediate(int[] groupIds, Column argument) { + throw unsupportedIntermediate(); + } + + @Override + public void evaluateIntermediate(int groupId, ColumnBuilder output) { + throw unsupportedIntermediate(); + } + + @Override + public void evaluateFinal(int groupId, ColumnBuilder output) { + if (sampleCounts.get(groupId) < 2) { + output.appendNull(); + return; + } + output.writeDouble( + calculateIrate( + previousTimes.get(groupId), + previousValues.get(groupId), + lastTimes.get(groupId), + lastValues.get(groupId))); + } + + @Override + public void reset() { + sampleCounts.reset(); + previousTimes.reset(); + previousValues.reset(); + lastTimes.reset(); + lastValues.reset(); + updateMemoryReservation(); + } + + private void update(int groupId, long time, double value) { + int sampleCount = sampleCounts.get(groupId); + if (sampleCount == 0) { + sampleCounts.set(groupId, 1); + lastTimes.set(groupId, time); + lastValues.set(groupId, value); + return; + } + long lastTime = lastTimes.get(groupId); + if (time == lastTime) { + throw duplicateTimestamp(time); + } + if (time < lastTime) { + throw orderedInputViolation(time, lastTime); + } + previousTimes.set(groupId, lastTime); + previousValues.set(groupId, lastValues.get(groupId)); + lastTimes.set(groupId, time); + lastValues.set(groupId, value); + sampleCounts.set(groupId, Math.incrementExact(sampleCount)); + } + + private void updateMemoryReservation() { + long currentSize = getEstimatedSize(); + long delta = currentSize - previousSize; + if (delta > 0) { + memoryReservationManager.reserveMemoryCumulatively(delta); + } else if (delta < 0) { + memoryReservationManager.releaseMemoryCumulatively(-delta); + } + previousSize = currentSize; + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/GroupedOrderedRateAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/GroupedOrderedRateAccumulator.java new file mode 100644 index 0000000000000..3c5e784d5545d --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/GroupedOrderedRateAccumulator.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.rate; + +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.AggregationMask; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.DoubleBigArray; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.IntBigArray; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.LongBigArray; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.RateFunctionType; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.RateFunctionValidation; +import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.RamUsageEstimator; + +public final class GroupedOrderedRateAccumulator extends AbstractGroupedRateAccumulator { + + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(GroupedOrderedRateAccumulator.class); + + private final IntBigArray sampleCounts = new IntBigArray(); + private final LongBigArray firstTimes = new LongBigArray(); + private final DoubleBigArray firstValues = new DoubleBigArray(); + private final LongBigArray lastTimes = new LongBigArray(); + private final DoubleBigArray lastValues = new DoubleBigArray(); + private final DoubleBigArray correctedIncreases = new DoubleBigArray(); + private final MemoryReservationManager memoryReservationManager; + private long previousSize; + + public GroupedOrderedRateAccumulator( + TSDataType valueDataType, MemoryReservationManager memoryReservationManager) { + super(valueDataType); + this.memoryReservationManager = memoryReservationManager; + updateMemoryReservation(); + } + + @Override + public long getEstimatedSize() { + return INSTANCE_SIZE + + windowsSizeOf() + + sampleCounts.sizeOf() + + firstTimes.sizeOf() + + firstValues.sizeOf() + + lastTimes.sizeOf() + + lastValues.sizeOf() + + correctedIncreases.sizeOf(); + } + + @Override + public void setGroupCount(long groupCount) { + ensureWindowCapacity(groupCount); + sampleCounts.ensureCapacity(groupCount); + firstTimes.ensureCapacity(groupCount); + firstValues.ensureCapacity(groupCount); + lastTimes.ensureCapacity(groupCount); + lastValues.ensureCapacity(groupCount); + correctedIncreases.ensureCapacity(groupCount); + updateMemoryReservation(); + } + + @Override + public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { + RateFunctionValidation.validateArgumentCount(arguments, RateFunctionType.RATE); + int selectedCount = mask.getSelectedPositionCount(); + int[] selectedPositions = mask.isSelectAll() ? null : mask.getSelectedPositions(); + for (int index = 0; index < selectedCount; index++) { + int position = mask.isSelectAll() ? index : selectedPositions[index]; + if (arguments[0].isNull(position)) { + continue; + } + int groupId = groupIds[position]; + double value = + RateFunctionValidation.readValue( + arguments[0], position, valueDataType, RateFunctionType.RATE); + long time = + RateFunctionValidation.readRequiredTime(arguments[1], position, RateFunctionType.RATE, 2); + long currentWindowStart = + RateFunctionValidation.readRequiredTime(arguments[2], position, RateFunctionType.RATE, 3); + long currentWindowEnd = + RateFunctionValidation.readRequiredTime(arguments[3], position, RateFunctionType.RATE, 4); + RateFunctionValidation.validateWindow( + RateFunctionType.RATE, time, currentWindowStart, currentWindowEnd); + initializeOrValidateWindow(groupId, currentWindowStart, currentWindowEnd); + update(groupId, time, value); + } + } + + @Override + public void addIntermediate(int[] groupIds, Column argument) { + throw unsupportedIntermediate(); + } + + @Override + public void evaluateIntermediate(int groupId, ColumnBuilder output) { + throw unsupportedIntermediate(); + } + + @Override + public void evaluateFinal(int groupId, ColumnBuilder output) { + int sampleCount = sampleCounts.get(groupId); + if (sampleCount < 2) { + output.appendNull(); + return; + } + output.writeDouble( + calculateRate( + sampleCount, + firstTimes.get(groupId), + firstValues.get(groupId), + lastTimes.get(groupId), + correctedIncreases.get(groupId), + windowStart(groupId), + windowEnd(groupId))); + } + + @Override + public void reset() { + resetWindows(); + sampleCounts.reset(); + firstTimes.reset(); + firstValues.reset(); + lastTimes.reset(); + lastValues.reset(); + correctedIncreases.reset(); + updateMemoryReservation(); + } + + private void update(int groupId, long time, double value) { + int sampleCount = sampleCounts.get(groupId); + if (sampleCount == 0) { + sampleCounts.set(groupId, 1); + firstTimes.set(groupId, time); + firstValues.set(groupId, value); + lastTimes.set(groupId, time); + lastValues.set(groupId, value); + return; + } + long lastTime = lastTimes.get(groupId); + if (time == lastTime) { + throw duplicateTimestamp(time); + } + if (time < lastTime) { + throw orderedInputViolation(time, lastTime); + } + double lastValue = lastValues.get(groupId); + double increment = value >= lastValue ? value - lastValue : value; + double correctedIncrease = validateFinite(correctedIncreases.get(groupId) + increment); + lastTimes.set(groupId, time); + lastValues.set(groupId, value); + correctedIncreases.set(groupId, correctedIncrease); + sampleCounts.set(groupId, Math.incrementExact(sampleCount)); + } + + private void updateMemoryReservation() { + long currentSize = getEstimatedSize(); + long delta = currentSize - previousSize; + if (delta > 0) { + memoryReservationManager.reserveMemoryCumulatively(delta); + } else if (delta < 0) { + memoryReservationManager.releaseMemoryCumulatively(-delta); + } + previousSize = currentSize; + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/TimeValueBufferBigArray.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/TimeValueBufferBigArray.java new file mode 100644 index 0000000000000..be8c882da6109 --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/grouped/rate/TimeValueBufferBigArray.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.rate; + +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.grouped.array.ObjectBigArray; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate.TimeValueBuffer; + +import org.apache.tsfile.utils.RamUsageEstimator; + +final class TimeValueBufferBigArray { + + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(TimeValueBufferBigArray.class); + + private final ObjectBigArray buffers = new ObjectBigArray<>(); + private long buffersRetainedBytes; + + public void ensureCapacity(long groupCount) { + buffers.ensureCapacity(groupCount); + } + + public TimeValueBuffer get(int groupId) { + return buffers.get(groupId); + } + + public void add(int groupId, long time, double value) { + TimeValueBuffer buffer = getOrCreate(groupId); + long previousSize = buffer.getEstimatedSize(); + buffer.add(time, value); + buffersRetainedBytes += buffer.getEstimatedSize() - previousSize; + } + + public void merge(int groupId, TimeValueBuffer other) { + TimeValueBuffer buffer = getOrCreate(groupId); + long previousSize = buffer.getEstimatedSize(); + buffer.merge(other); + buffersRetainedBytes += buffer.getEstimatedSize() - previousSize; + } + + public long sizeOf() { + return INSTANCE_SIZE + buffers.sizeOf() + buffersRetainedBytes; + } + + public void reset() { + buffers.reset(); + buffersRetainedBytes = 0; + } + + private TimeValueBuffer getOrCreate(int groupId) { + TimeValueBuffer buffer = buffers.get(groupId); + if (buffer == null) { + buffer = new TimeValueBuffer(); + buffers.set(groupId, buffer); + buffersRetainedBytes += buffer.getEstimatedSize(); + } + return buffer; + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/AbstractDeltaTableAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/AbstractDeltaTableAccumulator.java new file mode 100644 index 0000000000000..6af6db3f2be80 --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/AbstractDeltaTableAccumulator.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate; + +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.TableAccumulator; +import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.commons.exception.SemanticException; + +import org.apache.tsfile.enums.TSDataType; + +abstract class AbstractDeltaTableAccumulator implements TableAccumulator { + + protected final TSDataType valueDataType; + protected boolean windowInitialized; + protected long windowStart; + protected long windowEnd; + + protected AbstractDeltaTableAccumulator(TSDataType valueDataType) { + this.valueDataType = valueDataType; + } + + protected final void initializeOrValidateWindow( + long candidateWindowStart, long candidateWindowEnd) { + if (!windowInitialized) { + windowStart = candidateWindowStart; + windowEnd = candidateWindowEnd; + windowInitialized = true; + return; + } + if (windowStart != candidateWindowStart || windowEnd != candidateWindowEnd) { + throw inconsistentWindow(candidateWindowStart, candidateWindowEnd); + } + } + + protected final void resetWindow() { + windowInitialized = false; + windowStart = 0; + windowEnd = 0; + } + + protected final double calculateDelta( + int sampleCount, long firstTime, double firstValue, long lastTime, double lastValue) { + return ExtrapolationUtil.extrapolate( + sampleCount, + firstTime, + firstValue, + lastTime, + windowStart, + windowEnd, + lastValue - firstValue, + false); + } + + protected final SemanticException duplicateTimestamp(long time) { + return new SemanticException( + String.format( + CalcMessages + .EXCEPTION_AGGREGATE_FUNCTION_ARG_DOES_NOT_SUPPORT_DUPLICATE_TIME_COL_VALUES_IN_THE_SAME_AGGREGATION_GROUP_ARG_087A91BC, + RateFunctionType.DELTA.getFunctionName(), + time)); + } + + protected final SemanticException orderedInputViolation(long time, long previousTime) { + return new SemanticException( + String.format( + CalcMessages + .EXCEPTION_AGGREGATE_FUNCTION_ARG_EXPECTED_TIME_COL_IN_STRICTLY_ASCENDING_ORDER_BUT_GOT_ARG_AFTER_ARG_9289E0F9, + RateFunctionType.DELTA.getFunctionName(), + time, + previousTime)); + } + + protected final UnsupportedOperationException unsupportedIntermediate() { + return new UnsupportedOperationException( + String.format( + CalcMessages + .EXCEPTION_ORDERED_AGGREGATE_FUNCTION_ARG_DOES_NOT_SUPPORT_INTERMEDIATE_STATE_6B4B2B1B, + RateFunctionType.DELTA.getFunctionName())); + } + + private SemanticException inconsistentWindow(long candidateWindowStart, long candidateWindowEnd) { + return new SemanticException( + String.format( + CalcMessages + .EXCEPTION_AGGREGATE_FUNCTION_ARG_REQUIRES_CONSISTENT_WINDOW_BOUNDARIES_IN_THE_SAME_AGGREGATION_GROUP_EXPECTED_ARG_ARG_BUT_GOT_ARG_ARG_38631886, + RateFunctionType.DELTA.getFunctionName(), + windowStart, + windowEnd, + candidateWindowStart, + candidateWindowEnd)); + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/AbstractIncreaseTableAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/AbstractIncreaseTableAccumulator.java new file mode 100644 index 0000000000000..6463e940d35e4 --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/AbstractIncreaseTableAccumulator.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate; + +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.TableAccumulator; +import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.commons.exception.SemanticException; + +import org.apache.tsfile.enums.TSDataType; + +abstract class AbstractIncreaseTableAccumulator implements TableAccumulator { + + protected final TSDataType valueDataType; + protected boolean windowInitialized; + protected long windowStart; + protected long windowEnd; + + protected AbstractIncreaseTableAccumulator(TSDataType valueDataType) { + this.valueDataType = valueDataType; + } + + protected final void initializeOrValidateWindow( + long candidateWindowStart, long candidateWindowEnd) { + if (!windowInitialized) { + windowStart = candidateWindowStart; + windowEnd = candidateWindowEnd; + windowInitialized = true; + return; + } + if (windowStart != candidateWindowStart || windowEnd != candidateWindowEnd) { + throw inconsistentWindow(candidateWindowStart, candidateWindowEnd); + } + } + + protected final void resetWindow() { + windowInitialized = false; + windowStart = 0; + windowEnd = 0; + } + + protected final double calculateIncrease( + int sampleCount, long firstTime, double firstValue, long lastTime, double correctedIncrease) { + return ExtrapolationUtil.extrapolate( + sampleCount, + firstTime, + firstValue, + lastTime, + windowStart, + windowEnd, + correctedIncrease, + true); + } + + protected final SemanticException duplicateTimestamp(long time) { + return new SemanticException( + String.format( + CalcMessages + .EXCEPTION_AGGREGATE_FUNCTION_ARG_DOES_NOT_SUPPORT_DUPLICATE_TIME_COL_VALUES_IN_THE_SAME_AGGREGATION_GROUP_ARG_087A91BC, + RateFunctionType.INCREASE.getFunctionName(), + time)); + } + + protected final SemanticException orderedInputViolation(long time, long previousTime) { + return new SemanticException( + String.format( + CalcMessages + .EXCEPTION_AGGREGATE_FUNCTION_ARG_EXPECTED_TIME_COL_IN_STRICTLY_ASCENDING_ORDER_BUT_GOT_ARG_AFTER_ARG_9289E0F9, + RateFunctionType.INCREASE.getFunctionName(), + time, + previousTime)); + } + + protected final UnsupportedOperationException unsupportedIntermediate() { + return new UnsupportedOperationException( + String.format( + CalcMessages + .EXCEPTION_ORDERED_AGGREGATE_FUNCTION_ARG_DOES_NOT_SUPPORT_INTERMEDIATE_STATE_6B4B2B1B, + RateFunctionType.INCREASE.getFunctionName())); + } + + protected final double validateFinite(double value) { + if (!Double.isFinite(value)) { + throw new SemanticException( + CalcMessages + .EXCEPTION_RATE_FAMILY_AGGREGATE_FUNCTION_PRODUCED_A_NON_FINITE_INTERMEDIATE_RESULT_D46B30CD); + } + return value; + } + + private SemanticException inconsistentWindow(long candidateWindowStart, long candidateWindowEnd) { + return new SemanticException( + String.format( + CalcMessages + .EXCEPTION_AGGREGATE_FUNCTION_ARG_REQUIRES_CONSISTENT_WINDOW_BOUNDARIES_IN_THE_SAME_AGGREGATION_GROUP_EXPECTED_ARG_ARG_BUT_GOT_ARG_ARG_38631886, + RateFunctionType.INCREASE.getFunctionName(), + windowStart, + windowEnd, + candidateWindowStart, + candidateWindowEnd)); + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/AbstractIrateTableAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/AbstractIrateTableAccumulator.java new file mode 100644 index 0000000000000..7366691dbc4d4 --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/AbstractIrateTableAccumulator.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate; + +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.TableAccumulator; +import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.commons.exception.SemanticException; + +import org.apache.tsfile.enums.TSDataType; + +abstract class AbstractIrateTableAccumulator implements TableAccumulator { + + protected final TSDataType valueDataType; + + protected AbstractIrateTableAccumulator(TSDataType valueDataType) { + this.valueDataType = valueDataType; + } + + protected final double calculateIrate( + long previousTime, double previousValue, long lastTime, double lastValue) { + double increase = lastValue >= previousValue ? lastValue - previousValue : lastValue; + return validateFinite( + increase / ExtrapolationUtil.timestampDiffToSeconds(lastTime, previousTime)); + } + + protected final SemanticException duplicateTimestamp(long time) { + return new SemanticException( + String.format( + CalcMessages + .EXCEPTION_AGGREGATE_FUNCTION_ARG_DOES_NOT_SUPPORT_DUPLICATE_TIME_COL_VALUES_IN_THE_SAME_AGGREGATION_GROUP_ARG_087A91BC, + RateFunctionType.IRATE.getFunctionName(), + time)); + } + + protected final SemanticException orderedInputViolation(long time, long previousTime) { + return new SemanticException( + String.format( + CalcMessages + .EXCEPTION_AGGREGATE_FUNCTION_ARG_EXPECTED_TIME_COL_IN_STRICTLY_ASCENDING_ORDER_BUT_GOT_ARG_AFTER_ARG_9289E0F9, + RateFunctionType.IRATE.getFunctionName(), + time, + previousTime)); + } + + protected final UnsupportedOperationException unsupportedIntermediate() { + return new UnsupportedOperationException( + String.format( + CalcMessages + .EXCEPTION_ORDERED_AGGREGATE_FUNCTION_ARG_DOES_NOT_SUPPORT_INTERMEDIATE_STATE_6B4B2B1B, + RateFunctionType.IRATE.getFunctionName())); + } + + protected final double validateFinite(double value) { + if (!Double.isFinite(value)) { + throw new SemanticException( + CalcMessages + .EXCEPTION_RATE_FAMILY_AGGREGATE_FUNCTION_PRODUCED_A_NON_FINITE_INTERMEDIATE_RESULT_D46B30CD); + } + return value; + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/AbstractRateTableAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/AbstractRateTableAccumulator.java new file mode 100644 index 0000000000000..8c2cab7144810 --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/AbstractRateTableAccumulator.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate; + +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.TableAccumulator; +import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.commons.exception.SemanticException; + +import org.apache.tsfile.enums.TSDataType; + +abstract class AbstractRateTableAccumulator implements TableAccumulator { + + protected final TSDataType valueDataType; + protected boolean windowInitialized; + protected long windowStart; + protected long windowEnd; + + protected AbstractRateTableAccumulator(TSDataType valueDataType) { + this.valueDataType = valueDataType; + } + + protected final void initializeOrValidateWindow( + long candidateWindowStart, long candidateWindowEnd) { + if (!windowInitialized) { + windowStart = candidateWindowStart; + windowEnd = candidateWindowEnd; + windowInitialized = true; + return; + } + if (windowStart != candidateWindowStart || windowEnd != candidateWindowEnd) { + throw inconsistentWindow(candidateWindowStart, candidateWindowEnd); + } + } + + protected final void resetWindow() { + windowInitialized = false; + windowStart = 0; + windowEnd = 0; + } + + protected final double calculateRate( + int sampleCount, long firstTime, double firstValue, long lastTime, double correctedIncrease) { + double extrapolated = + ExtrapolationUtil.extrapolate( + sampleCount, + firstTime, + firstValue, + lastTime, + windowStart, + windowEnd, + correctedIncrease, + true); + return validateFinite( + extrapolated / ExtrapolationUtil.timestampDiffToSeconds(windowEnd, windowStart)); + } + + protected final SemanticException duplicateTimestamp(long time) { + return new SemanticException( + String.format( + CalcMessages + .EXCEPTION_AGGREGATE_FUNCTION_ARG_DOES_NOT_SUPPORT_DUPLICATE_TIME_COL_VALUES_IN_THE_SAME_AGGREGATION_GROUP_ARG_087A91BC, + RateFunctionType.RATE.getFunctionName(), + time)); + } + + protected final SemanticException orderedInputViolation(long time, long previousTime) { + return new SemanticException( + String.format( + CalcMessages + .EXCEPTION_AGGREGATE_FUNCTION_ARG_EXPECTED_TIME_COL_IN_STRICTLY_ASCENDING_ORDER_BUT_GOT_ARG_AFTER_ARG_9289E0F9, + RateFunctionType.RATE.getFunctionName(), + time, + previousTime)); + } + + protected final UnsupportedOperationException unsupportedIntermediate() { + return new UnsupportedOperationException( + String.format( + CalcMessages + .EXCEPTION_ORDERED_AGGREGATE_FUNCTION_ARG_DOES_NOT_SUPPORT_INTERMEDIATE_STATE_6B4B2B1B, + RateFunctionType.RATE.getFunctionName())); + } + + protected final double validateFinite(double value) { + if (!Double.isFinite(value)) { + throw new SemanticException( + CalcMessages + .EXCEPTION_RATE_FAMILY_AGGREGATE_FUNCTION_PRODUCED_A_NON_FINITE_INTERMEDIATE_RESULT_D46B30CD); + } + return value; + } + + private SemanticException inconsistentWindow(long candidateWindowStart, long candidateWindowEnd) { + return new SemanticException( + String.format( + CalcMessages + .EXCEPTION_AGGREGATE_FUNCTION_ARG_REQUIRES_CONSISTENT_WINDOW_BOUNDARIES_IN_THE_SAME_AGGREGATION_GROUP_EXPECTED_ARG_ARG_BUT_GOT_ARG_ARG_38631886, + RateFunctionType.RATE.getFunctionName(), + windowStart, + windowEnd, + candidateWindowStart, + candidateWindowEnd)); + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/ExtrapolationUtil.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/ExtrapolationUtil.java new file mode 100644 index 0000000000000..d01b8860e7ab1 --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/ExtrapolationUtil.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate; + +import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.commons.exception.SemanticException; +import org.apache.iotdb.commons.queryengine.utils.TimestampPrecisionUtils; + +import java.math.BigInteger; +import java.util.concurrent.TimeUnit; + +public final class ExtrapolationUtil { + + private static final double EXTRAPOLATION_THRESHOLD_FACTOR = 1.1; + + private ExtrapolationUtil() {} + + public static double extrapolate( + int sampleCount, + long firstTime, + double firstValue, + long lastTime, + long windowStart, + long windowEnd, + double increase, + boolean applyCounterZeroProtection) { + validateBoundaries(sampleCount, firstTime, lastTime, windowStart, windowEnd); + + double sampledInterval = timestampDiffToSeconds(lastTime, firstTime); + double durationToStart = timestampDiffToSeconds(firstTime, windowStart); + double durationToEnd = timestampDiffToSeconds(windowEnd, lastTime); + + if (!Double.isFinite(increase)) { + throw new SemanticException( + CalcMessages + .EXCEPTION_RATE_FAMILY_AGGREGATE_FUNCTION_PRODUCED_A_NON_FINITE_INTERMEDIATE_RESULT_D46B30CD); + } + if (applyCounterZeroProtection && increase == 0.0) { + return 0.0; + } + + double averageInterval = sampledInterval / (sampleCount - 1); + double threshold = averageInterval * EXTRAPOLATION_THRESHOLD_FACTOR; + + if (durationToStart >= threshold) { + durationToStart = averageInterval / 2.0; + } + + if (applyCounterZeroProtection && increase > 0.0) { + double durationToZero = sampledInterval * (firstValue / increase); + if (Double.isFinite(durationToZero) && durationToZero < durationToStart) { + durationToStart = durationToZero; + } + } + + if (durationToEnd >= threshold) { + durationToEnd = averageInterval / 2.0; + } + + double factor = (sampledInterval + durationToStart + durationToEnd) / sampledInterval; + double result = increase * factor; + if (!Double.isFinite(factor) || !Double.isFinite(result)) { + throw new SemanticException( + CalcMessages + .EXCEPTION_RATE_FAMILY_AGGREGATE_FUNCTION_PRODUCED_A_NON_FINITE_EXTRAPOLATION_RESULT_6482CF1D); + } + return result; + } + + public static double timestampDiffToSeconds(long later, long earlier) { + BigInteger ticks = BigInteger.valueOf(later).subtract(BigInteger.valueOf(earlier)); + double ticksPerSecond = TimestampPrecisionUtils.currPrecision.convert(1, TimeUnit.SECONDS); + double seconds = ticks.doubleValue() / ticksPerSecond; + if (!Double.isFinite(seconds)) { + throw new SemanticException( + CalcMessages + .EXCEPTION_RATE_FAMILY_AGGREGATE_FUNCTION_PRODUCED_A_NON_FINITE_TIME_INTERVAL_B26C4162); + } + return seconds; + } + + private static void validateBoundaries( + int sampleCount, long firstTime, long lastTime, long windowStart, long windowEnd) { + if (sampleCount < 2) { + throw new SemanticException( + CalcMessages + .EXCEPTION_RATE_FAMILY_AGGREGATE_FUNCTION_REQUIRES_AT_LEAST_TWO_VALID_SAMPLES_1A89901C); + } + if (firstTime >= lastTime) { + throw new SemanticException( + CalcMessages + .EXCEPTION_THE_FIRST_SAMPLE_TIME_MUST_BE_LESS_THAN_THE_LAST_SAMPLE_TIME_FC4D3517); + } + if (windowStart >= windowEnd) { + throw new SemanticException( + CalcMessages.EXCEPTION_THE_WINDOW_START_MUST_BE_LESS_THAN_THE_WINDOW_END_B1A38C98); + } + if (firstTime < windowStart || lastTime >= windowEnd) { + throw new SemanticException( + CalcMessages + .EXCEPTION_RATE_FAMILY_AGGREGATE_FUNCTION_REQUIRES_WINDOW_START_FIRST_TIME_LAST_TIME_WINDOW_END_60DE93C8); + } + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/NaiveDeltaAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/NaiveDeltaAccumulator.java new file mode 100644 index 0000000000000..dc535405171b1 --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/NaiveDeltaAccumulator.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate; + +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.AggregationMask; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.TableAccumulator; +import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.utils.RamUsageEstimator; + +public final class NaiveDeltaAccumulator extends AbstractDeltaTableAccumulator { + + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(NaiveDeltaAccumulator.class); + + private final TimeValueBuffer samples = new TimeValueBuffer(); + private final MemoryReservationManager memoryReservationManager; + private long previousSamplesSize; + + public NaiveDeltaAccumulator( + TSDataType valueDataType, MemoryReservationManager memoryReservationManager) { + super(valueDataType); + this.memoryReservationManager = memoryReservationManager; + updateMemoryReservation(); + } + + @Override + public long getEstimatedSize() { + return INSTANCE_SIZE + samples.getEstimatedSize(); + } + + @Override + public TableAccumulator copy() { + return new NaiveDeltaAccumulator(valueDataType, memoryReservationManager); + } + + @Override + public void addInput(Column[] arguments, AggregationMask mask) { + RateFunctionValidation.validateArgumentCount(arguments, RateFunctionType.DELTA); + int selectedCount = mask.getSelectedPositionCount(); + int[] selectedPositions = mask.isSelectAll() ? null : mask.getSelectedPositions(); + for (int index = 0; index < selectedCount; index++) { + int position = mask.isSelectAll() ? index : selectedPositions[index]; + if (arguments[0].isNull(position)) { + continue; + } + double value = + RateFunctionValidation.readValue( + arguments[0], position, valueDataType, RateFunctionType.DELTA); + long time = + RateFunctionValidation.readRequiredTime( + arguments[1], position, RateFunctionType.DELTA, 2); + long currentWindowStart = + RateFunctionValidation.readRequiredTime( + arguments[2], position, RateFunctionType.DELTA, 3); + long currentWindowEnd = + RateFunctionValidation.readRequiredTime( + arguments[3], position, RateFunctionType.DELTA, 4); + RateFunctionValidation.validateWindow( + RateFunctionType.DELTA, time, currentWindowStart, currentWindowEnd); + initializeOrValidateWindow(currentWindowStart, currentWindowEnd); + samples.add(time, value); + } + updateMemoryReservation(); + } + + @Override + public void addIntermediate(Column argument) { + for (int position = 0; position < argument.getPositionCount(); position++) { + if (argument.isNull(position)) { + continue; + } + RateFunctionIntermediateStateCodec.DecodedState decoded = + RateFunctionIntermediateStateCodec.decode( + RateFunctionType.DELTA, argument.getBinary(position)); + initializeOrValidateWindow(decoded.getWindowStart(), decoded.getWindowEnd()); + samples.merge(decoded.getSamples()); + } + updateMemoryReservation(); + } + + @Override + public void evaluateIntermediate(ColumnBuilder output) { + RateFunctionIntermediateStateCodec.encode( + RateFunctionType.DELTA, windowStart, windowEnd, samples, output); + } + + @Override + public void evaluateFinal(ColumnBuilder output) { + samples.sortAndValidate(RateFunctionType.DELTA.getFunctionName()); + if (samples.size() < 2) { + output.appendNull(); + return; + } + int lastIndex = samples.size() - 1; + output.writeDouble( + calculateDelta( + samples.size(), + samples.getTime(0), + samples.getValue(0), + samples.getTime(lastIndex), + samples.getValue(lastIndex))); + } + + @Override + public boolean hasFinalResult() { + return false; + } + + @Override + public void addStatistics(Statistics[] statistics) { + throw new UnsupportedOperationException(); + } + + @Override + public void reset() { + resetWindow(); + samples.reset(); + updateMemoryReservation(); + } + + private void updateMemoryReservation() { + long currentSize = samples.getEstimatedSize(); + long delta = currentSize - previousSamplesSize; + if (delta > 0) { + memoryReservationManager.reserveMemoryCumulatively(delta); + } else if (delta < 0) { + memoryReservationManager.releaseMemoryCumulatively(-delta); + } + previousSamplesSize = currentSize; + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/NaiveIncreaseAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/NaiveIncreaseAccumulator.java new file mode 100644 index 0000000000000..90c78814a37f4 --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/NaiveIncreaseAccumulator.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate; + +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.AggregationMask; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.TableAccumulator; +import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.utils.RamUsageEstimator; + +public final class NaiveIncreaseAccumulator extends AbstractIncreaseTableAccumulator { + + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(NaiveIncreaseAccumulator.class); + + private final TimeValueBuffer samples = new TimeValueBuffer(); + private final MemoryReservationManager memoryReservationManager; + private long previousSamplesSize; + + public NaiveIncreaseAccumulator( + TSDataType valueDataType, MemoryReservationManager memoryReservationManager) { + super(valueDataType); + this.memoryReservationManager = memoryReservationManager; + updateMemoryReservation(); + } + + @Override + public long getEstimatedSize() { + return INSTANCE_SIZE + samples.getEstimatedSize(); + } + + @Override + public TableAccumulator copy() { + return new NaiveIncreaseAccumulator(valueDataType, memoryReservationManager); + } + + @Override + public void addInput(Column[] arguments, AggregationMask mask) { + RateFunctionValidation.validateArgumentCount(arguments, RateFunctionType.INCREASE); + int selectedCount = mask.getSelectedPositionCount(); + int[] selectedPositions = mask.isSelectAll() ? null : mask.getSelectedPositions(); + for (int index = 0; index < selectedCount; index++) { + int position = mask.isSelectAll() ? index : selectedPositions[index]; + if (arguments[0].isNull(position)) { + continue; + } + double value = + RateFunctionValidation.readValue( + arguments[0], position, valueDataType, RateFunctionType.INCREASE); + long time = + RateFunctionValidation.readRequiredTime( + arguments[1], position, RateFunctionType.INCREASE, 2); + long currentWindowStart = + RateFunctionValidation.readRequiredTime( + arguments[2], position, RateFunctionType.INCREASE, 3); + long currentWindowEnd = + RateFunctionValidation.readRequiredTime( + arguments[3], position, RateFunctionType.INCREASE, 4); + RateFunctionValidation.validateWindow( + RateFunctionType.INCREASE, time, currentWindowStart, currentWindowEnd); + initializeOrValidateWindow(currentWindowStart, currentWindowEnd); + samples.add(time, value); + } + updateMemoryReservation(); + } + + @Override + public void addIntermediate(Column argument) { + for (int position = 0; position < argument.getPositionCount(); position++) { + if (argument.isNull(position)) { + continue; + } + RateFunctionIntermediateStateCodec.DecodedState decoded = + RateFunctionIntermediateStateCodec.decode( + RateFunctionType.INCREASE, argument.getBinary(position)); + initializeOrValidateWindow(decoded.getWindowStart(), decoded.getWindowEnd()); + samples.merge(decoded.getSamples()); + } + updateMemoryReservation(); + } + + @Override + public void evaluateIntermediate(ColumnBuilder output) { + RateFunctionIntermediateStateCodec.encode( + RateFunctionType.INCREASE, windowStart, windowEnd, samples, output); + } + + @Override + public void evaluateFinal(ColumnBuilder output) { + samples.sortAndValidate(RateFunctionType.INCREASE.getFunctionName()); + if (samples.size() < 2) { + output.appendNull(); + return; + } + int lastIndex = samples.size() - 1; + double correctedIncrease = calculateCorrectedIncrease(); + output.writeDouble( + calculateIncrease( + samples.size(), + samples.getTime(0), + samples.getValue(0), + samples.getTime(lastIndex), + correctedIncrease)); + } + + @Override + public boolean hasFinalResult() { + return false; + } + + @Override + public void addStatistics(Statistics[] statistics) { + throw new UnsupportedOperationException(); + } + + @Override + public void reset() { + resetWindow(); + samples.reset(); + updateMemoryReservation(); + } + + private double calculateCorrectedIncrease() { + double result = 0.0; + for (int index = 1; index < samples.size(); index++) { + double previous = samples.getValue(index - 1); + double current = samples.getValue(index); + result = validateFinite(result + (current >= previous ? current - previous : current)); + } + return result; + } + + private void updateMemoryReservation() { + long currentSize = samples.getEstimatedSize(); + long delta = currentSize - previousSamplesSize; + if (delta > 0) { + memoryReservationManager.reserveMemoryCumulatively(delta); + } else if (delta < 0) { + memoryReservationManager.releaseMemoryCumulatively(-delta); + } + previousSamplesSize = currentSize; + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/NaiveIrateAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/NaiveIrateAccumulator.java new file mode 100644 index 0000000000000..d66eb8aff9cd7 --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/NaiveIrateAccumulator.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate; + +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.AggregationMask; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.TableAccumulator; +import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.utils.RamUsageEstimator; + +public final class NaiveIrateAccumulator extends AbstractIrateTableAccumulator { + + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(NaiveIrateAccumulator.class); + + private final TimeValueBuffer samples = new TimeValueBuffer(); + private final MemoryReservationManager memoryReservationManager; + private long previousSamplesSize; + + public NaiveIrateAccumulator( + TSDataType valueDataType, MemoryReservationManager memoryReservationManager) { + super(valueDataType); + this.memoryReservationManager = memoryReservationManager; + updateMemoryReservation(); + } + + @Override + public long getEstimatedSize() { + return INSTANCE_SIZE + samples.getEstimatedSize(); + } + + @Override + public TableAccumulator copy() { + return new NaiveIrateAccumulator(valueDataType, memoryReservationManager); + } + + @Override + public void addInput(Column[] arguments, AggregationMask mask) { + RateFunctionValidation.validateArgumentCount(arguments, RateFunctionType.IRATE); + int selectedCount = mask.getSelectedPositionCount(); + int[] selectedPositions = mask.isSelectAll() ? null : mask.getSelectedPositions(); + for (int index = 0; index < selectedCount; index++) { + int position = mask.isSelectAll() ? index : selectedPositions[index]; + if (arguments[0].isNull(position)) { + continue; + } + double value = + RateFunctionValidation.readValue( + arguments[0], position, valueDataType, RateFunctionType.IRATE); + long time = + RateFunctionValidation.readRequiredTime( + arguments[1], position, RateFunctionType.IRATE, 2); + samples.add(time, value); + } + updateMemoryReservation(); + } + + @Override + public void addIntermediate(Column argument) { + for (int position = 0; position < argument.getPositionCount(); position++) { + if (argument.isNull(position)) { + continue; + } + RateFunctionIntermediateStateCodec.DecodedState decoded = + RateFunctionIntermediateStateCodec.decode( + RateFunctionType.IRATE, argument.getBinary(position)); + samples.merge(decoded.getSamples()); + } + updateMemoryReservation(); + } + + @Override + public void evaluateIntermediate(ColumnBuilder output) { + RateFunctionIntermediateStateCodec.encode(RateFunctionType.IRATE, 0, 0, samples, output); + } + + @Override + public void evaluateFinal(ColumnBuilder output) { + samples.sortAndValidate(RateFunctionType.IRATE.getFunctionName()); + if (samples.size() < 2) { + output.appendNull(); + return; + } + int lastIndex = samples.size() - 1; + output.writeDouble( + calculateIrate( + samples.getTime(lastIndex - 1), + samples.getValue(lastIndex - 1), + samples.getTime(lastIndex), + samples.getValue(lastIndex))); + } + + @Override + public boolean hasFinalResult() { + return false; + } + + @Override + public void addStatistics(Statistics[] statistics) { + throw new UnsupportedOperationException(); + } + + @Override + public void reset() { + samples.reset(); + updateMemoryReservation(); + } + + private void updateMemoryReservation() { + long currentSize = samples.getEstimatedSize(); + long delta = currentSize - previousSamplesSize; + if (delta > 0) { + memoryReservationManager.reserveMemoryCumulatively(delta); + } else if (delta < 0) { + memoryReservationManager.releaseMemoryCumulatively(-delta); + } + previousSamplesSize = currentSize; + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/NaiveRateAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/NaiveRateAccumulator.java new file mode 100644 index 0000000000000..68bfb41d2dad4 --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/NaiveRateAccumulator.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate; + +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.AggregationMask; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.TableAccumulator; +import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.utils.RamUsageEstimator; + +public final class NaiveRateAccumulator extends AbstractRateTableAccumulator { + + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(NaiveRateAccumulator.class); + + private final TimeValueBuffer samples = new TimeValueBuffer(); + private final MemoryReservationManager memoryReservationManager; + private long previousSamplesSize; + + public NaiveRateAccumulator( + TSDataType valueDataType, MemoryReservationManager memoryReservationManager) { + super(valueDataType); + this.memoryReservationManager = memoryReservationManager; + updateMemoryReservation(); + } + + @Override + public long getEstimatedSize() { + return INSTANCE_SIZE + samples.getEstimatedSize(); + } + + @Override + public TableAccumulator copy() { + return new NaiveRateAccumulator(valueDataType, memoryReservationManager); + } + + @Override + public void addInput(Column[] arguments, AggregationMask mask) { + RateFunctionValidation.validateArgumentCount(arguments, RateFunctionType.RATE); + int selectedCount = mask.getSelectedPositionCount(); + int[] selectedPositions = mask.isSelectAll() ? null : mask.getSelectedPositions(); + for (int index = 0; index < selectedCount; index++) { + int position = mask.isSelectAll() ? index : selectedPositions[index]; + if (arguments[0].isNull(position)) { + continue; + } + double value = + RateFunctionValidation.readValue( + arguments[0], position, valueDataType, RateFunctionType.RATE); + long time = + RateFunctionValidation.readRequiredTime(arguments[1], position, RateFunctionType.RATE, 2); + long currentWindowStart = + RateFunctionValidation.readRequiredTime(arguments[2], position, RateFunctionType.RATE, 3); + long currentWindowEnd = + RateFunctionValidation.readRequiredTime(arguments[3], position, RateFunctionType.RATE, 4); + RateFunctionValidation.validateWindow( + RateFunctionType.RATE, time, currentWindowStart, currentWindowEnd); + initializeOrValidateWindow(currentWindowStart, currentWindowEnd); + samples.add(time, value); + } + updateMemoryReservation(); + } + + @Override + public void addIntermediate(Column argument) { + for (int position = 0; position < argument.getPositionCount(); position++) { + if (argument.isNull(position)) { + continue; + } + RateFunctionIntermediateStateCodec.DecodedState decoded = + RateFunctionIntermediateStateCodec.decode( + RateFunctionType.RATE, argument.getBinary(position)); + initializeOrValidateWindow(decoded.getWindowStart(), decoded.getWindowEnd()); + samples.merge(decoded.getSamples()); + } + updateMemoryReservation(); + } + + @Override + public void evaluateIntermediate(ColumnBuilder output) { + RateFunctionIntermediateStateCodec.encode( + RateFunctionType.RATE, windowStart, windowEnd, samples, output); + } + + @Override + public void evaluateFinal(ColumnBuilder output) { + samples.sortAndValidate(RateFunctionType.RATE.getFunctionName()); + if (samples.size() < 2) { + output.appendNull(); + return; + } + int lastIndex = samples.size() - 1; + double correctedIncrease = calculateCorrectedIncrease(); + output.writeDouble( + calculateRate( + samples.size(), + samples.getTime(0), + samples.getValue(0), + samples.getTime(lastIndex), + correctedIncrease)); + } + + @Override + public boolean hasFinalResult() { + return false; + } + + @Override + public void addStatistics(Statistics[] statistics) { + throw new UnsupportedOperationException(); + } + + @Override + public void reset() { + resetWindow(); + samples.reset(); + updateMemoryReservation(); + } + + private double calculateCorrectedIncrease() { + double result = 0.0; + for (int index = 1; index < samples.size(); index++) { + double previous = samples.getValue(index - 1); + double current = samples.getValue(index); + result = validateFinite(result + (current >= previous ? current - previous : current)); + } + return result; + } + + private void updateMemoryReservation() { + long currentSize = samples.getEstimatedSize(); + long delta = currentSize - previousSamplesSize; + if (delta > 0) { + memoryReservationManager.reserveMemoryCumulatively(delta); + } else if (delta < 0) { + memoryReservationManager.releaseMemoryCumulatively(-delta); + } + previousSamplesSize = currentSize; + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/OrderedDeltaAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/OrderedDeltaAccumulator.java new file mode 100644 index 0000000000000..e3240f383be44 --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/OrderedDeltaAccumulator.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate; + +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.AggregationMask; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.TableAccumulator; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.utils.RamUsageEstimator; + +public final class OrderedDeltaAccumulator extends AbstractDeltaTableAccumulator { + + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(OrderedDeltaAccumulator.class); + + private int sampleCount; + private long firstTime; + private double firstValue; + private long lastTime; + private double lastValue; + + public OrderedDeltaAccumulator(TSDataType valueDataType) { + super(valueDataType); + } + + @Override + public long getEstimatedSize() { + return INSTANCE_SIZE; + } + + @Override + public TableAccumulator copy() { + return new OrderedDeltaAccumulator(valueDataType); + } + + @Override + public void addInput(Column[] arguments, AggregationMask mask) { + RateFunctionValidation.validateArgumentCount(arguments, RateFunctionType.DELTA); + int selectedCount = mask.getSelectedPositionCount(); + int[] selectedPositions = mask.isSelectAll() ? null : mask.getSelectedPositions(); + for (int index = 0; index < selectedCount; index++) { + int position = mask.isSelectAll() ? index : selectedPositions[index]; + if (arguments[0].isNull(position)) { + continue; + } + double value = + RateFunctionValidation.readValue( + arguments[0], position, valueDataType, RateFunctionType.DELTA); + long time = + RateFunctionValidation.readRequiredTime( + arguments[1], position, RateFunctionType.DELTA, 2); + long currentWindowStart = + RateFunctionValidation.readRequiredTime( + arguments[2], position, RateFunctionType.DELTA, 3); + long currentWindowEnd = + RateFunctionValidation.readRequiredTime( + arguments[3], position, RateFunctionType.DELTA, 4); + RateFunctionValidation.validateWindow( + RateFunctionType.DELTA, time, currentWindowStart, currentWindowEnd); + initializeOrValidateWindow(currentWindowStart, currentWindowEnd); + update(time, value); + } + } + + @Override + public void addIntermediate(Column argument) { + throw unsupportedIntermediate(); + } + + @Override + public void evaluateIntermediate(ColumnBuilder columnBuilder) { + throw unsupportedIntermediate(); + } + + @Override + public void evaluateFinal(ColumnBuilder output) { + if (sampleCount < 2) { + output.appendNull(); + return; + } + output.writeDouble(calculateDelta(sampleCount, firstTime, firstValue, lastTime, lastValue)); + } + + @Override + public boolean hasFinalResult() { + return false; + } + + @Override + public void addStatistics(Statistics[] statistics) { + throw new UnsupportedOperationException(); + } + + @Override + public void reset() { + resetWindow(); + sampleCount = 0; + firstTime = 0; + firstValue = 0; + lastTime = 0; + lastValue = 0; + } + + private void update(long time, double value) { + if (sampleCount == 0) { + firstTime = time; + firstValue = value; + lastTime = time; + lastValue = value; + sampleCount = 1; + return; + } + if (time == lastTime) { + throw duplicateTimestamp(time); + } + if (time < lastTime) { + throw orderedInputViolation(time, lastTime); + } + lastTime = time; + lastValue = value; + sampleCount = Math.incrementExact(sampleCount); + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/OrderedIncreaseAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/OrderedIncreaseAccumulator.java new file mode 100644 index 0000000000000..8de972af4cf03 --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/OrderedIncreaseAccumulator.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate; + +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.AggregationMask; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.TableAccumulator; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.utils.RamUsageEstimator; + +public final class OrderedIncreaseAccumulator extends AbstractIncreaseTableAccumulator { + + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(OrderedIncreaseAccumulator.class); + + private int sampleCount; + private long firstTime; + private double firstValue; + private long lastTime; + private double lastValue; + private double correctedIncrease; + + public OrderedIncreaseAccumulator(TSDataType valueDataType) { + super(valueDataType); + } + + @Override + public long getEstimatedSize() { + return INSTANCE_SIZE; + } + + @Override + public TableAccumulator copy() { + return new OrderedIncreaseAccumulator(valueDataType); + } + + @Override + public void addInput(Column[] arguments, AggregationMask mask) { + RateFunctionValidation.validateArgumentCount(arguments, RateFunctionType.INCREASE); + int selectedCount = mask.getSelectedPositionCount(); + int[] selectedPositions = mask.isSelectAll() ? null : mask.getSelectedPositions(); + for (int index = 0; index < selectedCount; index++) { + int position = mask.isSelectAll() ? index : selectedPositions[index]; + if (arguments[0].isNull(position)) { + continue; + } + double value = + RateFunctionValidation.readValue( + arguments[0], position, valueDataType, RateFunctionType.INCREASE); + long time = + RateFunctionValidation.readRequiredTime( + arguments[1], position, RateFunctionType.INCREASE, 2); + long currentWindowStart = + RateFunctionValidation.readRequiredTime( + arguments[2], position, RateFunctionType.INCREASE, 3); + long currentWindowEnd = + RateFunctionValidation.readRequiredTime( + arguments[3], position, RateFunctionType.INCREASE, 4); + RateFunctionValidation.validateWindow( + RateFunctionType.INCREASE, time, currentWindowStart, currentWindowEnd); + initializeOrValidateWindow(currentWindowStart, currentWindowEnd); + update(time, value); + } + } + + @Override + public void addIntermediate(Column argument) { + throw unsupportedIntermediate(); + } + + @Override + public void evaluateIntermediate(ColumnBuilder columnBuilder) { + throw unsupportedIntermediate(); + } + + @Override + public void evaluateFinal(ColumnBuilder output) { + if (sampleCount < 2) { + output.appendNull(); + return; + } + output.writeDouble( + calculateIncrease(sampleCount, firstTime, firstValue, lastTime, correctedIncrease)); + } + + @Override + public boolean hasFinalResult() { + return false; + } + + @Override + public void addStatistics(Statistics[] statistics) { + throw new UnsupportedOperationException(); + } + + @Override + public void reset() { + resetWindow(); + sampleCount = 0; + firstTime = 0; + firstValue = 0; + lastTime = 0; + lastValue = 0; + correctedIncrease = 0; + } + + private void update(long time, double value) { + if (sampleCount == 0) { + firstTime = time; + firstValue = value; + lastTime = time; + lastValue = value; + sampleCount = 1; + return; + } + if (time == lastTime) { + throw duplicateTimestamp(time); + } + if (time < lastTime) { + throw orderedInputViolation(time, lastTime); + } + double increment = value >= lastValue ? value - lastValue : value; + double newCorrectedIncrease = validateFinite(correctedIncrease + increment); + int newSampleCount = Math.incrementExact(sampleCount); + lastTime = time; + lastValue = value; + correctedIncrease = newCorrectedIncrease; + sampleCount = newSampleCount; + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/OrderedIrateAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/OrderedIrateAccumulator.java new file mode 100644 index 0000000000000..d8aa6ce517d6e --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/OrderedIrateAccumulator.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate; + +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.AggregationMask; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.TableAccumulator; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.utils.RamUsageEstimator; + +public final class OrderedIrateAccumulator extends AbstractIrateTableAccumulator { + + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(OrderedIrateAccumulator.class); + + private int sampleCount; + private long previousTime; + private double previousValue; + private long lastTime; + private double lastValue; + + public OrderedIrateAccumulator(TSDataType valueDataType) { + super(valueDataType); + } + + @Override + public long getEstimatedSize() { + return INSTANCE_SIZE; + } + + @Override + public TableAccumulator copy() { + return new OrderedIrateAccumulator(valueDataType); + } + + @Override + public void addInput(Column[] arguments, AggregationMask mask) { + RateFunctionValidation.validateArgumentCount(arguments, RateFunctionType.IRATE); + int selectedCount = mask.getSelectedPositionCount(); + int[] selectedPositions = mask.isSelectAll() ? null : mask.getSelectedPositions(); + for (int index = 0; index < selectedCount; index++) { + int position = mask.isSelectAll() ? index : selectedPositions[index]; + if (arguments[0].isNull(position)) { + continue; + } + double value = + RateFunctionValidation.readValue( + arguments[0], position, valueDataType, RateFunctionType.IRATE); + long time = + RateFunctionValidation.readRequiredTime( + arguments[1], position, RateFunctionType.IRATE, 2); + update(time, value); + } + } + + @Override + public void addIntermediate(Column argument) { + throw unsupportedIntermediate(); + } + + @Override + public void evaluateIntermediate(ColumnBuilder columnBuilder) { + throw unsupportedIntermediate(); + } + + @Override + public void evaluateFinal(ColumnBuilder output) { + if (sampleCount < 2) { + output.appendNull(); + return; + } + output.writeDouble(calculateIrate(previousTime, previousValue, lastTime, lastValue)); + } + + @Override + public boolean hasFinalResult() { + return false; + } + + @Override + public void addStatistics(Statistics[] statistics) { + throw new UnsupportedOperationException(); + } + + @Override + public void reset() { + sampleCount = 0; + previousTime = 0; + previousValue = 0; + lastTime = 0; + lastValue = 0; + } + + private void update(long time, double value) { + if (sampleCount == 0) { + lastTime = time; + lastValue = value; + sampleCount = 1; + return; + } + if (time == lastTime) { + throw duplicateTimestamp(time); + } + if (time < lastTime) { + throw orderedInputViolation(time, lastTime); + } + previousTime = lastTime; + previousValue = lastValue; + lastTime = time; + lastValue = value; + sampleCount = Math.incrementExact(sampleCount); + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/OrderedRateAccumulator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/OrderedRateAccumulator.java new file mode 100644 index 0000000000000..228a1b0a326e4 --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/OrderedRateAccumulator.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate; + +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.AggregationMask; +import org.apache.iotdb.calc.execution.operator.source.relational.aggregation.TableAccumulator; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.utils.RamUsageEstimator; + +public final class OrderedRateAccumulator extends AbstractRateTableAccumulator { + + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(OrderedRateAccumulator.class); + + private int sampleCount; + private long firstTime; + private double firstValue; + private long lastTime; + private double lastValue; + private double correctedIncrease; + + public OrderedRateAccumulator(TSDataType valueDataType) { + super(valueDataType); + } + + @Override + public long getEstimatedSize() { + return INSTANCE_SIZE; + } + + @Override + public TableAccumulator copy() { + return new OrderedRateAccumulator(valueDataType); + } + + @Override + public void addInput(Column[] arguments, AggregationMask mask) { + RateFunctionValidation.validateArgumentCount(arguments, RateFunctionType.RATE); + int selectedCount = mask.getSelectedPositionCount(); + int[] selectedPositions = mask.isSelectAll() ? null : mask.getSelectedPositions(); + for (int index = 0; index < selectedCount; index++) { + int position = mask.isSelectAll() ? index : selectedPositions[index]; + if (arguments[0].isNull(position)) { + continue; + } + double value = + RateFunctionValidation.readValue( + arguments[0], position, valueDataType, RateFunctionType.RATE); + long time = + RateFunctionValidation.readRequiredTime(arguments[1], position, RateFunctionType.RATE, 2); + long currentWindowStart = + RateFunctionValidation.readRequiredTime(arguments[2], position, RateFunctionType.RATE, 3); + long currentWindowEnd = + RateFunctionValidation.readRequiredTime(arguments[3], position, RateFunctionType.RATE, 4); + RateFunctionValidation.validateWindow( + RateFunctionType.RATE, time, currentWindowStart, currentWindowEnd); + initializeOrValidateWindow(currentWindowStart, currentWindowEnd); + update(time, value); + } + } + + @Override + public void addIntermediate(Column argument) { + throw unsupportedIntermediate(); + } + + @Override + public void evaluateIntermediate(ColumnBuilder columnBuilder) { + throw unsupportedIntermediate(); + } + + @Override + public void evaluateFinal(ColumnBuilder output) { + if (sampleCount < 2) { + output.appendNull(); + return; + } + output.writeDouble( + calculateRate(sampleCount, firstTime, firstValue, lastTime, correctedIncrease)); + } + + @Override + public boolean hasFinalResult() { + return false; + } + + @Override + public void addStatistics(Statistics[] statistics) { + throw new UnsupportedOperationException(); + } + + @Override + public void reset() { + resetWindow(); + sampleCount = 0; + firstTime = 0; + firstValue = 0; + lastTime = 0; + lastValue = 0; + correctedIncrease = 0; + } + + private void update(long time, double value) { + if (sampleCount == 0) { + firstTime = time; + firstValue = value; + lastTime = time; + lastValue = value; + sampleCount = 1; + return; + } + if (time == lastTime) { + throw duplicateTimestamp(time); + } + if (time < lastTime) { + throw orderedInputViolation(time, lastTime); + } + double increment = value >= lastValue ? value - lastValue : value; + double newCorrectedIncrease = validateFinite(correctedIncrease + increment); + int newSampleCount = Math.incrementExact(sampleCount); + lastTime = time; + lastValue = value; + correctedIncrease = newCorrectedIncrease; + sampleCount = newSampleCount; + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/RateFunctionIntermediateStateCodec.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/RateFunctionIntermediateStateCodec.java new file mode 100644 index 0000000000000..18476f42a91a8 --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/RateFunctionIntermediateStateCodec.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate; + +import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.commons.exception.SemanticException; + +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.utils.Binary; + +import java.nio.ByteBuffer; + +public final class RateFunctionIntermediateStateCodec { + + private static final int STATE_VERSION = 1; + private static final int WINDOWED_HEADER_SIZE = 24; + private static final int IRATE_HEADER_SIZE = 8; + private static final int SAMPLE_SIZE = 16; + + private RateFunctionIntermediateStateCodec() {} + + public static void encode( + RateFunctionType functionType, + long windowStart, + long windowEnd, + TimeValueBuffer samples, + ColumnBuilder output) { + if (samples == null || samples.isEmpty()) { + output.appendNull(); + return; + } + + long headerSize = functionType.isWindowed() ? WINDOWED_HEADER_SIZE : IRATE_HEADER_SIZE; + long serializedSize = + Math.addExact(headerSize, Math.multiplyExact((long) samples.size(), SAMPLE_SIZE)); + ByteBuffer target = ByteBuffer.allocate(Math.toIntExact(serializedSize)); + target.putInt(STATE_VERSION); + if (functionType.isWindowed()) { + target.putLong(windowStart); + target.putLong(windowEnd); + } + target.putInt(samples.size()); + samples.writePayload(target); + output.writeBinary(new Binary(target.array())); + } + + public static DecodedState decode(RateFunctionType functionType, Binary binary) { + try { + byte[] bytes = binary.getValues(); + int headerSize = functionType.isWindowed() ? WINDOWED_HEADER_SIZE : IRATE_HEADER_SIZE; + if (bytes.length < headerSize) { + throw invalidState(functionType); + } + ByteBuffer source = ByteBuffer.wrap(bytes); + if (source.getInt() != STATE_VERSION) { + throw invalidState(functionType); + } + + long windowStart = 0; + long windowEnd = 0; + if (functionType.isWindowed()) { + windowStart = source.getLong(); + windowEnd = source.getLong(); + if (windowStart >= windowEnd) { + throw invalidState(functionType); + } + } + + int sampleCount = source.getInt(); + if (sampleCount < 0 + || Math.addExact(headerSize, Math.multiplyExact(sampleCount, SAMPLE_SIZE)) + != bytes.length) { + throw invalidState(functionType); + } + + TimeValueBuffer samples = new TimeValueBuffer(); + for (int index = 0; index < sampleCount; index++) { + long time = source.getLong(); + double value = source.getDouble(); + if (!Double.isFinite(value) + || (functionType.isCounter() && value < 0.0) + || (functionType.isWindowed() && (time < windowStart || time >= windowEnd))) { + throw invalidState(functionType); + } + samples.add(time, value); + } + if (source.hasRemaining()) { + throw invalidState(functionType); + } + return new DecodedState(windowStart, windowEnd, samples); + } catch (ArithmeticException | IndexOutOfBoundsException exception) { + throw invalidState(functionType); + } + } + + private static SemanticException invalidState(RateFunctionType functionType) { + return new SemanticException( + String.format( + CalcMessages.EXCEPTION_INVALID_INTERMEDIATE_STATE_FOR_AGGREGATE_FUNCTION_ARG_2999C30B, + functionType.getFunctionName())); + } + + public static final class DecodedState { + private final long windowStart; + private final long windowEnd; + private final TimeValueBuffer samples; + + private DecodedState(long windowStart, long windowEnd, TimeValueBuffer samples) { + this.windowStart = windowStart; + this.windowEnd = windowEnd; + this.samples = samples; + } + + public long getWindowStart() { + return windowStart; + } + + public long getWindowEnd() { + return windowEnd; + } + + public TimeValueBuffer getSamples() { + return samples; + } + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/RateFunctionType.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/RateFunctionType.java new file mode 100644 index 0000000000000..db86ccfa0e99d --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/RateFunctionType.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate; + +public enum RateFunctionType { + RATE("rate", true, true), + INCREASE("increase", true, true), + IRATE("irate", true, false), + DELTA("delta", false, true); + + private final String functionName; + private final boolean counter; + private final boolean windowed; + + RateFunctionType(String functionName, boolean counter, boolean windowed) { + this.functionName = functionName; + this.counter = counter; + this.windowed = windowed; + } + + public String getFunctionName() { + return functionName; + } + + public boolean isCounter() { + return counter; + } + + public boolean isWindowed() { + return windowed; + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/RateFunctionValidation.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/RateFunctionValidation.java new file mode 100644 index 0000000000000..88f9ce627fa30 --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/RateFunctionValidation.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate; + +import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.commons.exception.SemanticException; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.enums.TSDataType; + +public final class RateFunctionValidation { + + private RateFunctionValidation() {} + + public static void validateArgumentCount(Column[] arguments, RateFunctionType functionType) { + int expected = functionType.isWindowed() ? 4 : 2; + if (arguments.length != expected) { + throw new SemanticException( + String.format( + CalcMessages + .EXCEPTION_AGGREGATE_FUNCTION_ARG_REQUIRES_ARG_ARGUMENTS_BUT_GOT_ARG_D6249DD2, + functionType.getFunctionName(), + expected, + arguments.length)); + } + } + + public static double readValue( + Column column, int position, TSDataType valueDataType, RateFunctionType functionType) { + double value; + switch (valueDataType) { + case INT32: + value = column.getInt(position); + break; + case INT64: + value = column.getLong(position); + break; + case FLOAT: + value = column.getFloat(position); + break; + case DOUBLE: + value = column.getDouble(position); + break; + default: + throw new SemanticException( + String.format( + CalcMessages + .EXCEPTION_AGGREGATE_FUNCTION_ARG_DOES_NOT_SUPPORT_VALUE_TYPE_ARG_9DD7388D, + functionType.getFunctionName(), + valueDataType)); + } + + if (!Double.isFinite(value)) { + throw new SemanticException( + String.format( + CalcMessages + .EXCEPTION_AGGREGATE_FUNCTION_ARG_DOES_NOT_SUPPORT_NON_FINITE_VALUE_COL_ARG_AC2AAC62, + functionType.getFunctionName(), + value)); + } + if (functionType.isCounter() && value < 0.0) { + throw new SemanticException( + String.format( + CalcMessages + .EXCEPTION_THE_VALUE_COL_ARGUMENT_OF_AGGREGATE_FUNCTION_ARG_MUST_BE_A_NON_NEGATIVE_NUMBER_BUT_GOT_ARG_4D5B7D74, + functionType.getFunctionName(), + value)); + } + return value; + } + + public static long readRequiredTime( + Column column, int position, RateFunctionType functionType, int argumentPosition) { + if (column.isNull(position)) { + throw new SemanticException( + String.format( + CalcMessages + .EXCEPTION_THE_ARGUMENT_ARG_OF_AGGREGATE_FUNCTION_ARG_MUST_NOT_BE_NULL_WHEN_VALUE_COL_IS_NOT_NULL_7F087E99, + argumentPosition, + functionType.getFunctionName())); + } + return column.getLong(position); + } + + public static void validateWindow( + RateFunctionType functionType, long time, long windowStart, long windowEnd) { + if (windowStart >= windowEnd) { + throw new SemanticException( + String.format( + CalcMessages + .EXCEPTION_THE_WINDOW_START_ARGUMENT_OF_AGGREGATE_FUNCTION_ARG_MUST_BE_LESS_THAN_WINDOW_END_17D2A79A, + functionType.getFunctionName())); + } + if (time < windowStart || time >= windowEnd) { + throw new SemanticException( + String.format( + CalcMessages + .EXCEPTION_THE_SAMPLE_TIME_OF_AGGREGATE_FUNCTION_ARG_MUST_SATISFY_WINDOW_START_TIME_COL_WINDOW_END_35014D15, + functionType.getFunctionName())); + } + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/TimeValueBuffer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/TimeValueBuffer.java new file mode 100644 index 0000000000000..8cd76705a674c --- /dev/null +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/source/relational/aggregation/rate/TimeValueBuffer.java @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.source.relational.aggregation.rate; + +import org.apache.iotdb.calc.i18n.CalcMessages; +import org.apache.iotdb.commons.exception.SemanticException; + +import org.apache.tsfile.utils.RamUsageEstimator; + +import java.nio.ByteBuffer; +import java.util.Arrays; + +public final class TimeValueBuffer { + + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(TimeValueBuffer.class); + private static final int INITIAL_CAPACITY = 16; + private static final int INSERTION_SORT_THRESHOLD = 16; + + private long[] times = new long[INITIAL_CAPACITY]; + private double[] values = new double[INITIAL_CAPACITY]; + private int size; + private boolean sortedAndValidated; + + public void add(long time, double value) { + ensureCapacity(Math.addExact(size, 1)); + times[size] = time; + values[size] = value; + size++; + sortedAndValidated = false; + } + + public void merge(TimeValueBuffer other) { + if (other == null || other.size == 0) { + return; + } + int newSize = Math.addExact(size, other.size); + ensureCapacity(newSize); + System.arraycopy(other.times, 0, times, size, other.size); + System.arraycopy(other.values, 0, values, size, other.size); + size = newSize; + sortedAndValidated = false; + } + + public int size() { + return size; + } + + public boolean isEmpty() { + return size == 0; + } + + public long getTime(int index) { + return times[index]; + } + + public double getValue(int index) { + return values[index]; + } + + public long getEstimatedSize() { + return INSTANCE_SIZE + + RamUsageEstimator.sizeOfLongArray(times.length) + + RamUsageEstimator.sizeOfDoubleArray(values.length); + } + + public void reset() { + size = 0; + sortedAndValidated = false; + } + + public void writePayload(ByteBuffer target) { + for (int index = 0; index < size; index++) { + target.putLong(times[index]); + target.putDouble(values[index]); + } + } + + public void sortAndValidate(String functionName) { + if (sortedAndValidated) { + return; + } + if (size > 1) { + introSort(0, size, 2 * floorLog2(size)); + for (int index = 1; index < size; index++) { + if (times[index - 1] == times[index]) { + throw new SemanticException( + String.format( + CalcMessages + .EXCEPTION_AGGREGATE_FUNCTION_ARG_DOES_NOT_SUPPORT_DUPLICATE_TIME_COL_VALUES_IN_THE_SAME_AGGREGATION_GROUP_ARG_087A91BC, + functionName, + times[index])); + } + } + } + sortedAndValidated = true; + } + + private void ensureCapacity(int requiredCapacity) { + if (requiredCapacity <= times.length) { + return; + } + int newCapacity = Math.max(requiredCapacity, Math.multiplyExact(times.length, 2)); + times = Arrays.copyOf(times, newCapacity); + values = Arrays.copyOf(values, newCapacity); + } + + private void introSort(int from, int to, int depthLimit) { + int length = to - from; + if (length <= INSERTION_SORT_THRESHOLD) { + insertionSort(from, to); + return; + } + if (depthLimit == 0) { + heapSort(from, to); + return; + } + + long pivot = median(times[from], times[from + length / 2], times[to - 1]); + int left = from - 1; + int right = to; + while (true) { + do { + left++; + } while (times[left] < pivot); + do { + right--; + } while (times[right] > pivot); + if (left >= right) { + break; + } + swap(left, right); + } + + introSort(from, right + 1, depthLimit - 1); + introSort(right + 1, to, depthLimit - 1); + } + + private void insertionSort(int from, int to) { + for (int index = from + 1; index < to; index++) { + long time = times[index]; + double value = values[index]; + int position = index - 1; + while (position >= from && times[position] > time) { + times[position + 1] = times[position]; + values[position + 1] = values[position]; + position--; + } + times[position + 1] = time; + values[position + 1] = value; + } + } + + private void heapSort(int from, int to) { + int length = to - from; + for (int root = length / 2 - 1; root >= 0; root--) { + siftDown(from, root, length); + } + for (int end = length - 1; end > 0; end--) { + swap(from, from + end); + siftDown(from, 0, end); + } + } + + private void siftDown(int base, int root, int length) { + while (root * 2 + 1 < length) { + int child = root * 2 + 1; + if (child + 1 < length && times[base + child] < times[base + child + 1]) { + child++; + } + if (times[base + root] >= times[base + child]) { + return; + } + swap(base + root, base + child); + root = child; + } + } + + private void swap(int left, int right) { + long time = times[left]; + times[left] = times[right]; + times[right] = time; + + double value = values[left]; + values[left] = values[right]; + values[right] = value; + } + + private static long median(long first, long second, long third) { + if (first < second) { + return second < third ? second : Math.max(first, third); + } + return first < third ? first : Math.max(second, third); + } + + private static int floorLog2(int value) { + return Integer.SIZE - 1 - Integer.numberOfLeadingZeros(value); + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/TableOperatorGenerator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/TableOperatorGenerator.java index a2782ac0798e3..ed52de7958b0d 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/planner/TableOperatorGenerator.java @@ -1448,6 +1448,8 @@ protected TableAggregator buildAggregator( timeColumnName, measurementColumnNames, aggregation.isDistinct(), + step, + aggregation.isInputOrderedByTimeAscending(), memoryReservationManager, ioTDBLocal); @@ -1731,6 +1733,8 @@ protected GroupedAggregator buildGroupByAggregator( Collections.emptyMap(), true, aggregation.isDistinct(), + step, + aggregation.isInputOrderedByTimeAscending(), memoryReservationManager, ioTDBLocal); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/utils/constant/SqlConstant.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/utils/constant/SqlConstant.java index d542528ec1edc..e08ddf253ff7e 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/utils/constant/SqlConstant.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/utils/constant/SqlConstant.java @@ -89,6 +89,10 @@ protected SqlConstant() { public static final String APPROX_MOST_FREQUENT = "approx_most_frequent"; public static final String APPROX_PERCENTILE = "approx_percentile"; public static final String PERCENTILE = "percentile"; + public static final String RATE = "rate"; + public static final String INCREASE = "increase"; + public static final String IRATE = "irate"; + public static final String DELTA = "delta"; // names of scalar functions public static final String DIFF = "diff"; diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 92242b62aa726..a4196d9ddeb0b 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -3741,6 +3741,10 @@ private DataNodeQueryMessages() {} public static final String EXCEPTION_AGGREGATION_FUNCTIONS_ARG_SHOULD_ONLY_HAVE_TWO_ARGUMENTS_3D12DCFD = "Aggregation functions [%s] should only have two arguments"; public static final String EXCEPTION_AGGREGATION_FUNCTIONS_ARG_SHOULD_HAVE_VALUE_COLUMN_AS_NUMERIC_TYPE_INT32_INT64_FLOAT_DOUBLE_TIMESTAMP_97A6CA87 = "Aggregation functions [%s] should have value column as numeric type [INT32, INT64, FLOAT, DOUBLE, TIMESTAMP]"; public static final String EXCEPTION_AGGREGATION_FUNCTIONS_ARG_SHOULD_HAVE_PERCENTAGE_AS_DECIMAL_TYPE_57033ADF = "Aggregation functions [%s] should have percentage as decimal type"; + public static final String EXCEPTION_AGGREGATE_FUNCTION_ARG_REQUIRES_2_ARGUMENTS_VALUE_TIME_E2F55C08 = "Aggregate function [%s] requires 2 arguments: value, time"; + public static final String EXCEPTION_AGGREGATE_FUNCTION_ARG_REQUIRES_4_ARGUMENTS_VALUE_TIME_WINDOW_START_WINDOW_END_FBEC794B = "Aggregate function [%s] requires 4 arguments: value, time, window_start, window_end"; + public static final String EXCEPTION_AGGREGATE_FUNCTION_ARG_ONLY_SUPPORTS_INT32_INT64_FLOAT_AND_DOUBLE_AS_THE_FIRST_ARGUMENT_8D201434 = "Aggregate function [%s] only supports INT32, INT64, FLOAT and DOUBLE as the first argument"; + public static final String EXCEPTION_THE_TIME_ARGUMENTS_OF_AGGREGATE_FUNCTION_ARG_SHOULD_BE_TIMESTAMP_OR_INT64_TYPE_9C736DE3 = "The time arguments of aggregate function [%s] should be TIMESTAMP or INT64 type"; public static final String EXCEPTION_CANNOT_NEST_AGGREGATIONS_INSIDE_AGGREGATION_ARG_ARG_6E5073A4 = "Cannot nest aggregations inside aggregation '%s': %s"; public static final String EXCEPTION_CANNOT_NEST_WINDOW_FUNCTIONS_INSIDE_AGGREGATION_ARG_ARG_8F94A897 = "Cannot nest window functions inside aggregation '%s': %s"; public static final String EXCEPTION_PARTITION_BY_EXPRESSION_ARG_MUST_BE_AN_AGGREGATE_EXPRESSION_OR_APPEAR_IN_GROUP_BY_CLAUSE_E3C696D6 = "PARTITION BY expression '%s' must be an aggregate expression or appear in GROUP BY clause"; diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 9ce80200a7b75..21153ae457e43 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -4500,6 +4500,10 @@ private DataNodeQueryMessages() {} public static final String EXCEPTION_AGGREGATION_FUNCTIONS_ARG_SHOULD_ONLY_HAVE_TWO_ARGUMENTS_3D12DCFD = "聚合函数 [%s] 应当只有两个参数"; public static final String EXCEPTION_AGGREGATION_FUNCTIONS_ARG_SHOULD_HAVE_VALUE_COLUMN_AS_NUMERIC_TYPE_INT32_INT64_FLOAT_DOUBLE_TIMESTAMP_97A6CA87 = "聚合函数 [%s] 的值列应为数值类型 [INT32, INT64, FLOAT, DOUBLE, TIMESTAMP]"; public static final String EXCEPTION_AGGREGATION_FUNCTIONS_ARG_SHOULD_HAVE_PERCENTAGE_AS_DECIMAL_TYPE_57033ADF = "聚合函数 [%s] 的百分比参数应为 decimal 类型"; + public static final String EXCEPTION_AGGREGATE_FUNCTION_ARG_REQUIRES_2_ARGUMENTS_VALUE_TIME_E2F55C08 = "聚合函数 [%s] 需要 2 个参数：value、time"; + public static final String EXCEPTION_AGGREGATE_FUNCTION_ARG_REQUIRES_4_ARGUMENTS_VALUE_TIME_WINDOW_START_WINDOW_END_FBEC794B = "聚合函数 [%s] 需要 4 个参数：value、time、window_start、window_end"; + public static final String EXCEPTION_AGGREGATE_FUNCTION_ARG_ONLY_SUPPORTS_INT32_INT64_FLOAT_AND_DOUBLE_AS_THE_FIRST_ARGUMENT_8D201434 = "聚合函数 [%s] 的第一个参数仅支持 INT32、INT64、FLOAT 和 DOUBLE 类型"; + public static final String EXCEPTION_THE_TIME_ARGUMENTS_OF_AGGREGATE_FUNCTION_ARG_SHOULD_BE_TIMESTAMP_OR_INT64_TYPE_9C736DE3 = "聚合函数 [%s] 的时间参数应为 TIMESTAMP 或 INT64 类型"; public static final String EXCEPTION_CANNOT_NEST_AGGREGATIONS_INSIDE_AGGREGATION_ARG_ARG_6E5073A4 = "不能在聚合 '%s' 中嵌套聚合：%s"; public static final String EXCEPTION_CANNOT_NEST_WINDOW_FUNCTIONS_INSIDE_AGGREGATION_ARG_ARG_8F94A897 = "不能在聚合 '%s' 中嵌套窗口函数：%s"; public static final String EXCEPTION_PARTITION_BY_EXPRESSION_ARG_MUST_BE_AN_AGGREGATE_EXPRESSION_OR_APPEAR_IN_GROUP_BY_CLAUSE_E3C696D6 = "PARTITION BY 表达式 '%s' 必须是聚合表达式或出现在 GROUP BY 子句中"; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index 9a2119874bd3f..9e736a35c581a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@ -1439,6 +1439,24 @@ && isNumericType(argumentTypes.get(0)) functionName)); } break; + case SqlConstant.IRATE: + validateRateFunctionArguments( + functionName, + argumentTypes, + 2, + DataNodeQueryMessages + .EXCEPTION_AGGREGATE_FUNCTION_ARG_REQUIRES_2_ARGUMENTS_VALUE_TIME_E2F55C08); + break; + case SqlConstant.RATE: + case SqlConstant.INCREASE: + case SqlConstant.DELTA: + validateRateFunctionArguments( + functionName, + argumentTypes, + 4, + DataNodeQueryMessages + .EXCEPTION_AGGREGATE_FUNCTION_ARG_REQUIRES_4_ARGUMENTS_VALUE_TIME_WINDOW_START_WINDOW_END_FBEC794B); + break; case SqlConstant.COUNT: break; default: @@ -1480,6 +1498,10 @@ && isNumericType(argumentTypes.get(0)) case SqlConstant.REGR_INTERCEPT: case SqlConstant.SKEWNESS: case SqlConstant.KURTOSIS: + case SqlConstant.RATE: + case SqlConstant.INCREASE: + case SqlConstant.IRATE: + case SqlConstant.DELTA: return DOUBLE; case SqlConstant.APPROX_MOST_FREQUENT: return STRING; @@ -1609,6 +1631,33 @@ && isNumericType(argumentTypes.get(0)) throw new SemanticException(DataNodeQueryMessages.UNKNOWN_FUNCTION + functionName); } + private static void validateRateFunctionArguments( + String functionName, + List argumentTypes, + int expectedArgumentCount, + String argumentCountError) { + if (argumentTypes.size() != expectedArgumentCount) { + throw new SemanticException(String.format(argumentCountError, functionName)); + } + if (!CommonMetadataUtils.isSupportedMathNumericType(argumentTypes.get(0))) { + throw new SemanticException( + String.format( + DataNodeQueryMessages + .EXCEPTION_AGGREGATE_FUNCTION_ARG_ONLY_SUPPORTS_INT32_INT64_FLOAT_AND_DOUBLE_AS_THE_FIRST_ARGUMENT_8D201434, + functionName)); + } + for (int i = 1; i < argumentTypes.size(); i++) { + Type argumentType = argumentTypes.get(i); + if (!INT64.equals(argumentType) && !TIMESTAMP.equals(argumentType)) { + throw new SemanticException( + String.format( + DataNodeQueryMessages + .EXCEPTION_THE_TIME_ARGUMENTS_OF_AGGREGATE_FUNCTION_ARG_SHOULD_BE_TIMESTAMP_OR_INT64_TYPE_9C736DE3, + functionName)); + } + } + } + @Override public boolean isAggregationFunction( final SessionInfo session, final String functionName, final AccessControl accessControl) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index c731bee31cc9b..25845758a081b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -147,6 +147,10 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static org.apache.iotdb.calc.utils.constant.SqlConstant.COUNT; +import static org.apache.iotdb.calc.utils.constant.SqlConstant.DELTA; +import static org.apache.iotdb.calc.utils.constant.SqlConstant.INCREASE; +import static org.apache.iotdb.calc.utils.constant.SqlConstant.IRATE; +import static org.apache.iotdb.calc.utils.constant.SqlConstant.RATE; import static org.apache.iotdb.commons.partition.DataPartition.NOT_ASSIGNED; import static org.apache.iotdb.commons.queryengine.plan.relational.function.FunctionKind.AGGREGATE; import static org.apache.iotdb.commons.queryengine.plan.relational.metadata.FunctionNullability.getAggregationFunctionNullability; @@ -1320,9 +1324,10 @@ public List visitAggregation(AggregationNode node, PlanContext context // push down aggregation if the child of aggregation node only has the union Node if (childrenNodes.size() == 1) { node.setChild(childrenNodes.get(0)); + AggregationNode physicalAggregation = withRateFunctionInputOrdering(node, childOrdering); if (childrenNodes.get(0) instanceof UnionNode - && node.getAggregations().values().stream() + && physicalAggregation.getAggregations().values().stream() .noneMatch(aggregation -> aggregation.isDistinct() || aggregation.hasMask())) { UnionNode unionNode = (UnionNode) childrenNodes.get(0); List children = unionNode.getChildren(); @@ -1343,7 +1348,8 @@ public List visitAggregation(AggregationNode node, PlanContext context } // 2. split the aggregation into partial and final - Pair splitResult = split(node, symbolAllocator, queryId); + Pair splitResult = + split(physicalAggregation, symbolAllocator, queryId); AggregationNode intermediate = splitResult.right; // 3. add the aggregation node above the project node @@ -1362,7 +1368,7 @@ public List visitAggregation(AggregationNode node, PlanContext context intermediate.getStep(), intermediate.getHashSymbol(), intermediate.getGroupIdSymbol()); - if (node.isStreamable() && childOrdering != null) { + if (physicalAggregation.isStreamable() && childOrdering != null) { nodeOrderingMap.put(planNodeId, expectedOrderingSchema); } return aggregationNode; @@ -1378,7 +1384,7 @@ public List visitAggregation(AggregationNode node, PlanContext context return Collections.singletonList(splitResult.left); } - return Collections.singletonList(node); + return Collections.singletonList(physicalAggregation); } // We cannot do multi-stage Aggregate if any aggregation-function is distinct. @@ -1386,10 +1392,12 @@ public List visitAggregation(AggregationNode node, PlanContext context // MarkDistinctNode will merge all data from different child. if (node.getAggregations().values().stream() .anyMatch(aggregation -> aggregation.isDistinct() || aggregation.hasMask())) { - node.setChild( + PlanNode physicalChild = mergeChildrenViaCollectOrMergeSort( - nodeOrderingMap.get(childrenNodes.get(0).getPlanNodeId()), childrenNodes)); - return Collections.singletonList(node); + nodeOrderingMap.get(childrenNodes.get(0).getPlanNodeId()), childrenNodes); + node.setChild(physicalChild); + return Collections.singletonList( + withRateFunctionInputOrdering(node, nodeOrderingMap.get(physicalChild.getPlanNodeId()))); } Pair splitResult = split(node, symbolAllocator, queryId); AggregationNode intermediate = splitResult.right; @@ -1421,6 +1429,58 @@ public List visitAggregation(AggregationNode node, PlanContext context return Collections.singletonList(splitResult.left); } + private static AggregationNode withRateFunctionInputOrdering( + AggregationNode node, OrderingScheme childOrdering) { + Map aggregations = new LinkedHashMap<>(); + node.getAggregations() + .forEach( + (symbol, aggregation) -> + aggregations.put( + symbol, + new AggregationNode.Aggregation( + aggregation.getResolvedFunction(), + aggregation.getArguments(), + aggregation.isDistinct(), + aggregation.getFilter(), + aggregation.getOrderingScheme(), + aggregation.getMask(), + isInputOrderedByTimeAscending( + aggregation, node.getStep(), node.getGroupingKeys(), childOrdering)))); + return AggregationNode.builderFrom(node).setAggregations(aggregations).build(); + } + + static boolean isInputOrderedByTimeAscending( + AggregationNode.Aggregation aggregation, + AggregationNode.Step step, + List groupingKeys, + OrderingScheme childOrdering) { + String functionName = aggregation.getResolvedFunction().getSignature().getName(); + if (step != SINGLE + || childOrdering == null + || aggregation.getArguments().size() < 2 + || !(RATE.equalsIgnoreCase(functionName) + || INCREASE.equalsIgnoreCase(functionName) + || IRATE.equalsIgnoreCase(functionName) + || DELTA.equalsIgnoreCase(functionName))) { + return false; + } + + Symbol timeSymbol = Symbol.from(aggregation.getArguments().get(1)); + List orderBy = childOrdering.getOrderBy(); + int timeIndex = orderBy.indexOf(timeSymbol); + if (timeIndex < 0 || !childOrdering.getOrdering(timeSymbol).isAscending()) { + return false; + } + + Set groupingKeySet = new HashSet<>(groupingKeys); + for (int i = 0; i < timeIndex; i++) { + if (!groupingKeySet.contains(orderBy.get(i))) { + return false; + } + } + return true; + } + private boolean prefixMatched(OrderingScheme childOrdering, List preGroupedSymbols) { List orderKeys = childOrdering.getOrderBy(); if (orderKeys.size() < preGroupedSymbols.size()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushAggregationIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushAggregationIntoTableScan.java index d44ad05431da3..9f3c5c32e25a1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushAggregationIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushAggregationIntoTableScan.java @@ -49,6 +49,10 @@ import java.util.Objects; import java.util.stream.Collectors; +import static org.apache.iotdb.calc.utils.constant.SqlConstant.DELTA; +import static org.apache.iotdb.calc.utils.constant.SqlConstant.INCREASE; +import static org.apache.iotdb.calc.utils.constant.SqlConstant.IRATE; +import static org.apache.iotdb.calc.utils.constant.SqlConstant.RATE; import static org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory.TAG; import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationTableScanNode.combineAggregationAndTableScan; import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.Util.split; @@ -148,6 +152,9 @@ private PushDownLevel calculatePushDownLevel( boolean hasProject = projectNode != null; Map assignments = hasProject ? projectNode.getAssignments().getMap() : null; + if (containsRateFunction(values)) { + return PushDownLevel.NOOP; + } // calculate Function part for (AggregationNode.Aggregation aggregation : values) { if (aggregation.isDistinct()) { @@ -263,6 +270,17 @@ private boolean isDateBinFunctionOfTime( } } + static boolean containsRateFunction(Collection aggregations) { + return aggregations.stream() + .map(aggregation -> aggregation.getResolvedFunction().getSignature().getName()) + .anyMatch( + functionName -> + RATE.equalsIgnoreCase(functionName) + || INCREASE.equalsIgnoreCase(functionName) + || IRATE.equalsIgnoreCase(functionName) + || DELTA.equalsIgnoreCase(functionName)); + } + private enum PushDownLevel { NOOP, PARTIAL, diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/relational/planner/node/AggregationNode.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/relational/planner/node/AggregationNode.java index 688dc0b4f0461..4d0b83eaa2097 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/relational/planner/node/AggregationNode.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/relational/planner/node/AggregationNode.java @@ -508,6 +508,7 @@ public static class Aggregation { private final Optional filter; private final Optional orderingScheme; private final Optional mask; + private final boolean inputOrderedByTimeAscending; public Aggregation( ResolvedFunction resolvedFunction, @@ -516,6 +517,17 @@ public Aggregation( Optional filter, Optional orderingScheme, Optional mask) { + this(resolvedFunction, arguments, distinct, filter, orderingScheme, mask, false); + } + + public Aggregation( + ResolvedFunction resolvedFunction, + List arguments, + boolean distinct, + Optional filter, + Optional orderingScheme, + Optional mask, + boolean inputOrderedByTimeAscending) { this.resolvedFunction = requireNonNull( resolvedFunction, QueryMessages.EXCEPTION_RESOLVEDFUNCTION_IS_NULL_81B5B93A); @@ -533,6 +545,7 @@ public Aggregation( this.orderingScheme = requireNonNull(orderingScheme, QueryMessages.EXCEPTION_ORDERINGSCHEME_IS_NULL_4D4D2F6F); this.mask = requireNonNull(mask, QueryMessages.EXCEPTION_MASK_IS_NULL_5A7CEA49); + this.inputOrderedByTimeAscending = inputOrderedByTimeAscending; } public ResolvedFunction getResolvedFunction() { @@ -559,6 +572,10 @@ public Optional getMask() { return mask; } + public boolean isInputOrderedByTimeAscending() { + return inputOrderedByTimeAscending; + } + public boolean hasMask() { return mask.isPresent(); } @@ -573,6 +590,7 @@ public boolean equals(Object o) { } Aggregation that = (Aggregation) o; return distinct == that.distinct + && inputOrderedByTimeAscending == that.inputOrderedByTimeAscending && Objects.equals(resolvedFunction, that.resolvedFunction) && Objects.equals(arguments, that.arguments) && Objects.equals(filter, that.filter) @@ -582,7 +600,14 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(resolvedFunction, arguments, distinct, filter, orderingScheme, mask); + return Objects.hash( + resolvedFunction, + arguments, + distinct, + filter, + orderingScheme, + mask, + inputOrderedByTimeAscending); } public void verifyArguments(Step step) { @@ -622,6 +647,7 @@ public void serialize(ByteBuffer byteBuffer) { orderingScheme.ifPresent(scheme -> scheme.serialize(byteBuffer)); ReadWriteIOUtils.write(mask.isPresent(), byteBuffer); mask.ifPresent(symbol -> Symbol.serialize(symbol, byteBuffer)); + ReadWriteIOUtils.write(inputOrderedByTimeAscending, byteBuffer); } public void serialize(DataOutputStream stream) throws IOException { @@ -643,6 +669,7 @@ public void serialize(DataOutputStream stream) throws IOException { if (mask.isPresent()) { Symbol.serialize(mask.get(), stream); } + ReadWriteIOUtils.write(inputOrderedByTimeAscending, stream); } public static Aggregation deserialize(ByteBuffer byteBuffer) { @@ -665,7 +692,9 @@ public static Aggregation deserialize(ByteBuffer byteBuffer) { if (ReadWriteIOUtils.readBool(byteBuffer)) { mask = Optional.of(Symbol.deserialize(byteBuffer)); } - return new Aggregation(function, arguments, distinct, filter, orderingScheme, mask); + boolean inputOrderedByTimeAscending = ReadWriteIOUtils.readBool(byteBuffer); + return new Aggregation( + function, arguments, distinct, filter, orderingScheme, mask, inputOrderedByTimeAscending); } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinAggregationFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinAggregationFunction.java index e8c5806fbc526..d232b7b77ef63 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinAggregationFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinAggregationFunction.java @@ -67,7 +67,11 @@ public enum TableBuiltinAggregationFunction { REGR_INTERCEPT("regr_intercept"), SKEWNESS("skewness"), KURTOSIS("kurtosis"), - PERCENTILE("percentile"); + PERCENTILE("percentile"), + RATE("rate"), + INCREASE("increase"), + IRATE("irate"), + DELTA("delta"); private final String functionName; @@ -123,6 +127,11 @@ public static Type getIntermediateType(String name, List originalArgumentT case "approx_percentile": case "percentile": return RowType.anonymous(Collections.emptyList()); + case "rate": + case "increase": + case "irate": + case "delta": + return BLOB; case "extreme": case "max": case "min": diff --git a/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift b/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift index d52dc08813fbd..bdde4fbea5d3c 100644 --- a/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift +++ b/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift @@ -319,6 +319,10 @@ enum TAggregationType { SKEWNESS, KURTOSIS PERCENTILE, + RATE, + INCREASE, + IRATE, + DELTA, } struct TShowConfigurationTemplateResp {