Kouhei Sutou
null+****@clear*****
Tue Oct 31 18:02:53 JST 2017
Kouhei Sutou 2017-10-31 18:02:53 +0900 (Tue, 31 Oct 2017) New Revision: 225ee45245bbcfd2dd13885c48effe9efb183b9a https://github.com/groonga/groonga/commit/225ee45245bbcfd2dd13885c48effe9efb183b9a Message: logical_range_filter: support dynamic columns "initial" and "filtered" stages are only supported. TODO: * Add tests for cache * Document Added files: test/command/suite/sharding/logical_range_filter/columns/stage/filtered/filter.expected test/command/suite/sharding/logical_range_filter/columns/stage/filtered/filter.test test/command/suite/sharding/logical_range_filter/columns/stage/filtered/no_hit/output_columns.expected test/command/suite/sharding/logical_range_filter/columns/stage/filtered/no_hit/output_columns.test test/command/suite/sharding/logical_range_filter/columns/stage/filtered/output_columns.expected test/command/suite/sharding/logical_range_filter/columns/stage/filtered/output_columns.test test/command/suite/sharding/logical_range_filter/columns/stage/initial/filter.expected test/command/suite/sharding/logical_range_filter/columns/stage/initial/filter.test test/command/suite/sharding/logical_range_filter/columns/stage/initial/no_condition.expected test/command/suite/sharding/logical_range_filter/columns/stage/initial/no_condition.test test/command/suite/sharding/logical_range_filter/columns/stage/initial/no_hit/filter.expected test/command/suite/sharding/logical_range_filter/columns/stage/initial/no_hit/filter.test test/command/suite/sharding/logical_range_filter/columns/stage/initial/no_hit/output_columns.expected test/command/suite/sharding/logical_range_filter/columns/stage/initial/no_hit/output_columns.test test/command/suite/sharding/logical_range_filter/columns/stage/initial/no_hit/sort_keys.expected test/command/suite/sharding/logical_range_filter/columns/stage/initial/no_hit/sort_keys.test test/command/suite/sharding/logical_range_filter/columns/stage/initial/no_records.expected test/command/suite/sharding/logical_range_filter/columns/stage/initial/no_records.test Modified files: plugins/sharding/dynamic_columns.rb plugins/sharding/logical_range_filter.rb Modified: plugins/sharding/dynamic_columns.rb (+6 -0) =================================================================== --- plugins/sharding/dynamic_columns.rb 2017-10-31 18:00:55 +0900 (35fdae585) +++ plugins/sharding/dynamic_columns.rb 2017-10-31 18:02:53 +0900 (cf8fa1a52) @@ -54,6 +54,12 @@ module Groonga each_output(&block) end + def empty? + @initial_contexts.empty? and + @filtered_contexts.empty? and + @output_contexts.empty? + end + def close each do |context| context.close Modified: plugins/sharding/logical_range_filter.rb (+47 -5) =================================================================== --- plugins/sharding/logical_range_filter.rb 2017-10-31 18:00:55 +0900 (1c8f86446) +++ plugins/sharding/logical_range_filter.rb 2017-10-31 18:02:53 +0900 (f4defa05c) @@ -71,6 +71,8 @@ module Groonga key << "#{input[:limit]}\0" key << "#{input[:output_columns]}\0" key << "#{input[:use_range_index]}\0" + dynamic_columns = DynamicColumns.parse(input) + key << dynamic_columns.cache_key key end @@ -81,10 +83,12 @@ module Groonga attr_reader :filter attr_reader :offset attr_reader :limit + attr_reader :dynamic_columns attr_accessor :current_offset attr_accessor :current_limit attr_reader :result_sets attr_reader :unsorted_result_sets + attr_reader :temporary_tables attr_reader :threshold def initialize(input) @input = input @@ -94,6 +98,7 @@ module Groonga @filter = @input[:filter] @offset = (@input[:offset] || 0).to_i @limit = (@input[:limit] || 10).to_i + @dynamic_columns = DynamicColumns.parse(@input) @current_offset = @offset @current_limit = @limit @@ -101,6 +106,8 @@ module Groonga @result_sets = [] @unsorted_result_sets = [] + @temporary_tables = [] + @threshold = compute_threshold end @@ -111,6 +118,12 @@ module Groonga @result_sets.each do |result_set| result_set.close if result_set.temporary? end + + @dynamic_columns.close + + @temporary_tables.each do |table| + table.close + end end private @@ -179,6 +192,12 @@ module Groonga if****@conte*****_sets.empty? result_set = HashTable.create(:flags => ObjectFlags::WITH_SUBREC, :key_type => first_shard.table) + @context.dynamic_columns.each_initial do |dynamic_column| + dynamic_column.apply(result_set) + end + @context.dynamic_columns.each_filtered do |dynamic_column| + dynamic_column.apply(result_set) + end @context.result_sets << result_set end end @@ -190,6 +209,8 @@ module Groonga @shard = shard @shard_range = shard_range + @target_table =****@shard***** + @filter =****@conte***** @result_sets =****@conte*****_sets @unsorted_result_sets =****@conte*****_result_sets @@ -201,7 +222,7 @@ module Groonga def execute return if @cover_type == :none - return if****@shard*****? + return if @target_table.empty? shard_key =****@shard***** if shard_key.nil? @@ -224,6 +245,14 @@ module Groonga range_index = nil end + @context.dynamic_columns.each_initial do |dynamic_column| + if @target_table ==****@shard***** + @target_table = @target_table.select_all + @context.temporary_tables << @target_table + end + dynamic_column.apply(@target_table) + end + execute_filter(range_index, expression_builder) end @@ -273,11 +302,15 @@ module Groonga __LINE__, __method__) end + unles****@conte*****_columns.empty? + reason = "dynamic columns are used" + return decide_use_range_index(false, reason, __LINE__, __method__) + end + current_limit =****@conte*****_limit if current_limit < 0 reason = "limit is negative: <#{current_limit}>" - return decide_use_range_index(false, reason, - __LINE__, __method__) + return decide_use_range_index(false, reason, __LINE__, __method__) end required_n_records =****@conte*****_offset + current_limit @@ -449,7 +482,7 @@ module Groonga end def filter_shard_all(range_index, expression_builder) - table =****@shard***** + table = @target_table if****@filte*****? if table.size <=****@conte*****_offset @context.current_offset -= table.size @@ -593,7 +626,7 @@ module Groonga end def filter_table - table =****@shard***** + table = @target_table create_expression(table) do |expression| yield(expression) result_set = table.select(expression) @@ -602,11 +635,20 @@ module Groonga end def sort_result_set(result_set) + @context.temporary_tables.delete(result_set) + if result_set.empty? result_set.close if result_set.temporary? return end + @context.dynamic_columns.each_filtered do |dynamic_column| + if result_set ==****@shard***** + result_set = result_set.select_all + end + dynamic_column.apply(result_set) + end + if result_set.size <=****@conte*****_offset @context.current_offset -= result_set.size result_set.close if result_set.temporary? Added: test/command/suite/sharding/logical_range_filter/columns/stage/filtered/filter.expected (+94 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_range_filter/columns/stage/filtered/filter.expected 2017-10-31 18:02:53 +0900 (556f414c0) @@ -0,0 +1,94 @@ +plugin_register sharding +[[0,0.0,0.0],true] +table_create Logs_20170315 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170315 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +table_create Logs_20170316 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170316 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170316 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +table_create Logs_20170317 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170317 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170317 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 1000}, +{"timestamp": "2017/03/15 01:00:00", "price": 900}, +{"timestamp": "2017/03/15 02:00:00", "price": 300} +] +[[0,0.0,0.0],3] +load --table Logs_20170316 +[ +{"timestamp": "2017/03/16 10:00:00", "price": 530}, +{"timestamp": "2017/03/16 11:00:00", "price": 520}, +{"timestamp": "2017/03/16 12:00:00", "price": 110} +] +[[0,0.0,0.0],3] +load --table Logs_20170317 +[ +{"timestamp": "2017/03/17 20:00:00", "price": 800}, +{"timestamp": "2017/03/17 21:00:00", "price": 400}, +{"timestamp": "2017/03/17 22:00:00", "price": 300} +] +[[0,0.0,0.0],3] +table_create Times TABLE_PAT_KEY Time +[[0,0.0,0.0],true] +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp +[[0,0.0,0.0],true] +column_create Times logs_20170316 COLUMN_INDEX Logs_20170316 timestamp +[[0,0.0,0.0],true] +column_create Times logs_20170317 COLUMN_INDEX Logs_20170317 timestamp +[[0,0.0,0.0],true] +log_level --level debug +[[0,0.0,0.0],true] +logical_range_filter Logs --shard_key timestamp --columns[filtered_id].stage filtered --columns[filtered_id].type UInt32 --columns[filtered_id].flags COLUMN_SCALAR --columns[filtered_id].value '_id' --filter 'price <= 900' --offset 1 --limit 3 --output_columns _id,filtered_id,price +[ + [ + 0, + 0.0, + 0.0 + ], + [ + [ + [ + "_id", + "UInt32" + ], + [ + "filtered_id", + "UInt32" + ], + [ + "price", + "UInt32" + ] + ], + [ + 3, + 3, + 300 + ], + [ + 1, + 1, + 530 + ], + [ + 2, + 2, + 520 + ] + ] +] +#|d| [logical_range_filter][select] <Logs_20170315>: dynamic columns are used +#|d| [logical_range_filter][select] <Logs_20170316>: dynamic columns are used +log_level --level notice +[[0,0.0,0.0],true] Added: test/command/suite/sharding/logical_range_filter/columns/stage/filtered/filter.test (+56 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_range_filter/columns/stage/filtered/filter.test 2017-10-31 18:02:53 +0900 (f0caf2748) @@ -0,0 +1,56 @@ +#@on-error omit +plugin_register sharding +#@on-error default + +table_create Logs_20170315 TABLE_NO_KEY +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +column_create Logs_20170315 price COLUMN_SCALAR UInt32 + +table_create Logs_20170316 TABLE_NO_KEY +column_create Logs_20170316 timestamp COLUMN_SCALAR Time +column_create Logs_20170316 price COLUMN_SCALAR UInt32 + +table_create Logs_20170317 TABLE_NO_KEY +column_create Logs_20170317 timestamp COLUMN_SCALAR Time +column_create Logs_20170317 price COLUMN_SCALAR UInt32 + +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 1000}, +{"timestamp": "2017/03/15 01:00:00", "price": 900}, +{"timestamp": "2017/03/15 02:00:00", "price": 300} +] + +load --table Logs_20170316 +[ +{"timestamp": "2017/03/16 10:00:00", "price": 530}, +{"timestamp": "2017/03/16 11:00:00", "price": 520}, +{"timestamp": "2017/03/16 12:00:00", "price": 110} +] + +load --table Logs_20170317 +[ +{"timestamp": "2017/03/17 20:00:00", "price": 800}, +{"timestamp": "2017/03/17 21:00:00", "price": 400}, +{"timestamp": "2017/03/17 22:00:00", "price": 300} +] + +table_create Times TABLE_PAT_KEY Time +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp +column_create Times logs_20170316 COLUMN_INDEX Logs_20170316 timestamp +column_create Times logs_20170317 COLUMN_INDEX Logs_20170317 timestamp + +#@add-important-log-levels debug +log_level --level debug +logical_range_filter Logs \ + --shard_key timestamp \ + --columns[filtered_id].stage filtered \ + --columns[filtered_id].type UInt32 \ + --columns[filtered_id].flags COLUMN_SCALAR \ + --columns[filtered_id].value '_id' \ + --filter 'price <= 900' \ + --offset 1 \ + --limit 3 \ + --output_columns _id,filtered_id,price +log_level --level notice +#@remove-important-log-levels debug Added: test/command/suite/sharding/logical_range_filter/columns/stage/filtered/no_hit/output_columns.expected (+25 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_range_filter/columns/stage/filtered/no_hit/output_columns.expected 2017-10-31 18:02:53 +0900 (3abccffd5) @@ -0,0 +1,25 @@ +plugin_register sharding +[[0,0.0,0.0],true] +table_create Logs_20170315 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170315 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 1000}, +{"timestamp": "2017/03/15 01:00:00", "price": 900}, +{"timestamp": "2017/03/15 02:00:00", "price": 300} +] +[[0,0.0,0.0],3] +table_create Times TABLE_PAT_KEY Time +[[0,0.0,0.0],true] +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp +[[0,0.0,0.0],true] +log_level --level debug +[[0,0.0,0.0],true] +logical_range_filter Logs --shard_key timestamp --columns[price_with_tax].stage filtered --columns[price_with_tax].type UInt32 --columns[price_with_tax].flags COLUMN_SCALAR --columns[price_with_tax].value 'price * 1.08' --min "2017/04/01 00:00:00" --offset 1 --limit 3 --output_columns price,price_with_tax +[[0,0.0,0.0],[[["price","UInt32"],["price_with_tax","UInt32"]]]] +log_level --level notice +[[0,0.0,0.0],true] Added: test/command/suite/sharding/logical_range_filter/columns/stage/filtered/no_hit/output_columns.test (+32 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_range_filter/columns/stage/filtered/no_hit/output_columns.test 2017-10-31 18:02:53 +0900 (00b26c902) @@ -0,0 +1,32 @@ +#@on-error omit +plugin_register sharding +#@on-error default + +table_create Logs_20170315 TABLE_NO_KEY +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +column_create Logs_20170315 price COLUMN_SCALAR UInt32 + +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 1000}, +{"timestamp": "2017/03/15 01:00:00", "price": 900}, +{"timestamp": "2017/03/15 02:00:00", "price": 300} +] + +table_create Times TABLE_PAT_KEY Time +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp + +#@add-important-log-levels debug +log_level --level debug +logical_range_filter Logs \ + --shard_key timestamp \ + --columns[price_with_tax].stage filtered \ + --columns[price_with_tax].type UInt32 \ + --columns[price_with_tax].flags COLUMN_SCALAR \ + --columns[price_with_tax].value 'price * 1.08' \ + --min "2017/04/01 00:00:00" \ + --offset 1 \ + --limit 3 \ + --output_columns price,price_with_tax +log_level --level notice +#@remove-important-log-levels debug Added: test/command/suite/sharding/logical_range_filter/columns/stage/filtered/output_columns.expected (+87 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_range_filter/columns/stage/filtered/output_columns.expected 2017-10-31 18:02:53 +0900 (25749d43f) @@ -0,0 +1,87 @@ +plugin_register sharding +[[0,0.0,0.0],true] +table_create Logs_20170315 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170315 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +table_create Logs_20170316 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170316 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170316 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +table_create Logs_20170317 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170317 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170317 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 1000}, +{"timestamp": "2017/03/15 01:00:00", "price": 900}, +{"timestamp": "2017/03/15 02:00:00", "price": 300} +] +[[0,0.0,0.0],3] +load --table Logs_20170316 +[ +{"timestamp": "2017/03/16 10:00:00", "price": 530}, +{"timestamp": "2017/03/16 11:00:00", "price": 520}, +{"timestamp": "2017/03/16 12:00:00", "price": 110} +] +[[0,0.0,0.0],3] +load --table Logs_20170317 +[ +{"timestamp": "2017/03/17 20:00:00", "price": 800}, +{"timestamp": "2017/03/17 21:00:00", "price": 400}, +{"timestamp": "2017/03/17 22:00:00", "price": 300} +] +[[0,0.0,0.0],3] +table_create Times TABLE_PAT_KEY Time +[[0,0.0,0.0],true] +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp +[[0,0.0,0.0],true] +column_create Times logs_20170316 COLUMN_INDEX Logs_20170316 timestamp +[[0,0.0,0.0],true] +column_create Times logs_20170317 COLUMN_INDEX Logs_20170317 timestamp +[[0,0.0,0.0],true] +log_level --level debug +[[0,0.0,0.0],true] +logical_range_filter Logs --shard_key timestamp --columns[price_with_tax].stage filtered --columns[price_with_tax].type UInt32 --columns[price_with_tax].flags COLUMN_SCALAR --columns[price_with_tax].value 'price * 1.08' --filter 'true' --offset 1 --limit 3 --output_columns price,price_with_tax +[ + [ + 0, + 0.0, + 0.0 + ], + [ + [ + [ + "price", + "UInt32" + ], + [ + "price_with_tax", + "UInt32" + ] + ], + [ + 900, + 972 + ], + [ + 300, + 324 + ], + [ + 530, + 572 + ] + ] +] +#|d| [logical_range_filter][select] <Logs_20170315>: dynamic columns are used +#|d| [logical_range_filter][select] <Logs_20170316>: dynamic columns are used +log_level --level notice +[[0,0.0,0.0],true] Added: test/command/suite/sharding/logical_range_filter/columns/stage/filtered/output_columns.test (+56 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_range_filter/columns/stage/filtered/output_columns.test 2017-10-31 18:02:53 +0900 (8dec81d2d) @@ -0,0 +1,56 @@ +#@on-error omit +plugin_register sharding +#@on-error default + +table_create Logs_20170315 TABLE_NO_KEY +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +column_create Logs_20170315 price COLUMN_SCALAR UInt32 + +table_create Logs_20170316 TABLE_NO_KEY +column_create Logs_20170316 timestamp COLUMN_SCALAR Time +column_create Logs_20170316 price COLUMN_SCALAR UInt32 + +table_create Logs_20170317 TABLE_NO_KEY +column_create Logs_20170317 timestamp COLUMN_SCALAR Time +column_create Logs_20170317 price COLUMN_SCALAR UInt32 + +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 1000}, +{"timestamp": "2017/03/15 01:00:00", "price": 900}, +{"timestamp": "2017/03/15 02:00:00", "price": 300} +] + +load --table Logs_20170316 +[ +{"timestamp": "2017/03/16 10:00:00", "price": 530}, +{"timestamp": "2017/03/16 11:00:00", "price": 520}, +{"timestamp": "2017/03/16 12:00:00", "price": 110} +] + +load --table Logs_20170317 +[ +{"timestamp": "2017/03/17 20:00:00", "price": 800}, +{"timestamp": "2017/03/17 21:00:00", "price": 400}, +{"timestamp": "2017/03/17 22:00:00", "price": 300} +] + +table_create Times TABLE_PAT_KEY Time +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp +column_create Times logs_20170316 COLUMN_INDEX Logs_20170316 timestamp +column_create Times logs_20170317 COLUMN_INDEX Logs_20170317 timestamp + +#@add-important-log-levels debug +log_level --level debug +logical_range_filter Logs \ + --shard_key timestamp \ + --columns[price_with_tax].stage filtered \ + --columns[price_with_tax].type UInt32 \ + --columns[price_with_tax].flags COLUMN_SCALAR \ + --columns[price_with_tax].value 'price * 1.08' \ + --filter 'true' \ + --offset 1 \ + --limit 3 \ + --output_columns price,price_with_tax +log_level --level notice +#@remove-important-log-levels debug Added: test/command/suite/sharding/logical_range_filter/columns/stage/initial/filter.expected (+87 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_range_filter/columns/stage/initial/filter.expected 2017-10-31 18:02:53 +0900 (d60c263b4) @@ -0,0 +1,87 @@ +plugin_register sharding +[[0,0.0,0.0],true] +table_create Logs_20170315 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170315 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +table_create Logs_20170316 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170316 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170316 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +table_create Logs_20170317 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170317 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170317 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 1000}, +{"timestamp": "2017/03/15 01:00:00", "price": 900}, +{"timestamp": "2017/03/15 02:00:00", "price": 300} +] +[[0,0.0,0.0],3] +load --table Logs_20170316 +[ +{"timestamp": "2017/03/16 10:00:00", "price": 530}, +{"timestamp": "2017/03/16 11:00:00", "price": 520}, +{"timestamp": "2017/03/16 12:00:00", "price": 110} +] +[[0,0.0,0.0],3] +load --table Logs_20170317 +[ +{"timestamp": "2017/03/17 20:00:00", "price": 800}, +{"timestamp": "2017/03/17 21:00:00", "price": 400}, +{"timestamp": "2017/03/17 22:00:00", "price": 300} +] +[[0,0.0,0.0],3] +table_create Times TABLE_PAT_KEY Time +[[0,0.0,0.0],true] +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp +[[0,0.0,0.0],true] +column_create Times logs_20170316 COLUMN_INDEX Logs_20170316 timestamp +[[0,0.0,0.0],true] +column_create Times logs_20170317 COLUMN_INDEX Logs_20170317 timestamp +[[0,0.0,0.0],true] +log_level --level debug +[[0,0.0,0.0],true] +logical_range_filter Logs --shard_key timestamp --columns[price_with_tax].stage initial --columns[price_with_tax].type UInt32 --columns[price_with_tax].flags COLUMN_SCALAR --columns[price_with_tax].value 'price * 1.08' --filter 'price_with_tax > 550' --offset 1 --limit 3 --output_columns price,price_with_tax +[ + [ + 0, + 0.0, + 0.0 + ], + [ + [ + [ + "price", + "UInt32" + ], + [ + "price_with_tax", + "UInt32" + ] + ], + [ + 900, + 972 + ], + [ + 530, + 572 + ], + [ + 520, + 561 + ] + ] +] +#|d| [logical_range_filter][select] <Logs_20170315>: dynamic columns are used +#|d| [logical_range_filter][select] <Logs_20170316>: dynamic columns are used +log_level --level notice +[[0,0.0,0.0],true] Added: test/command/suite/sharding/logical_range_filter/columns/stage/initial/filter.test (+56 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_range_filter/columns/stage/initial/filter.test 2017-10-31 18:02:53 +0900 (35fe665d2) @@ -0,0 +1,56 @@ +#@on-error omit +plugin_register sharding +#@on-error default + +table_create Logs_20170315 TABLE_NO_KEY +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +column_create Logs_20170315 price COLUMN_SCALAR UInt32 + +table_create Logs_20170316 TABLE_NO_KEY +column_create Logs_20170316 timestamp COLUMN_SCALAR Time +column_create Logs_20170316 price COLUMN_SCALAR UInt32 + +table_create Logs_20170317 TABLE_NO_KEY +column_create Logs_20170317 timestamp COLUMN_SCALAR Time +column_create Logs_20170317 price COLUMN_SCALAR UInt32 + +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 1000}, +{"timestamp": "2017/03/15 01:00:00", "price": 900}, +{"timestamp": "2017/03/15 02:00:00", "price": 300} +] + +load --table Logs_20170316 +[ +{"timestamp": "2017/03/16 10:00:00", "price": 530}, +{"timestamp": "2017/03/16 11:00:00", "price": 520}, +{"timestamp": "2017/03/16 12:00:00", "price": 110} +] + +load --table Logs_20170317 +[ +{"timestamp": "2017/03/17 20:00:00", "price": 800}, +{"timestamp": "2017/03/17 21:00:00", "price": 400}, +{"timestamp": "2017/03/17 22:00:00", "price": 300} +] + +table_create Times TABLE_PAT_KEY Time +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp +column_create Times logs_20170316 COLUMN_INDEX Logs_20170316 timestamp +column_create Times logs_20170317 COLUMN_INDEX Logs_20170317 timestamp + +#@add-important-log-levels debug +log_level --level debug +logical_range_filter Logs \ + --shard_key timestamp \ + --columns[price_with_tax].stage initial \ + --columns[price_with_tax].type UInt32 \ + --columns[price_with_tax].flags COLUMN_SCALAR \ + --columns[price_with_tax].value 'price * 1.08' \ + --filter 'price_with_tax > 550' \ + --offset 1 \ + --limit 3 \ + --output_columns price,price_with_tax +log_level --level notice +#@remove-important-log-levels debug Added: test/command/suite/sharding/logical_range_filter/columns/stage/initial/no_condition.expected (+87 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_range_filter/columns/stage/initial/no_condition.expected 2017-10-31 18:02:53 +0900 (724bf5ac3) @@ -0,0 +1,87 @@ +plugin_register sharding +[[0,0.0,0.0],true] +table_create Logs_20170315 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170315 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +table_create Logs_20170316 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170316 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170316 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +table_create Logs_20170317 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170317 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170317 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 1000}, +{"timestamp": "2017/03/15 01:00:00", "price": 900}, +{"timestamp": "2017/03/15 02:00:00", "price": 300} +] +[[0,0.0,0.0],3] +load --table Logs_20170316 +[ +{"timestamp": "2017/03/16 10:00:00", "price": 530}, +{"timestamp": "2017/03/16 11:00:00", "price": 520}, +{"timestamp": "2017/03/16 12:00:00", "price": 110} +] +[[0,0.0,0.0],3] +load --table Logs_20170317 +[ +{"timestamp": "2017/03/17 20:00:00", "price": 800}, +{"timestamp": "2017/03/17 21:00:00", "price": 400}, +{"timestamp": "2017/03/17 22:00:00", "price": 300} +] +[[0,0.0,0.0],3] +table_create Times TABLE_PAT_KEY Time +[[0,0.0,0.0],true] +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp +[[0,0.0,0.0],true] +column_create Times logs_20170316 COLUMN_INDEX Logs_20170316 timestamp +[[0,0.0,0.0],true] +column_create Times logs_20170317 COLUMN_INDEX Logs_20170317 timestamp +[[0,0.0,0.0],true] +log_level --level debug +[[0,0.0,0.0],true] +logical_range_filter Logs --shard_key timestamp --columns[price_with_tax].stage initial --columns[price_with_tax].type UInt32 --columns[price_with_tax].flags COLUMN_SCALAR --columns[price_with_tax].value 'price * 1.08' --offset 1 --limit 3 --output_columns price,price_with_tax +[ + [ + 0, + 0.0, + 0.0 + ], + [ + [ + [ + "price", + "UInt32" + ], + [ + "price_with_tax", + "UInt32" + ] + ], + [ + 900, + 972 + ], + [ + 300, + 324 + ], + [ + 530, + 572 + ] + ] +] +#|d| [logical_range_filter][select] <Logs_20170315>: dynamic columns are used +#|d| [logical_range_filter][select] <Logs_20170316>: dynamic columns are used +log_level --level notice +[[0,0.0,0.0],true] Added: test/command/suite/sharding/logical_range_filter/columns/stage/initial/no_condition.test (+55 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_range_filter/columns/stage/initial/no_condition.test 2017-10-31 18:02:53 +0900 (76bc367bd) @@ -0,0 +1,55 @@ +#@on-error omit +plugin_register sharding +#@on-error default + +table_create Logs_20170315 TABLE_NO_KEY +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +column_create Logs_20170315 price COLUMN_SCALAR UInt32 + +table_create Logs_20170316 TABLE_NO_KEY +column_create Logs_20170316 timestamp COLUMN_SCALAR Time +column_create Logs_20170316 price COLUMN_SCALAR UInt32 + +table_create Logs_20170317 TABLE_NO_KEY +column_create Logs_20170317 timestamp COLUMN_SCALAR Time +column_create Logs_20170317 price COLUMN_SCALAR UInt32 + +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 1000}, +{"timestamp": "2017/03/15 01:00:00", "price": 900}, +{"timestamp": "2017/03/15 02:00:00", "price": 300} +] + +load --table Logs_20170316 +[ +{"timestamp": "2017/03/16 10:00:00", "price": 530}, +{"timestamp": "2017/03/16 11:00:00", "price": 520}, +{"timestamp": "2017/03/16 12:00:00", "price": 110} +] + +load --table Logs_20170317 +[ +{"timestamp": "2017/03/17 20:00:00", "price": 800}, +{"timestamp": "2017/03/17 21:00:00", "price": 400}, +{"timestamp": "2017/03/17 22:00:00", "price": 300} +] + +table_create Times TABLE_PAT_KEY Time +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp +column_create Times logs_20170316 COLUMN_INDEX Logs_20170316 timestamp +column_create Times logs_20170317 COLUMN_INDEX Logs_20170317 timestamp + +#@add-important-log-levels debug +log_level --level debug +logical_range_filter Logs \ + --shard_key timestamp \ + --columns[price_with_tax].stage initial \ + --columns[price_with_tax].type UInt32 \ + --columns[price_with_tax].flags COLUMN_SCALAR \ + --columns[price_with_tax].value 'price * 1.08' \ + --offset 1 \ + --limit 3 \ + --output_columns price,price_with_tax +log_level --level notice +#@remove-important-log-levels debug Added: test/command/suite/sharding/logical_range_filter/columns/stage/initial/no_hit/filter.expected (+47 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_range_filter/columns/stage/initial/no_hit/filter.expected 2017-10-31 18:02:53 +0900 (a49a45d18) @@ -0,0 +1,47 @@ +plugin_register sharding +[[0,0.0,0.0],true] +table_create Logs_20170315 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170315 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 1000}, +{"timestamp": "2017/03/15 01:00:00", "price": 900}, +{"timestamp": "2017/03/15 02:00:00", "price": 300} +] +[[0,0.0,0.0],3] +table_create Times TABLE_PAT_KEY Time +[[0,0.0,0.0],true] +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp +[[0,0.0,0.0],true] +log_level --level debug +[[0,0.0,0.0],true] +logical_range_filter Logs --shard_key timestamp --columns[price_with_tax].stage initial --columns[price_with_tax].type UInt32 --columns[price_with_tax].flags COLUMN_SCALAR --columns[price_with_tax].value 'price * 1.08' --filter 'price_with_tax > 550' --min "2017/04/01 00:00:00" --offset 1 --limit 3 +[ + [ + 0, + 0.0, + 0.0 + ], + [ + [ + [ + "price_with_tax", + "UInt32" + ], + [ + "price", + "UInt32" + ], + [ + "timestamp", + "Time" + ] + ] + ] +] +log_level --level notice +[[0,0.0,0.0],true] Added: test/command/suite/sharding/logical_range_filter/columns/stage/initial/no_hit/filter.test (+32 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_range_filter/columns/stage/initial/no_hit/filter.test 2017-10-31 18:02:53 +0900 (b8c98a37d) @@ -0,0 +1,32 @@ +#@on-error omit +plugin_register sharding +#@on-error default + +table_create Logs_20170315 TABLE_NO_KEY +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +column_create Logs_20170315 price COLUMN_SCALAR UInt32 + +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 1000}, +{"timestamp": "2017/03/15 01:00:00", "price": 900}, +{"timestamp": "2017/03/15 02:00:00", "price": 300} +] + +table_create Times TABLE_PAT_KEY Time +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp + +#@add-important-log-levels debug +log_level --level debug +logical_range_filter Logs \ + --shard_key timestamp \ + --columns[price_with_tax].stage initial \ + --columns[price_with_tax].type UInt32 \ + --columns[price_with_tax].flags COLUMN_SCALAR \ + --columns[price_with_tax].value 'price * 1.08' \ + --filter 'price_with_tax > 550' \ + --min "2017/04/01 00:00:00" \ + --offset 1 \ + --limit 3 +log_level --level notice +#@remove-important-log-levels debug Added: test/command/suite/sharding/logical_range_filter/columns/stage/initial/no_hit/output_columns.expected (+25 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_range_filter/columns/stage/initial/no_hit/output_columns.expected 2017-10-31 18:02:53 +0900 (a4441866f) @@ -0,0 +1,25 @@ +plugin_register sharding +[[0,0.0,0.0],true] +table_create Logs_20170315 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170315 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 1000}, +{"timestamp": "2017/03/15 01:00:00", "price": 900}, +{"timestamp": "2017/03/15 02:00:00", "price": 300} +] +[[0,0.0,0.0],3] +table_create Times TABLE_PAT_KEY Time +[[0,0.0,0.0],true] +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp +[[0,0.0,0.0],true] +log_level --level debug +[[0,0.0,0.0],true] +logical_range_filter Logs --shard_key timestamp --columns[price_with_tax].stage initial --columns[price_with_tax].type UInt32 --columns[price_with_tax].flags COLUMN_SCALAR --columns[price_with_tax].value 'price * 1.08' --min "2017/04/01 00:00:00" --offset 1 --limit 3 --output_columns price,price_with_tax +[[0,0.0,0.0],[[["price","UInt32"],["price_with_tax","UInt32"]]]] +log_level --level notice +[[0,0.0,0.0],true] Added: test/command/suite/sharding/logical_range_filter/columns/stage/initial/no_hit/output_columns.test (+32 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_range_filter/columns/stage/initial/no_hit/output_columns.test 2017-10-31 18:02:53 +0900 (d535762ba) @@ -0,0 +1,32 @@ +#@on-error omit +plugin_register sharding +#@on-error default + +table_create Logs_20170315 TABLE_NO_KEY +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +column_create Logs_20170315 price COLUMN_SCALAR UInt32 + +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 1000}, +{"timestamp": "2017/03/15 01:00:00", "price": 900}, +{"timestamp": "2017/03/15 02:00:00", "price": 300} +] + +table_create Times TABLE_PAT_KEY Time +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp + +#@add-important-log-levels debug +log_level --level debug +logical_range_filter Logs \ + --shard_key timestamp \ + --columns[price_with_tax].stage initial \ + --columns[price_with_tax].type UInt32 \ + --columns[price_with_tax].flags COLUMN_SCALAR \ + --columns[price_with_tax].value 'price * 1.08' \ + --min "2017/04/01 00:00:00" \ + --offset 1 \ + --limit 3 \ + --output_columns price,price_with_tax +log_level --level notice +#@remove-important-log-levels debug Added: test/command/suite/sharding/logical_range_filter/columns/stage/initial/no_hit/sort_keys.expected (+47 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_range_filter/columns/stage/initial/no_hit/sort_keys.expected 2017-10-31 18:02:53 +0900 (33b794695) @@ -0,0 +1,47 @@ +plugin_register sharding +[[0,0.0,0.0],true] +table_create Logs_20170315 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170315 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 1000}, +{"timestamp": "2017/03/15 01:00:00", "price": 900}, +{"timestamp": "2017/03/15 02:00:00", "price": 300} +] +[[0,0.0,0.0],3] +table_create Times TABLE_PAT_KEY Time +[[0,0.0,0.0],true] +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp +[[0,0.0,0.0],true] +log_level --level debug +[[0,0.0,0.0],true] +logical_range_filter Logs --shard_key timestamp --columns[price_with_tax].stage initial --columns[price_with_tax].type UInt32 --columns[price_with_tax].flags COLUMN_SCALAR --columns[price_with_tax].value 'price * 1.08' --sort_keys price_with_tax --min "2017/04/01 00:00:00" --offset 1 --limit 3 +[ + [ + 0, + 0.0, + 0.0 + ], + [ + [ + [ + "price_with_tax", + "UInt32" + ], + [ + "price", + "UInt32" + ], + [ + "timestamp", + "Time" + ] + ] + ] +] +log_level --level notice +[[0,0.0,0.0],true] Added: test/command/suite/sharding/logical_range_filter/columns/stage/initial/no_hit/sort_keys.test (+32 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_range_filter/columns/stage/initial/no_hit/sort_keys.test 2017-10-31 18:02:53 +0900 (ab2285ac7) @@ -0,0 +1,32 @@ +#@on-error omit +plugin_register sharding +#@on-error default + +table_create Logs_20170315 TABLE_NO_KEY +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +column_create Logs_20170315 price COLUMN_SCALAR UInt32 + +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 1000}, +{"timestamp": "2017/03/15 01:00:00", "price": 900}, +{"timestamp": "2017/03/15 02:00:00", "price": 300} +] + +table_create Times TABLE_PAT_KEY Time +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp + +#@add-important-log-levels debug +log_level --level debug +logical_range_filter Logs \ + --shard_key timestamp \ + --columns[price_with_tax].stage initial \ + --columns[price_with_tax].type UInt32 \ + --columns[price_with_tax].flags COLUMN_SCALAR \ + --columns[price_with_tax].value 'price * 1.08' \ + --sort_keys price_with_tax \ + --min "2017/04/01 00:00:00" \ + --offset 1 \ + --limit 3 +log_level --level notice +#@remove-important-log-levels debug Added: test/command/suite/sharding/logical_range_filter/columns/stage/initial/no_records.expected (+22 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_range_filter/columns/stage/initial/no_records.expected 2017-10-31 18:02:53 +0900 (52475f64b) @@ -0,0 +1,22 @@ +plugin_register sharding +[[0,0.0,0.0],true] +table_create Logs_20170315 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170315 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +load --table Logs_20170315 +[ +] +[[0,0.0,0.0],0] +table_create Times TABLE_PAT_KEY Time +[[0,0.0,0.0],true] +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp +[[0,0.0,0.0],true] +log_level --level debug +[[0,0.0,0.0],true] +logical_range_filter Logs --shard_key timestamp --columns[price_with_tax].stage initial --columns[price_with_tax].type UInt32 --columns[price_with_tax].flags COLUMN_SCALAR --columns[price_with_tax].value 'price * 1.08' --offset 1 --limit 3 --output_columns price,price_with_tax +[[0,0.0,0.0],[[["price","UInt32"],["price_with_tax","UInt32"]]]] +log_level --level notice +[[0,0.0,0.0],true] Added: test/command/suite/sharding/logical_range_filter/columns/stage/initial/no_records.test (+29 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_range_filter/columns/stage/initial/no_records.test 2017-10-31 18:02:53 +0900 (f3e7c40c9) @@ -0,0 +1,29 @@ +#@on-error omit +plugin_register sharding +#@on-error default + +table_create Logs_20170315 TABLE_NO_KEY +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +column_create Logs_20170315 price COLUMN_SCALAR UInt32 + + +load --table Logs_20170315 +[ +] + +table_create Times TABLE_PAT_KEY Time +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp + +#@add-important-log-levels debug +log_level --level debug +logical_range_filter Logs \ + --shard_key timestamp \ + --columns[price_with_tax].stage initial \ + --columns[price_with_tax].type UInt32 \ + --columns[price_with_tax].flags COLUMN_SCALAR \ + --columns[price_with_tax].value 'price * 1.08' \ + --offset 1 \ + --limit 3 \ + --output_columns price,price_with_tax +log_level --level notice +#@remove-important-log-levels debug -------------- next part -------------- HTML����������������������������... URL: https://lists.osdn.me/mailman/archives/groonga-commit/attachments/20171031/48db7a10/attachment-0001.htm
