Kouhei Sutou
null+****@clear*****
Tue Dec 20 15:04:39 JST 2016
Kouhei Sutou 2016-12-20 15:04:39 +0900 (Tue, 20 Dec 2016) New Revision: 8d72cb56a50ec0713eca12a32b3182481acbd198 https://github.com/groonga/groonga-command-parser/commit/8d72cb56a50ec0713eca12a32b3182481acbd198 Message: Use json-stream instead of ffi-yajl Because ffi-yajl doesn't work on Windows. Modified files: groonga-command-parser.gemspec lib/groonga/command/parser/load-values-parser.rb test/test-load-value-parser.rb test/test-parser.rb Modified: groonga-command-parser.gemspec (+1 -2) =================================================================== --- groonga-command-parser.gemspec 2016-12-20 14:47:32 +0900 (eab04f0) +++ groonga-command-parser.gemspec 2016-12-20 15:04:39 +0900 (bef237d) @@ -53,8 +53,7 @@ Gem::Specification.new do |spec| spec.require_paths = ["lib"] spec.add_runtime_dependency("groonga-command", ">= 1.0.9") - spec.add_runtime_dependency("ffi") - spec.add_runtime_dependency("ffi-yajl") + spec.add_runtime_dependency("json-stream") spec.add_development_dependency("test-unit") spec.add_development_dependency("test-unit-notify") Modified: lib/groonga/command/parser/load-values-parser.rb (+38 -124) =================================================================== --- lib/groonga/command/parser/load-values-parser.rb 2016-12-20 14:47:32 +0900 (9f3e1a1) +++ lib/groonga/command/parser/load-values-parser.rb 2016-12-20 15:04:39 +0900 (871e64e) @@ -1,6 +1,4 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2015 Kouhei Sutou <kou �� clear-code.com> +# Copyright (C) 2015-2016 Kouhei Sutou <kou �� clear-code.com> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -16,12 +14,7 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -ENV["FORCE_FFI_YAJL"] = "ffi" -require "ffi_yajl" - -module FFI_Yajl - attach_function :yajl_get_bytes_consumed, [:yajl_handle], :size_t -end +require "json/stream" module Groonga module Command @@ -30,9 +23,7 @@ module Groonga attr_writer :on_value attr_writer :on_end def initialize - initialize_callbacks - @handle = nil - @callbacks_memory = nil + initialize_parser @on_value = nil @on_end = nil @containers = [] @@ -43,39 +34,28 @@ module Groonga data_size = data.bytesize return self if data_size.zero? - ensure_handle - - status = FFI_Yajl.yajl_parse(@handle, data, data_size) - - if status != :yajl_status_ok - consumed = FFI_Yajl.yajl_get_bytes_consumed(@handle) - if consumed > 0 - consumed -= 1 - end - if****@conta*****? - message = "there are garbages before JSON" - else - message = FFI_Yajl.yajl_get_error(@handle, 0, nil, 0).chomp - end + before_pos =****@parse***** + status = catch do |tag| + @tag = tag begin - raise Error.new(message, + @parser << data + rescue JSON::Stream::ParserError => error + pos =****@parse***** + consumed = pos - before_pos - 1 + raise Error.new(error.message, data[0, consumed], data[consumed..-1]) - ensure - finalize_handle end + :continue end - if****@conta*****? - consumed = FFI_Yajl.yajl_get_bytes_consumed(@handle) - begin - if consumed < data_size - @on_end.call(data[consumed..-1]) - else - @on_end.call(nil) - end - ensure - finalize_handle + if status == :done + pos =****@parse***** + consumed = pos - before_pos + if consumed < data_size + @on_end.call(data[consumed..-1]) + else + @on_end.call(nil) end end @@ -83,51 +63,30 @@ module Groonga end private - def callback(*arguments) - FFI::Function.new(:int, [:pointer, *arguments]) do |_, *values| - yield(*values) - 1 - end - end - - def initialize_callbacks - @null_callback = callback do - push_value(nil) - end - @boolean_callback = callback(:int) do |c_boolean| - push_value(c_boolean != 0) - end - @number_callback = callback(:string, :size_t) do |data, size| - number_data = data.slice(0, size) - if /[\.eE]/ =~ number_data - number = number_data.to_f - else - number = number_data.to_i - end - push_value(number) - end - @string_callback = callback(:string, :size_t) do |data, size| - string = data.slice(0, size) - string.force_encoding(Encoding::UTF_8) - push_value(string) + def initialize_parser + @parser = JSON::Stream::Parser.new + @parser.singleton_class.__send__(:attr_reader, :pos) + @parser.end_document do + throw(@tag, :done) end - @start_map_callback = callback do + @parser.start_object do push_container({}) end - @map_key_callback = callback(:string, :size_t) do |data, size| - key = data.slice(0, size) - key.force_encoding(Encoding::UTF_8) - @keys.push(key) - end - @end_map_callback = callback do + @parser.end_object do pop_container end - @start_array_callback = callback do + @parser.start_array do push_container([]) end - @end_array_callback = callback do + @parser.end_array do pop_container end + @parser.key do |key| + push_key(key) + end + @parser.value do |value| + push_value(value) + end end def push_container(container) @@ -143,6 +102,10 @@ module Groonga end end + def push_key(key) + @keys.push(key) + end + def push_value(value) container =****@conta***** case container @@ -152,55 +115,6 @@ module Groonga container.push(value) end end - - def ensure_handle - return if @handle - initialize_handle - end - - def initialize_handle - @callbacks_memory = FFI::MemoryPointer.new(FFI_Yajl::YajlCallbacks) - callbacks = FFI_Yajl::YajlCallbacks.new(@callbacks_memory) - callbacks[:yajl_null] = @null_callback - callbacks[:yajl_boolean] = @boolean_callback - callbacks[:yajl_integer] = nil - callbacks[:yajl_double] = nil - callbacks[:yajl_number] = @number_callback - callbacks[:yajl_string] = @string_callback - callbacks[:yajl_start_map] = @start_map_callback - callbacks[:yajl_map_key] = @map_key_callback - callbacks[:yajl_end_map] = @end_map_callback - callbacks[:yajl_start_array] = @start_array_callback - callbacks[:yajl_end_array] = @end_array_callback - - @handle = FFI_Yajl.yajl_alloc(@callbacks_memory, nil, nil) - FFI_Yajl.yajl_config(@handle, - :yajl_allow_trailing_garbage, - :int, - 1) - @finalizer = Finalizer.new(@handle) - ObjectSpace.define_finalizer(self, @finalizer) - end - - def finalize_handle - @callbacks_memory = nil - @finalizer.call(object_id) - ObjectSpace.undefine_finalizer(self) - @finalizer = nil - @handle = nil - end - - class Finalizer - def initialize(handle) - @handle = handle - end - - def call(object_id) - return if****@handl*****? - FFI_Yajl.yajl_free(@handle) - @handle = nil - end - end end end end Modified: test/test-load-value-parser.rb (+46 -2) =================================================================== --- test/test-load-value-parser.rb 2016-12-20 14:47:32 +0900 (efb7c0f) +++ test/test-load-value-parser.rb 2016-12-20 15:04:39 +0900 (8cc6b4e) @@ -21,13 +21,17 @@ class LoadValuesParserTest < Test::Unit::TestCase @parser.on_value = lambda do |value| @values << value end + @parse_done = false @parser.on_end = lambda do |rest| + @parse_done = true + @parse_rest = rest end end def parse(data) data.each_line do |line| @parser << line + break if @parse_done end @values end @@ -140,10 +144,50 @@ class LoadValuesParserTest < Test::Unit::TestCase } } ] + JSON + end + end + + sub_test_case "error" do + test "unfinished" do + assert_equal([ + { + "object" => { + "string" => "abc", + }, + }, + ], + parse(<<-JSON)) [ - 1 -] + { + "object": { + "string": "abc" + } + }, + { + JSON + assert_false(@parse_done) + end + + test "too much" do + assert_equal([ + { + "object" => { + "string" => "abc", + }, + }, + ], + parse(<<-JSON)) +[ + { + "object": { + "string": "abc" + } + } +]garbage JSON + assert_equal([true, "garbage\n"], + [@parse_done, @parse_rest]) end end end Modified: test/test-parser.rb (+3 -5) =================================================================== --- test/test-parser.rb 2016-12-20 14:47:32 +0900 (9da0693) +++ test/test-parser.rb 2016-12-20 15:04:39 +0900 (9692a59) @@ -1,6 +1,4 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2011-2014 Kouhei Sutou <kou �� clear-code.com> +# Copyright (C) 2011-2016 Kouhei Sutou <kou �� clear-code.com> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -374,7 +372,7 @@ EOS end def test_no_record_separate_comma - message = "parse error: after array element, I expect ',' or ']'" + message = "Expected comma or object or array close: char 37" before = <<-BEFORE [ {"_key": "alice", "name": "Alice"} @@ -394,7 +392,7 @@ EOC end def test_garbage_before_json - message = "there are garbages before JSON" + message = "Expected value: char 0" before = "" after = <<-AFTER XXX -------------- next part -------------- HTML����������������������������...다운로드
