From d0ed42d23bfbe93340ad1c4b808042a388ea864d Mon Sep 17 00:00:00 2001 From: Andrew Cholakian Date: Fri, 5 Aug 2016 15:16:55 -0500 Subject: [PATCH 1/4] Add decode_size_limit_bytes option. Resolves #29 --- CHANGELOG.md | 3 +++ lib/logstash/codecs/json_lines.rb | 7 ++++++- spec/codecs/json_lines_spec.rb | 16 ++++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7f5211..97a5394 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.2.0 + - Add decode_size_limit_bytes option to limit the maximum size of JSON document that can be parsed.[#35](https://github.com/logstash-plugins/logstash-codec-json_lines/pull/35) + ## 3.1.0 - Feat: event `target => namespace` support (ECS) [#41](https://github.com/logstash-plugins/logstash-codec-json_lines/pull/41) - Refactor: dropped support for old Logstash versions (< 6.0) diff --git a/lib/logstash/codecs/json_lines.rb b/lib/logstash/codecs/json_lines.rb index bdff789..8e2256a 100644 --- a/lib/logstash/codecs/json_lines.rb +++ b/lib/logstash/codecs/json_lines.rb @@ -42,6 +42,11 @@ class LogStash::Codecs::JSONLines < LogStash::Codecs::Base # Change the delimiter that separates lines config :delimiter, :validate => :string, :default => "\n" + # Maximum number of bytes for a single line before a fatal exception is raised + # which will stop Logsash. + # The default is 20MB which is quite large for a JSON document + config :decode_size_limit_bytes, :validate => :number, :default => 20 * (1024 * 1024) # 20MB + # Defines a target field for placing decoded fields. # If this setting is omitted, data gets stored at the root (top level) of the event. # The target is only relevant while decoding data into a new event. @@ -50,7 +55,7 @@ class LogStash::Codecs::JSONLines < LogStash::Codecs::Base public def register - @buffer = FileWatch::BufferedTokenizer.new(@delimiter) + @buffer = FileWatch::BufferedTokenizer.new(@delimiter, @decode_size_limit_bytes) @converter = LogStash::Util::Charset.new(@charset) @converter.logger = @logger end diff --git a/spec/codecs/json_lines_spec.rb b/spec/codecs/json_lines_spec.rb index 34dd230..7a58421 100644 --- a/spec/codecs/json_lines_spec.rb +++ b/spec/codecs/json_lines_spec.rb @@ -118,6 +118,22 @@ end end + describe "decode_size_limits_bytes" do + let(:maximum_payload) { "a" * subject.decode_size_limit_bytes } + + it "should not raise an error if the number of bytes is not exceeded" do + expect { + subject.decode(maximum_payload) + }.not_to raise_error + end + + it "should raise an error if the max bytes are exceeded" do + expect { + subject.decode(maximum_payload << "z") + }.to raise_error(RuntimeError, "input buffer full") + end + end + end context "#encode" do From c12979a8b0570eeb1c6082ba62ee17fd4ad6cb5d Mon Sep 17 00:00:00 2001 From: andsel Date: Tue, 27 Aug 2024 16:43:42 +0200 Subject: [PATCH 2/4] Bumped new version --- logstash-codec-json_lines.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logstash-codec-json_lines.gemspec b/logstash-codec-json_lines.gemspec index 24dae2d..06f47e0 100644 --- a/logstash-codec-json_lines.gemspec +++ b/logstash-codec-json_lines.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-codec-json_lines' - s.version = '3.1.0' + s.version = '3.2.0' s.licenses = ['Apache License (2.0)'] s.summary = "Reads and writes newline-delimited JSON" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 6de5617e91f777cb3efb48581cefce748b0aa756 Mon Sep 17 00:00:00 2001 From: andsel Date: Wed, 28 Aug 2024 16:30:12 +0200 Subject: [PATCH 3/4] Switched to the exception thrown by BufferedTokenizerExt --- spec/codecs/json_lines_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/codecs/json_lines_spec.rb b/spec/codecs/json_lines_spec.rb index 7a58421..85d4361 100644 --- a/spec/codecs/json_lines_spec.rb +++ b/spec/codecs/json_lines_spec.rb @@ -130,7 +130,7 @@ it "should raise an error if the max bytes are exceeded" do expect { subject.decode(maximum_payload << "z") - }.to raise_error(RuntimeError, "input buffer full") + }.to raise_error(java.lang.IllegalStateException, "input buffer full") end end From 3b5a05ee906c528b8b2dac4cc658e28b364a2f93 Mon Sep 17 00:00:00 2001 From: Andrea Selva Date: Wed, 28 Aug 2024 17:12:02 +0200 Subject: [PATCH 4/4] Update CHANGELOG.md Co-authored-by: Mashhur <99575341+mashhurs@users.noreply.github.com> --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97a5394..1199f33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ ## 3.2.0 - - Add decode_size_limit_bytes option to limit the maximum size of JSON document that can be parsed.[#35](https://github.com/logstash-plugins/logstash-codec-json_lines/pull/35) + - Add decode_size_limit_bytes option to limit the maximum size of each JSON line that can be parsed.[#43](https://github.com/logstash-plugins/logstash-codec-json_lines/pull/43) ## 3.1.0 - Feat: event `target => namespace` support (ECS) [#41](https://github.com/logstash-plugins/logstash-codec-json_lines/pull/41)