logstash input crashes and stays dead

Description

After setting up logstash in agent mode with syslog as input and elasticsearch as output it crashed out of the box at random intervals.
I removed all filters, and it still crashes.

My config looks like this:
input {
syslog {
type => "syslog"
port => "600"
}
}

filter {
}

output {
elasticsearch {
host => "127.0.0.1"
}
}

Crashes like this:
Exception in thread "input|syslog|tcp|127.0.0.1:58729}" java.lang.ArrayIndexOutOfBoundsException: 22
at org.jruby.RubyMatchData.updateCharOffset(RubyMatchData.java:137)
at org.jruby.RubyMatchData.offsetCommon(RubyMatchData.java:549)
at org.jruby.RubyMatchData.offset19(RubyMatchData.java:532)
at org.jruby.RubyMatchData$i$1$0$offset19.call(RubyMatchData$i$1$0$offset19.gen:65535)
at org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:167)
at rubyjit.match_8D1685C54EEAB1155AD78EFD3BC431D8C1612805._file_(file:/opt/logstash/logstash-1.1.0-monolithic.jar!/gems/jls-grok-0.9.6/lib/grok-pure.rb:135)
at rubyjit.match_8D1685C54EEAB1155AD78EFD3BC431D8C1612805._file_(file:/opt/logstash/logstash-1.1.0-monolithic.jar!/gems/jls-grok-0.9.6/lib/grok-pure.rb)
at org.jruby.internal.runtime.methods.JittedMethod.call(JittedMethod.java:167)
at org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:167)
at rubyjit.match_7A14C06CB34166F3CCD73F0400BD0C687B510E05.block_0$RUBY$_file_(file:/opt/logstash/logstash-1.1.0-monolithic.jar!/gems/jls-grok-0.9.6/lib/grok/pure/pile.rb:63)
at rubyjit$match_7A14C06CB34166F3CCD73F0400BD0C687B510E05$block_0$RUBY$_file.call(rubyjit$match_7A14C06CB34166F3CCD73F0400BD0C687B510E05$block_0$RUBY$file_:65535)
at org.jruby.runtime.CompiledBlock19.yield(CompiledBlock19.java:130)
at org.jruby.runtime.Block.yield(Block.java:130)
at org.jruby.RubyArray.eachCommon(RubyArray.java:1605)
at org.jruby.RubyArray.each(RubyArray.java:1612)
at org.jruby.RubyArray$i$0$0$each.call(RubyArray$i$0$0$each.gen:65535)
at org.jruby.runtime.callsite.CachingCallSite.callBlock(CachingCallSite.java:142)
at org.jruby.runtime.callsite.CachingCallSite.callIter(CachingCallSite.java:153)
at rubyjit.match_7A14C06CB34166F3CCD73F0400BD0C687B510E05._file_(file:/opt/logstash/logstash-1.1.0-monolithic.jar!/gems/jls-grok-0.9.6/lib/grok/pure/pile.rb:62)
at rubyjit.match_7A14C06CB34166F3CCD73F0400BD0C687B510E05._file_(file:/opt/logstash/logstash-1.1.0-monolithic.jar!/gems/jls-grok-0.9.6/lib/grok/pure/pile.rb)
at org.jruby.internal.runtime.methods.JittedMethod.call(JittedMethod.java:167)
at org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:167)
at rubyjit.filter_49507F2D8F7761C803966D4E8D9367540384EADF.block_1$RUBY$_file_(file:/opt/logstash/logstash-1.1.0-monolithic.jar!/logstash/filters/grok.rb:167)
at rubyjit$filter_49507F2D8F7761C803966D4E8D9367540384EADF$block_1$RUBY$_file.call(rubyjit$filter_49507F2D8F7761C803966D4E8D9367540384EADF$block_1$RUBY$file_:65535)
at org.jruby.runtime.CompiledBlock19.yield(CompiledBlock19.java:130)
at org.jruby.runtime.Block.yield(Block.java:130)
at org.jruby.RubyArray.eachCommon(RubyArray.java:1605)
at org.jruby.RubyArray.each(RubyArray.java:1612)
at org.jruby.RubyArray$i$0$0$each.call(RubyArray$i$0$0$each.gen:65535)
at org.jruby.runtime.callsite.CachingCallSite.callBlock(CachingCallSite.java:142)
at org.jruby.runtime.callsite.CachingCallSite.callIter(CachingCallSite.java:153)
at rubyjit.filter_49507F2D8F7761C803966D4E8D9367540384EADF.block_0$RUBY$_file_(file:/opt/logstash/logstash-1.1.0-monolithic.jar!/logstash/filters/grok.rb:166)
at rubyjit$filter_49507F2D8F7761C803966D4E8D9367540384EADF$block_0$RUBY$_file.call(rubyjit$filter_49507F2D8F7761C803966D4E8D9367540384EADF$block_0$RUBY$file_:65535)
at org.jruby.runtime.CompiledBlock19.yield(CompiledBlock19.java:130)
at org.jruby.runtime.Block.yield(Block.java:130)
at org.jruby.RubyHash$13.visit(RubyHash.java:1188)
at org.jruby.RubyHash.visitAll(RubyHash.java:594)
at org.jruby.RubyHash.iteratorVisitAll(RubyHash.java:1139)
at org.jruby.RubyHash.each_pairCommon(RubyHash.java:1184)
at org.jruby.RubyHash.each19(RubyHash.java:1175)
at org.jruby.RubyHash$i$0$0$each19.call(RubyHash$i$0$0$each19.gen:65535)
at org.jruby.runtime.callsite.CachingCallSite.callBlock(CachingCallSite.java:142)
at org.jruby.runtime.callsite.CachingCallSite.callIter(CachingCallSite.java:153)
at rubyjit.filter_49507F2D8F7761C803966D4E8D9367540384EADF._file_(file:/opt/logstash/logstash-1.1.0-monolithic.jar!/logstash/filters/grok.rb:157)
at rubyjit.filter_49507F2D8F7761C803966D4E8D9367540384EADF._file_(file:/opt/logstash/logstash-1.1.0-monolithic.jar!/logstash/filters/grok.rb)
at org.jruby.internal.runtime.methods.JittedMethod.call(JittedMethod.java:167)
at org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:167)
at rubyjit.syslog_relay_9EA6EADF0D414BED20A7CB77AF555AB9E7592103._file_(file:/opt/logstash/logstash-1.1.0-monolithic.jar!/logstash/inputs/syslog.rb:200)
at rubyjit.syslog_relay_9EA6EADF0D414BED20A7CB77AF555AB9E7592103._file_(file:/opt/logstash/logstash-1.1.0-monolithic.jar!/logstash/inputs/syslog.rb)
at org.jruby.internal.runtime.methods.JittedMethod.call(JittedMethod.java:207)
at org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:201)
at org.jruby.ast.FCallTwoArgNode.interpret(FCallTwoArgNode.java:38)
at org.jruby.ast.NewlineNode.interpret(NewlineNode.java:104)
at org.jruby.ast.BlockNode.interpret(BlockNode.java:71)
at org.jruby.ast.IfNode.interpret(IfNode.java:117)
at org.jruby.ast.NewlineNode.interpret(NewlineNode.java:104)
at org.jruby.ast.BlockNode.interpret(BlockNode.java:71)
at org.jruby.evaluator.ASTInterpreter.INTERPRET_BLOCK(ASTInterpreter.java:112)
at org.jruby.runtime.Interpreted19Block.evalBlockBody(Interpreted19Block.java:203)
at org.jruby.runtime.Interpreted19Block.yield(Interpreted19Block.java:154)
at org.jruby.runtime.Block.yield(Block.java:130)
at org.jruby.RubyIO.each_lineInternal(RubyIO.java:3133)
at org.jruby.RubyIO.each(RubyIO.java:3141)
at org.jruby.RubyIO$i$0$1$each.call(RubyIO$i$0$1$each.gen:65535)
at org.jruby.internal.runtime.methods.DynamicMethod.call(DynamicMethod.java:203)
at org.jruby.runtime.callsite.CachingCallSite.cacheAndCall(CachingCallSite.java:302)
at org.jruby.runtime.callsite.CachingCallSite.callBlock(CachingCallSite.java:144)
at org.jruby.runtime.callsite.CachingCallSite.callIter(CachingCallSite.java:153)
at org.jruby.ast.CallNoArgBlockNode.interpret(CallNoArgBlockNode.java:64)
at org.jruby.ast.NewlineNode.interpret(NewlineNode.java:104)
at org.jruby.ast.RescueNode.executeBody(RescueNode.java:216)
at org.jruby.ast.RescueNode.interpretWithJavaExceptions(RescueNode.java:120)
at org.jruby.ast.RescueNode.interpret(RescueNode.java:110)
at org.jruby.ast.BeginNode.interpret(BeginNode.java:83)
at org.jruby.ast.NewlineNode.interpret(NewlineNode.java:104)
at org.jruby.ast.BlockNode.interpret(BlockNode.java:71)
at org.jruby.evaluator.ASTInterpreter.INTERPRET_BLOCK(ASTInterpreter.java:112)
at org.jruby.runtime.Interpreted19Block.evalBlockBody(Interpreted19Block.java:203)
at org.jruby.runtime.Interpreted19Block.yield(Interpreted19Block.java:191)
at org.jruby.runtime.Interpreted19Block.call(Interpreted19Block.java:122)
at org.jruby.runtime.Block.call(Block.java:89)
at org.jruby.RubyProc.call(RubyProc.java:270)
at org.jruby.RubyProc.call(RubyProc.java:224)
at org.jruby.internal.runtime.RubyRunnable.run(RubyRunnable.java:95)
at java.lang.Thread.run(Thread.java:662)

I can successfully do Ctrl-C on it for it to shut down normally and restart it. It works for between 15minutes and a few hours before it crashes again.

Activity

Show:
John E. Vincent
February 13, 2012, 2:59 PM

Working with viddy on IRC about this. Obviously SOMETHING is coming in that grok isn't liking.

Marius Eikenes
February 14, 2012, 12:39 PM
Edited

I have been working on this crap for hours, now I have finally found the culprit...
It seems to be our Norway specific characters "æøå"

I started logstash on port 600, and ran netcat like this:

This happens;

Jordan Sissel
August 21, 2012, 7:25 AM

The 'array index out of bounds' was a bug in JRuby's regexp engine that I believe is now fixed in the version of JRuby shipped with logstash 1.1.1

Jordan Sissel
August 21, 2012, 7:26 AM

Tentatively marking resolved. Please re-open if you find this is not fixed!

Assignee

Jordan Sissel

Reporter

Marius Eikenes

Labels

None

Fix versions

Affects versions

Configure