Attachment 1476062 Details for Bug 1616169 – IOException: No space left on device

IOException: No space left on device

BZ_1616169.txt (text/plain), 43.47 KB, created by Junqi Zhao on 2018-08-15 07:32:14 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Junqi Zhao

Created: 2018-08-15 07:32:14 UTC

Size: 43.47 KB

patch

obsolete

>oc logs -c elasticsearch logging-es-data-master-w6l9n07t-10-lvvc4 -n logging
>Enter passphrase for key '/home/fedora/online/id_rsa': 
>[2018-08-15 03:24:41,560][INFO ][container.run            ] Begin Elasticsearch startup script
>[2018-08-15 03:24:41,566][INFO ][container.run            ] Comparing the specified RAM to the maximum recommended for Elasticsearch...
>[2018-08-15 03:24:41,567][INFO ][container.run            ] Inspecting the maximum RAM available...
>[2018-08-15 03:24:41,571][INFO ][container.run            ] ES_JAVA_OPTS: '-Dmapper.allow_dots_in_name=true -Xms6144m -Xmx6144m'
>[2018-08-15 03:24:41,592][INFO ][container.run            ] Setting heap dump location /elasticsearch/persistent/heapdump.hprof
>[2018-08-15 03:24:41,594][INFO ][container.run            ] ES_JAVA_OPTS: '-Dmapper.allow_dots_in_name=true -Xms6144m -Xmx6144m -XX:HeapDumpPath=/elasticsearch/persistent/heapdump.hprof -Dsg.display_lic_none=false'
>[2018-08-15 03:24:41,596][INFO ][container.run            ] Checking if Elasticsearch is ready on https://localhost:9200
>2018-08-15 03:24:45,111 main ERROR Unable to write to stream /elasticsearch/persistent/logging-es/logs/logging-es.log for appender rolling: org.apache.logging.log4j.core.appender.AppenderLoggingException: Error writing to stream /elasticsearch/persistent/logging-es/logs/logging-es.log
>2018-08-15 03:24:45,113 main ERROR An exception occurred processing Appender rolling org.apache.logging.log4j.core.appender.AppenderLoggingException: Error writing to stream /elasticsearch/persistent/logging-es/logs/logging-es.log
>	at org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:264)
>	at org.apache.logging.log4j.core.appender.FileManager.writeToDestination(FileManager.java:261)
>	at org.apache.logging.log4j.core.appender.rolling.RollingFileManager.writeToDestination(RollingFileManager.java:219)
>	at org.apache.logging.log4j.core.appender.OutputStreamManager.flushBuffer(OutputStreamManager.java:294)
>	at org.apache.logging.log4j.core.appender.OutputStreamManager.flush(OutputStreamManager.java:303)
>	at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:179)
>	at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:170)
>	at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:161)
>	at org.apache.logging.log4j.core.appender.RollingFileAppender.append(RollingFileAppender.java:308)
>	at org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:156)
>	at org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:129)
>	at org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:120)
>	at org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:84)
>	at org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:448)
>	at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:433)
>	at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
>	at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:403)
>	at org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:63)
>	at org.apache.logging.log4j.core.Logger.logMessage(Logger.java:146)
>	at org.apache.logging.log4j.spi.ExtendedLoggerWrapper.logMessage(ExtendedLoggerWrapper.java:217)
>	at org.elasticsearch.common.logging.PrefixLogger.logMessage(PrefixLogger.java:102)
>	at org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2116)
>	at org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2100)
>	at org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:1994)
>	at org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1966)
>	at org.apache.logging.log4j.spi.AbstractLogger.info(AbstractLogger.java:1303)
>	at org.elasticsearch.node.Node.<init>(Node.java:254)
>	at org.elasticsearch.node.Node.<init>(Node.java:245)
>	at org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:233)
>	at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:233)
>	at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:342)
>	at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:132)
>	at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:123)
>	at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:70)
>	at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:134)
>	at org.elasticsearch.cli.Command.main(Command.java:90)
>	at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:91)
>	at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:84)
>Caused by: java.io.IOException: No space left on device
>	at java.io.FileOutputStream.writeBytes(Native Method)
>	at java.io.FileOutputStream.write(FileOutputStream.java:326)
>	at org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:262)
>	... 37 more
>
>2018-08-15 03:24:45,154 main ERROR Unable to write to stream /elasticsearch/persistent/logging-es/logs/logging-es_deprecation.log for appender deprecation_rolling: org.apache.logging.log4j.core.appender.AppenderLoggingException: Error writing to stream /elasticsearch/persistent/logging-es/logs/logging-es_deprecation.log
>2018-08-15 03:24:45,154 main ERROR An exception occurred processing Appender deprecation_rolling org.apache.logging.log4j.core.appender.AppenderLoggingException: Error writing to stream /elasticsearch/persistent/logging-es/logs/logging-es_deprecation.log
>	at org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:264)
>	at org.apache.logging.log4j.core.appender.FileManager.writeToDestination(FileManager.java:261)
>	at org.apache.logging.log4j.core.appender.rolling.RollingFileManager.writeToDestination(RollingFileManager.java:219)
>	at org.apache.logging.log4j.core.appender.OutputStreamManager.flushBuffer(OutputStreamManager.java:294)
>	at org.apache.logging.log4j.core.appender.OutputStreamManager.flush(OutputStreamManager.java:303)
>	at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:179)
>	at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:170)
>	at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:161)
>	at org.apache.logging.log4j.core.appender.RollingFileAppender.append(RollingFileAppender.java:308)
>	at org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:156)
>	at org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:129)
>	at org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:120)
>	at org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:84)
>	at org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:448)
>	at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:433)
>	at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
>	at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:403)
>	at org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:63)
>	at org.apache.logging.log4j.core.Logger.logMessage(Logger.java:146)
>	at org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2116)
>	at org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2100)
>	at org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2005)
>	at org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1874)
>	at org.apache.logging.log4j.spi.AbstractLogger.warn(AbstractLogger.java:2574)
>	at org.elasticsearch.common.logging.DeprecationLogger.deprecated(DeprecationLogger.java:322)
>	at org.elasticsearch.common.logging.DeprecationLogger.deprecated(DeprecationLogger.java:299)
>	at org.elasticsearch.common.logging.DeprecationLogger.deprecated(DeprecationLogger.java:129)
>	at org.elasticsearch.env.NodeEnvironment.<init>(NodeEnvironment.java:216)
>	at org.elasticsearch.node.Node.<init>(Node.java:265)
>	at org.elasticsearch.node.Node.<init>(Node.java:245)
>	at org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:233)
>	at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:233)
>	at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:342)
>	at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:132)
>	at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:123)
>	at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:70)
>	at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:134)
>	at org.elasticsearch.cli.Command.main(Command.java:90)
>	at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:91)
>	at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:84)
>Caused by: java.io.IOException: No space left on device
>	at java.io.FileOutputStream.writeBytes(Native Method)
>	at java.io.FileOutputStream.write(FileOutputStream.java:326)
>	at org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:262)
>	... 39 more
>
>2018-08-15 03:24:45,218 main ERROR Recovering from StringBuilderEncoder.encode('[2018-08-15T03:24:45,207][ERROR][o.e.b.Bootstrap          ] Exception
>java.lang.IllegalStateException: Failed to create node environment
>	at org.elasticsearch.node.Node.<init>(Node.java:268) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.node.Node.<init>(Node.java:245) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:233) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:233) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:342) [elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:132) [elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:123) [elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:70) [elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:134) [elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.cli.Command.main(Command.java:90) [elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:91) [elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:84) [elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>Caused by: java.io.IOException: No space left on device
>	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
>	at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) ~[?:?]
>	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) ~[?:?]
>	at sun.nio.ch.IOUtil.write(IOUtil.java:65) ~[?:?]
>	at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) ~[?:?]
>	at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) ~[?:1.8.0_181]
>	at java.nio.channels.Channels.writeFully(Channels.java:101) ~[?:1.8.0_181]
>	at java.nio.channels.Channels.access$000(Channels.java:61) ~[?:1.8.0_181]
>	at java.nio.channels.Channels$1.write(Channels.java:174) ~[?:1.8.0_181]
>	at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) ~[?:1.8.0_181]
>	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) ~[?:1.8.0_181]
>	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) ~[?:1.8.0_181]
>	at org.apache.lucene.store.OutputStreamIndexOutput.getChecksum(OutputStreamIndexOutput.java:80) ~[lucene-core-6.6.1.jar:6.6.1 9aa465a89b64ff2dabe7b4d50c472de32c298683 - varunthacker - 2017-08-29 21:54:39]
>	at org.apache.lucene.codecs.CodecUtil.writeCRC(CodecUtil.java:548) ~[lucene-core-6.6.1.jar:6.6.1 9aa465a89b64ff2dabe7b4d50c472de32c298683 - varunthacker - 2017-08-29 21:54:39]
>	at org.apache.lucene.codecs.CodecUtil.writeFooter(CodecUtil.java:393) ~[lucene-core-6.6.1.jar:6.6.1 9aa465a89b64ff2dabe7b4d50c472de32c298683 - varunthacker - 2017-08-29 21:54:39]
>	at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:140) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.env.NodeEnvironment.loadOrCreateNodeMetaData(NodeEnvironment.java:419) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.env.NodeEnvironment.<init>(NodeEnvironment.java:263) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.node.Node.<init>(Node.java:265) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	... 11 more
>	Suppressed: java.io.IOException: No space left on device
>		at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
>		at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) ~[?:?]
>		at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) ~[?:?]
>		at sun.nio.ch.IOUtil.write(IOUtil.java:65) ~[?:?]
>		at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) ~[?:?]
>		at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) ~[?:1.8.0_181]
>		at java.nio.channels.Channels.writeFully(Channels.java:101) ~[?:1.8.0_181]
>		at java.nio.channels.Channels.access$000(Channels.java:61) ~[?:1.8.0_181]
>		at java.nio.channels.Channels$1.write(Channels.java:174) ~[?:1.8.0_181]
>		at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) ~[?:1.8.0_181]
>		at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) ~[?:1.8.0_181]
>		at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) ~[?:1.8.0_181]
>		at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:68) ~[lucene-core-6.6.1.jar:6.6.1 9aa465a89b64ff2dabe7b4d50c472de32c298683 - varunthacker - 2017-08-29 21:54:39]
>		at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:141) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>		at org.elasticsearch.env.NodeEnvironment.loadOrCreateNodeMetaData(NodeEnvironment.java:419) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>		at org.elasticsearch.env.NodeEnvironment.<init>(NodeEnvironment.java:263) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>		at org.elasticsearch.node.Node.<init>(Node.java:265) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>		at org.elasticsearch.node.Node.<init>(Node.java:245) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>		at org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:233) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>		at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:233) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>		at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:342) [elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>		at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:132) [elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>		at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:123) [elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>		at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:70) [elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>		at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:134) [elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>		at org.elasticsearch.cli.Command.main(Command.java:90) [elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>		at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:91) [elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>		at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:84) [elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>		Suppressed: java.io.IOException: No space left on device
>			at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
>			at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) ~[?:?]
>			at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) ~[?:?]
>			at sun.nio.ch.IOUtil.write(IOUtil.java:65) ~[?:?]
>			at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) ~[?:?]
>			at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) ~[?:1.8.0_181]
>			at java.nio.channels.Channels.writeFully(Channels.java:101) ~[?:1.8.0_181]
>			at java.nio.channels.Channels.access$000(Channels.java:61) ~[?:1.8.0_181]
>			at java.nio.channels.Channels$1.write(Channels.java:174) ~[?:1.8.0_181]
>			at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) ~[?:1.8.0_181]
>			at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) ~[?:1.8.0_181]
>			at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) ~[?:1.8.0_181]
>			at java.io.FilterOutputStream.close(FilterOutputStream.java:158) ~[?:1.8.0_181]
>			at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:70) ~[lucene-core-6.6.1.jar:6.6.1 9aa465a89b64ff2dabe7b4d50c472de32c298683 - varunthacker - 2017-08-29 21:54:39]
>			at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:141) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>			at org.elasticsearch.env.NodeEnvironment.loadOrCreateNodeMetaData(NodeEnvironment.java:419) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>			at org.elasticsearch.env.NodeEnvironment.<init>(NodeEnvironment.java:263) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>			at org.elasticsearch.node.Node.<init>(Node.java:265) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>			at org.elasticsearch.node.Node.<init>(Node.java:245) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>			at org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:233) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>			at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:233) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>			at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:342) [elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>			at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:132) [elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>			at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:123) [elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>			at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:70) [elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>			at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:134) [elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>			at org.elasticsearch.cli.Command.main(Command.java:90) [elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>			at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:91) [elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>			at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:84) [elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>') error: org.apache.logging.log4j.core.appender.AppenderLoggingException: Error writing to stream /elasticsearch/persistent/logging-es/logs/logging-es.log org.apache.logging.log4j.core.appender.AppenderLoggingException: Error writing to stream /elasticsearch/persistent/logging-es/logs/logging-es.log
>	at org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:264)
>	at org.apache.logging.log4j.core.appender.FileManager.writeToDestination(FileManager.java:261)
>	at org.apache.logging.log4j.core.appender.rolling.RollingFileManager.writeToDestination(RollingFileManager.java:219)
>	at org.apache.logging.log4j.core.appender.OutputStreamManager.flushBuffer(OutputStreamManager.java:294)
>	at org.apache.logging.log4j.core.appender.OutputStreamManager.drain(OutputStreamManager.java:351)
>	at org.apache.logging.log4j.core.layout.ByteBufferDestinationHelper.writeToUnsynchronized(ByteBufferDestinationHelper.java:45)
>	at org.apache.logging.log4j.core.layout.TextEncoderHelper.drainIfByteBufferFull(TextEncoderHelper.java:256)
>	at org.apache.logging.log4j.core.layout.TextEncoderHelper.writeAndEncodeAsMuchAsPossible(TextEncoderHelper.java:199)
>	at org.apache.logging.log4j.core.layout.TextEncoderHelper.writeChunkedEncodedText(TextEncoderHelper.java:112)
>	at org.apache.logging.log4j.core.layout.TextEncoderHelper.writeEncodedText(TextEncoderHelper.java:79)
>	at org.apache.logging.log4j.core.layout.TextEncoderHelper.encodeChunkedText(TextEncoderHelper.java:143)
>	at org.apache.logging.log4j.core.layout.TextEncoderHelper.encodeText(TextEncoderHelper.java:58)
>	at org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:68)
>	at org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:32)
>	at org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:219)
>	at org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:57)
>	at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:177)
>	at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:170)
>	at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:161)
>	at org.apache.logging.log4j.core.appender.RollingFileAppender.append(RollingFileAppender.java:308)
>	at org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:156)
>	at org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:129)
>	at org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:120)
>	at org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:84)
>	at org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:448)
>	at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:433)
>	at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
>	at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:403)
>	at org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:63)
>	at org.apache.logging.log4j.core.Logger.logMessage(Logger.java:146)
>	at org.apache.logging.log4j.spi.ExtendedLoggerWrapper.logMessage(ExtendedLoggerWrapper.java:217)
>	at org.elasticsearch.common.logging.PrefixLogger.logMessage(PrefixLogger.java:102)
>	at org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2116)
>	at org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2100)
>	at org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:1994)
>	at org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1966)
>	at org.apache.logging.log4j.spi.AbstractLogger.error(AbstractLogger.java:739)
>	at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:390)
>	at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:132)
>	at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:123)
>	at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:70)
>	at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:134)
>	at org.elasticsearch.cli.Command.main(Command.java:90)
>	at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:91)
>	at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:84)
>Caused by: java.io.IOException: No space left on device
>	at java.io.FileOutputStream.writeBytes(Native Method)
>	at java.io.FileOutputStream.write(FileOutputStream.java:326)
>	at org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:262)
>	... 44 more
>
>2018-08-15 03:24:45,220 main ERROR Unable to write to stream /elasticsearch/persistent/logging-es/logs/logging-es.log for appender rolling: org.apache.logging.log4j.core.appender.AppenderLoggingException: Error writing to stream /elasticsearch/persistent/logging-es/logs/logging-es.log
>2018-08-15 03:24:45,221 main ERROR An exception occurred processing Appender rolling org.apache.logging.log4j.core.appender.AppenderLoggingException: Error writing to stream /elasticsearch/persistent/logging-es/logs/logging-es.log
>	at org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:264)
>	at org.apache.logging.log4j.core.appender.FileManager.writeToDestination(FileManager.java:261)
>	at org.apache.logging.log4j.core.appender.rolling.RollingFileManager.writeToDestination(RollingFileManager.java:219)
>	at org.apache.logging.log4j.core.appender.OutputStreamManager.write(OutputStreamManager.java:240)
>	at org.apache.logging.log4j.core.appender.FileManager.write(FileManager.java:228)
>	at org.apache.logging.log4j.core.appender.rolling.RollingFileManager.write(RollingFileManager.java:213)
>	at org.apache.logging.log4j.core.appender.OutputStreamManager.writeBytes(OutputStreamManager.java:207)
>	at org.apache.logging.log4j.core.layout.TextEncoderHelper.encodeTextFallBack(TextEncoderHelper.java:39)
>	at org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:71)
>	at org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:32)
>	at org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:219)
>	at org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:57)
>	at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:177)
>	at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:170)
>	at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:161)
>	at org.apache.logging.log4j.core.appender.RollingFileAppender.append(RollingFileAppender.java:308)
>	at org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:156)
>	at org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:129)
>	at org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:120)
>	at org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:84)
>	at org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:448)
>	at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:433)
>	at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
>	at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:403)
>	at org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:63)
>	at org.apache.logging.log4j.core.Logger.logMessage(Logger.java:146)
>	at org.apache.logging.log4j.spi.ExtendedLoggerWrapper.logMessage(ExtendedLoggerWrapper.java:217)
>	at org.elasticsearch.common.logging.PrefixLogger.logMessage(PrefixLogger.java:102)
>	at org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2116)
>	at org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2100)
>	at org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:1994)
>	at org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1966)
>	at org.apache.logging.log4j.spi.AbstractLogger.error(AbstractLogger.java:739)
>	at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:390)
>	at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:132)
>	at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:123)
>	at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:70)
>	at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:134)
>	at org.elasticsearch.cli.Command.main(Command.java:90)
>	at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:91)
>	at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:84)
>Caused by: java.io.IOException: No space left on device
>	at java.io.FileOutputStream.writeBytes(Native Method)
>	at java.io.FileOutputStream.write(FileOutputStream.java:326)
>	at org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:262)
>	... 40 more
>
>2018-08-15 03:24:45,230 main ERROR Recovering from StringBuilderEncoder.encode('[2018-08-15T03:24:45,224][WARN ][o.e.b.ElasticsearchUncaughtExceptionHandler] [logging-es-data-master-w6l9n07t] uncaught exception in thread [main]
>org.elasticsearch.bootstrap.StartupException: java.lang.IllegalStateException: Failed to create node environment
>	at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:136) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:123) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:70) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:134) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.cli.Command.main(Command.java:90) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:91) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:84) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>Caused by: java.lang.IllegalStateException: Failed to create node environment
>	at org.elasticsearch.node.Node.<init>(Node.java:268) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.node.Node.<init>(Node.java:245) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:233) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:233) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:342) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:132) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	... 6 more
>Caused by: java.io.IOException: No space left on device
>	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
>	at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) ~[?:?]
>	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) ~[?:?]
>	at sun.nio.ch.IOUtil.write(IOUtil.java:65) ~[?:?]
>	at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) ~[?:?]
>	at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) ~[?:1.8.0_181]
>	at java.nio.channels.Channels.writeFully(Channels.java:101) ~[?:1.8.0_181]
>	at java.nio.channels.Channels.access$000(Channels.java:61) ~[?:1.8.0_181]
>	at java.nio.channels.Channels$1.write(Channels.java:174) ~[?:1.8.0_181]
>	at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) ~[?:1.8.0_181]
>	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) ~[?:1.8.0_181]
>	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) ~[?:1.8.0_181]
>	at org.apache.lucene.store.OutputStreamIndexOutput.getChecksum(OutputStreamIndexOutput.java:80) ~[lucene-core-6.6.1.jar:6.6.1 9aa465a89b64ff2dabe7b4d50c472de32c298683 - varunthacker - 2017-08-29 21:54:39]
>	at org.apache.lucene.codecs.CodecUtil.writeCRC(CodecUtil.java:548) ~[lucene-core-6.6.1.jar:6.6.1 9aa465a89b64ff2dabe7b4d50c472de32c298683 - varunthacker - 2017-08-29 21:54:39]
>	at org.apache.lucene.codecs.CodecUtil.writeFooter(CodecUtil.java:393) ~[lucene-core-6.6.1.jar:6.6.1 9aa465a89b64ff2dabe7b4d50c472de32c298683 - varunthacker - 2017-08-29 21:54:39]
>	at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:140) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.env.NodeEnvironment.loadOrCreateNodeMetaData(NodeEnvironment.java:419) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.env.NodeEnvironment.<init>(NodeEnvironment.java:263) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.node.Node.<init>(Node.java:265) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.node.Node.<init>(Node.java:245) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:233) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:233) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:342) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:132) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>	... 6 more
>	Suppressed: java.io.IOException: No space left on device
>		at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
>		at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) ~[?:?]
>		at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) ~[?:?]
>		at sun.nio.ch.IOUtil.write(IOUtil.java:65) ~[?:?]
>		at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) ~[?:?]
>		at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) ~[?:1.8.0_181]
>		at java.nio.channels.Channels.writeFully(Channels.java:101) ~[?:1.8.0_181]
>		at java.nio.channels.Channels.access$000(Channels.java:61) ~[?:1.8.0_181]
>		at java.nio.channels.Channels$1.write(Channels.java:174) ~[?:1.8.0_181]
>		at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) ~[?:1.8.0_181]
>		at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) ~[?:1.8.0_181]
>		at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) ~[?:1.8.0_181]
>		at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:68) ~[lucene-core-6.6.1.jar:6.6.1 9aa465a89b64ff2dabe7b4d50c472de32c298683 - varunthacker - 2017-08-29 21:54:39]
>		at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:141) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>		at org.elasticsearch.env.NodeEnvironment.loadOrCreateNodeMetaData(NodeEnvironment.java:419) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>		at org.elasticsearch.env.NodeEnvironment.<init>(NodeEnvironment.java:263) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>		at org.elasticsearch.node.Node.<init>(Node.java:265) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>		at org.elasticsearch.node.Node.<init>(Node.java:245) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>		at org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:233) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>		at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:233) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>		at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:342) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>		at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:132) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>		at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:123) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>		at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:70) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>		at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:134) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>		at org.elasticsearch.cli.Command.main(Command.java:90) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>		at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:91) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>		at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:84) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>		Suppressed: java.io.IOException: No space left on device
>			at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
>			at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) ~[?:?]
>			at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) ~[?:?]
>			at sun.nio.ch.IOUtil.write(IOUtil.java:65) ~[?:?]
>			at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) ~[?:?]
>			at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) ~[?:1.8.0_181]
>			at java.nio.channels.Channels.writeFully(Channels.java:101) ~[?:1.8.0_181]
>			at java.nio.channels.Channels.access$000(Channels.java:61) ~[?:1.8.0_181]
>			at java.nio.channels.Channels$1.write(Channels.java:174) ~[?:1.8.0_181]
>			at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) ~[?:1.8.0_181]
>			at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) ~[?:1.8.0_181]
>			at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) ~[?:1.8.0_181]
>			at java.io.FilterOutputStream.close(FilterOutputStream.java:158) ~[?:1.8.0_181]
>			at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:70) ~[lucene-core-6.6.1.jar:6.6.1 9aa465a89b64ff2dabe7b4d50c472de32c298683 - varunthacker - 2017-08-29 21:54:39]
>			at org.elasticsearch.gateway.MetaDataStateFormat.write(MetaDataStateFormat.java:141) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>			at org.elasticsearch.env.NodeEnvironment.loadOrCreateNodeMetaData(NodeEnvironment.java:419) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>			at org.elasticsearch.env.NodeEnvironment.<init>(NodeEnvironment.java:263) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>			at org.elasticsearch.node.Node.<init>(Node.java:265) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>			at org.elasticsearch.node.Node.<init>(Node.java:245) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>			at org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:233) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>			at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:233) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>			at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:342) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>			at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:132) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>			at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:123) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>			at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:70) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>			at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:134) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>			at org.elasticsearch.cli.Command.main(Command.java:90) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>			at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:91) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>			at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:84) ~[elasticsearch-5.6.10.redhat-1.jar:5.6.10.redhat-1]
>') error: org.apache.logging.log4j.core.appender.AppenderLoggingException: Error writing to stream /elasticsearch/persistent/logging-es/logs/logging-es.log org.apache.logging.log4j.core.appender.AppenderLoggingException: Error writing to stream /elasticsearch/persistent/logging-es/logs/logging-es.log
>	at org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:264)
>	at org.apache.logging.log4j.core.appender.FileManager.writeToDestination(FileManager.java:261)
>	at org.apache.logging.log4j.core.appender.rolling.RollingFileManager.writeToDestination(RollingFileManager.java:219)
>	at org.apache.logging.log4j.core.appender.OutputStreamManager.flushBuffer(OutputStreamManager.java:294)
>	at org.apache.logging.log4j.core.appender.OutputStreamManager.drain(OutputStreamManager.java:351)
>	at org.apache.logging.log4j.core.layout.TextEncoderHelper.drainIfByteBufferFull(TextEncoderHelper.java:260)
>	at org.apache.logging.log4j.core.layout.TextEncoderHelper.writeAndEncodeAsMuchAsPossible(TextEncoderHelper.java:199)
>	at org.apache.logging.log4j.core.layout.TextEncoderHelper.encodeChunkedText(TextEncoderHelper.java:147)
>	at org.apache.logging.log4j.core.layout.TextEncoderHelper.encodeText(TextEncoderHelper.java:58)
>	at org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:68)
>	at org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:32)
>	at org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:219)
>	at org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:57)
>	at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:177)
>	at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:170)
>	at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:161)
>	at org.apache.logging.log4j.core.appender.RollingFileAppender.append(RollingFileAppender.java:308)
>	at org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:156)
>	at org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:129)
>	at org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:120)
>	at org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:84)
>	at org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:448)
>	at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:433)
>	at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
>	at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:403)
>	at org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:63)
>	at org.apache.logging.log4j.core.Logger.logMessage(Logger.java:146)
>	at org.apache.logging.log4j.spi.ExtendedLoggerWrapper.logMessage(ExtendedLoggerWrapper.java:217)
>	at org.elasticsearch.common.logging.PrefixLogger.logMessage(PrefixLogger.java:102)
>	at org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2116)
>	at org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2100)
>	at org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:1989)
>	at org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1851)
>	at org.apache.logging.log4j.spi.AbstractLogger.warn(AbstractLogger.java:2589)
>	at org.elasticsearch.bootstrap.ElasticsearchUncaughtExceptionHandler.onNonFatalUncaught(ElasticsearchUncaughtExceptionHandler.java:81)
>	at org.elasticsearch.bootstrap.ElasticsearchUncaughtExceptionHandler.uncaughtException(ElasticsearchUncaughtExceptionHandler.java:64)
>	at java.lang.ThreadGroup.uncaughtException(ThreadGroup.java:1057)
>	at java.lang.ThreadGroup.uncaughtException(ThreadGroup.java:1052)
>	at java.lang.Thread.dispatchUncaughtException(Thread.java:1959)
>Caused by: java.io.IOException: No space left on device
>	at java.io.FileOutputStream.writeBytes(Native Method)
>	at java.io.FileOutputStream.write(FileOutputStream.java:326)
>	at org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:262)
>	... 38 more
>

Actions: View

Attachments on bug 1616169: 1476062