Attachment 916549 Details for Bug 1117505 – master-agent.log

master-agent.log

master-agent.log (text/x-log), 37.77 KB, created by Armine Hovsepyan on 2014-07-08 20:18:35 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Armine Hovsepyan

Created: 2014-07-08 20:18:35 UTC

Size: 37.77 KB

patch

obsolete

>2014-07-08 16:10:00,420 WARN  [RHQ Server Polling Thread] (enterprise.communications.command.client.ServerPollingThread)- {ServerPollingThread.server-offline}The server has gone offline; client has been told to stop sending commands
>	^[OS2014-07-08 16:10:23,253 INFO  [ResourceContainer.invoker.daemon-3] (org.rhq.plugins.server.JBossAS7JMXComponent)- Loading AS7 connection [service:jmx:remoting-jmx://127.0.0.1:6999] with install path [null]...
>2014-07-08 16:10:23,263 ERROR [Remoting "endpoint" read-1] (org.xnio.listener)- A channel event listener threw an exception
>java.lang.NoClassDefFoundError: Could not initialize class org.jboss.sasl.digest.DigestMD5Client
>	at org.jboss.sasl.digest.DigestMD5ClientFactory.createSaslClient(DigestMD5ClientFactory.java:41)
>	at org.jboss.remoting3.remote.ClientConnectionOpenListener$Capabilities$1.run(ClientConnectionOpenListener.java:428)
>	at org.jboss.remoting3.remote.ClientConnectionOpenListener$Capabilities$1.run(ClientConnectionOpenListener.java:426)
>	at java.security.AccessController.doPrivileged(Native Method)
>	at org.jboss.remoting3.remote.ClientConnectionOpenListener$Capabilities.handleEvent(ClientConnectionOpenListener.java:426)
>	at org.jboss.remoting3.remote.ClientConnectionOpenListener$Capabilities.handleEvent(ClientConnectionOpenListener.java:242)
>	at org.xnio.ChannelListeners.invokeChannelListener(ChannelListeners.java:72)
>	at org.xnio.channels.TranslatingSuspendableChannel.handleReadable(TranslatingSuspendableChannel.java:189)
>	at org.xnio.channels.TranslatingSuspendableChannel$1.handleEvent(TranslatingSuspendableChannel.java:103)
>	at org.xnio.ChannelListeners.invokeChannelListener(ChannelListeners.java:72)
>	at org.xnio.channels.TranslatingSuspendableChannel.handleReadable(TranslatingSuspendableChannel.java:189)
>	at org.xnio.ssl.JsseConnectedSslStreamChannel.handleReadable(JsseConnectedSslStreamChannel.java:180)
>	at org.xnio.channels.TranslatingSuspendableChannel$1.handleEvent(TranslatingSuspendableChannel.java:103)
>	at org.xnio.ChannelListeners.invokeChannelListener(ChannelListeners.java:72)
>	at org.xnio.nio.NioHandle.run(NioHandle.java:90)
>	at org.xnio.nio.WorkerThread.run(WorkerThread.java:198)
>2014-07-08 16:10:28,264 WARN  [InventoryManager.availability-1] (rhq.core.pc.inventory.AvailabilityExecutor)- Availability collection failed with exception on Resource[id=10067, uuid=87547a87-9469-4db5-baf4-a23100e3531f, type={RHQServer}RHQ Server Subsystem Services, key=RHQServerSubsystems, name=RHQ Server Subsystems, parent=EAP (127.0.0.1:6990) RHQ Server, version=EAP 6.3.0.Alpha1], availability will be reported as DOWN, reason=Failed to start component for Resource[id=10067, uuid=87547a87-9469-4db5-baf4-a23100e3531f, type={RHQServer}RHQ Server Subsystem Services, key=RHQServerSubsystems, name=RHQ Server Subsystems, parent=EAP (127.0.0.1:6990) RHQ Server, version=EAP 6.3.0.Alpha1] - Could not connect [service:jmx:remoting-jmx://127.0.0.1:6999] java.lang.RuntimeException: Operation failed with status WAITING - Cause: org.mc4j.ems.connection.EmsConnectException: Operation failed with status WAITING
>2014-07-08 16:10:28,347 INFO  [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Sending availability report to Server...
>2014-07-08 16:10:28,348 WARN  [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Could not transmit availability report to server
>java.lang.IllegalStateException: The sender object is currently not sending commands now. Command not sent: [Command: type=[remotepojo]; cmd-in-response=[false]; config=[{rhq.send-throttle=true}]; params=[{invocation=NameBasedInvocation[mergeAvailabilityReport], targetInterfaceName=org.rhq.core.clientapi.server.discovery.DiscoveryServerService}]]
>	at org.rhq.enterprise.communications.command.client.ClientCommandSender.sendSynch(ClientCommandSender.java:623)
>	at org.rhq.enterprise.communications.command.client.ClientRemotePojoFactory$RemotePojoProxyHandler.invoke(ClientRemotePojoFactory.java:418)
>	at com.sun.proxy.$Proxy3.mergeAvailabilityReport(Unknown Source)
>	at org.rhq.core.pc.inventory.InventoryManager.handleReport(InventoryManager.java:1183)
>	at org.rhq.core.pc.inventory.AvailabilityExecutor.run(AvailabilityExecutor.java:90)
>	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
>	at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
>	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
>	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
>	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
>	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
>	at java.lang.Thread.run(Thread.java:744)
>2014-07-08 16:10:58,396 INFO  [ResourceContainer.invoker.daemon-3] (org.rhq.plugins.server.JBossAS7JMXComponent)- Loading AS7 connection [service:jmx:remoting-jmx://127.0.0.1:6999] with install path [null]...
>2014-07-08 16:10:58,408 ERROR [Remoting "endpoint" read-1] (org.xnio.listener)- A channel event listener threw an exception
>java.lang.NoClassDefFoundError: Could not initialize class org.jboss.sasl.digest.DigestMD5Client
>	at org.jboss.sasl.digest.DigestMD5ClientFactory.createSaslClient(DigestMD5ClientFactory.java:41)
>	at org.jboss.remoting3.remote.ClientConnectionOpenListener$Capabilities$1.run(ClientConnectionOpenListener.java:428)
>	at org.jboss.remoting3.remote.ClientConnectionOpenListener$Capabilities$1.run(ClientConnectionOpenListener.java:426)
>	at java.security.AccessController.doPrivileged(Native Method)
>	at org.jboss.remoting3.remote.ClientConnectionOpenListener$Capabilities.handleEvent(ClientConnectionOpenListener.java:426)
>	at org.jboss.remoting3.remote.ClientConnectionOpenListener$Capabilities.handleEvent(ClientConnectionOpenListener.java:242)
>	at org.xnio.ChannelListeners.invokeChannelListener(ChannelListeners.java:72)
>	at org.xnio.channels.TranslatingSuspendableChannel.handleReadable(TranslatingSuspendableChannel.java:189)
>	at org.xnio.channels.TranslatingSuspendableChannel$1.handleEvent(TranslatingSuspendableChannel.java:103)
>	at org.xnio.ChannelListeners.invokeChannelListener(ChannelListeners.java:72)
>	at org.xnio.channels.TranslatingSuspendableChannel.handleReadable(TranslatingSuspendableChannel.java:189)
>	at org.xnio.ssl.JsseConnectedSslStreamChannel.handleReadable(JsseConnectedSslStreamChannel.java:180)
>	at org.xnio.channels.TranslatingSuspendableChannel$1.handleEvent(TranslatingSuspendableChannel.java:103)
>	at org.xnio.ChannelListeners.invokeChannelListener(ChannelListeners.java:72)
>	at org.xnio.nio.NioHandle.run(NioHandle.java:90)
>	at org.xnio.nio.WorkerThread.run(WorkerThread.java:198)
>2014-07-08 16:11:00,575 INFO  [RHQ Server Polling Thread] (enterprise.communications.command.client.ServerPollingThread)- {ServerPollingThread.server-online}The server has come back online; client has been told to start sending commands again
>2014-07-08 16:11:03,406 WARN  [ResourceContainer.invoker.daemon-3] (org.rhq.plugins.server.JBossAS7JMXComponent)- Could not establish connection to the JBoss AS instance [11] times for resource [RHQServerSubsystems]
>org.mc4j.ems.connection.EmsConnectException: Could not connect [service:jmx:remoting-jmx://127.0.0.1:6999] java.lang.RuntimeException: Operation failed with status WAITING
>	at org.mc4j.ems.impl.jmx.connection.support.providers.AbstractConnectionProvider.connect(AbstractConnectionProvider.java:106)
>	at org.rhq.plugins.server.JBossAS7JMXComponent.loadConnection(JBossAS7JMXComponent.java:148)
>	at org.rhq.plugins.server.JBossAS7JMXComponent.start(JBossAS7JMXComponent.java:98)
>	at sun.reflect.GeneratedMethodAccessor51.invoke(Unknown Source)
>	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>	at java.lang.reflect.Method.invoke(Method.java:606)
>	at org.rhq.core.pc.inventory.ResourceContainer$ComponentInvocation.call(ResourceContainer.java:759)
>	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
>	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
>	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
>	at java.lang.Thread.run(Thread.java:744)
>Caused by: java.lang.RuntimeException: Operation failed with status WAITING
>	at org.jboss.remotingjmx.RemotingConnector.internalRemotingConnect(RemotingConnector.java:235)
>	at org.jboss.remotingjmx.RemotingConnector.internalConnect(RemotingConnector.java:148)
>	at org.jboss.remotingjmx.RemotingConnector.connect(RemotingConnector.java:99)
>	at javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:268)
>	at org.mc4j.ems.impl.jmx.connection.support.providers.JMXRemotingConnectionProvider.doConnect(JMXRemotingConnectionProvider.java:143)
>	at org.mc4j.ems.impl.jmx.connection.support.providers.AbstractConnectionProvider.connect(AbstractConnectionProvider.java:101)
>	... 10 more
>2014-07-08 16:11:03,407 WARN  [InventoryManager.availability-1] (rhq.core.pc.inventory.AvailabilityExecutor)- Availability collection failed with exception on Resource[id=10067, uuid=87547a87-9469-4db5-baf4-a23100e3531f, type={RHQServer}RHQ Server Subsystem Services, key=RHQServerSubsystems, name=RHQ Server Subsystems, parent=EAP (127.0.0.1:6990) RHQ Server, version=EAP 6.3.0.Alpha1], availability will be reported as DOWN, reason=Failed to start component for Resource[id=10067, uuid=87547a87-9469-4db5-baf4-a23100e3531f, type={RHQServer}RHQ Server Subsystem Services, key=RHQServerSubsystems, name=RHQ Server Subsystems, parent=EAP (127.0.0.1:6990) RHQ Server, version=EAP 6.3.0.Alpha1] - Could not connect [service:jmx:remoting-jmx://127.0.0.1:6999] java.lang.RuntimeException: Operation failed with status WAITING - Cause: org.mc4j.ems.connection.EmsConnectException: Operation failed with status WAITING
>2014-07-08 16:11:03,598 INFO  [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Sending availability report to Server...
>2014-07-08 16:11:29,927 INFO  [ClientCommandSenderTask Timer Thread #21] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.changing-endpoint}Communicator is changing endpoint from [InvokerLocator [servlet://uninventory.bc.jonqe.lab.eng.bos.redhat.com:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]] to [InvokerLocator [servlet://uninventory.bc.jonqe.lab.eng.bos.redhat.com:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]]
>2014-07-08 16:11:29,927 INFO  [ClientCommandSenderTask Timer Thread #21] (org.rhq.enterprise.agent.AgentMain)- {AgentMain.not-sending-dup-connect}Not sending another connect message since one was recently sent: [servlet://uninventory.bc.jonqe.lab.eng.bos.redhat.com:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet@Tue Jul 08 16:11:00 EDT 2014]
>2014-07-08 16:11:29,927 INFO  [ClientCommandSenderTask Timer Thread #21] (org.rhq.enterprise.agent.AgentMain)- {AgentMain.failed-over-to-server}The agent has triggered its failover mechanism and switched to server [servlet://uninventory.bc.jonqe.lab.eng.bos.redhat.com:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]
>2014-07-08 16:11:29,932 WARN  [ClientCommandSenderTask Timer Thread #21] (org.rhq.enterprise.agent.FailoverFailureCallback)- {AgentMain.too-many-failover-attempts}Too many failover attempts have been made [1]. Exception that triggered the failover: [org.rhq.enterprise.communications.util.NotProcessedException]
>2014-07-08 16:11:33,904 INFO  [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Sending availability report to Server...
>2014-07-08 16:11:33,911 WARN  [ClientCommandSenderTask Timer Thread #21] (org.rhq.enterprise.agent.FailoverFailureCallback)- {AgentMain.too-many-failover-attempts}Too many failover attempts have been made [1]. Exception that triggered the failover: [org.rhq.enterprise.communications.util.NotProcessedException]
>2014-07-08 16:11:33,911 ERROR [ClientCommandSenderTask Timer Thread #21] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.init-callback-failed}The initialize callback has failed. It will be tried again. Cause: org.rhq.enterprise.communications.util.NotProcessedException:null. Cause: org.rhq.enterprise.communications.util.NotProcessedException
>2014-07-08 16:11:33,912 WARN  [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Could not transmit availability report to server
>org.rhq.enterprise.communications.util.NotProcessedException
>	at org.rhq.enterprise.communications.GlobalSuspendCommandListener.receivedCommand(GlobalSuspendCommandListener.java:55)
>	at org.rhq.enterprise.communications.command.server.CommandProcessor.notifyListenersOfReceivedCommand(CommandProcessor.java:402)
>	at org.rhq.enterprise.communications.command.server.CommandProcessor.handleIncomingInvocationRequest(CommandProcessor.java:242)
>	at org.rhq.enterprise.communications.command.server.CommandProcessor.invoke(CommandProcessor.java:184)
>	at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:967)
>	at org.jboss.remoting.transport.servlet.ServletServerInvoker.processRequest(ServletServerInvoker.java:416)
>	at sun.reflect.GeneratedMethodAccessor143.invoke(Unknown Source)
>	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>	at java.lang.reflect.Method.invoke(Method.java:606)
>	at sun.reflect.misc.Trampoline.invoke(MethodUtil.java:75)
>	at sun.reflect.GeneratedMethodAccessor124.invoke(Unknown Source)
>	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>	at java.lang.reflect.Method.invoke(Method.java:606)
>	at sun.reflect.misc.MethodUtil.invoke(MethodUtil.java:279)
>	at com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:112)
>	at com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:46)
>	at com.sun.jmx.mbeanserver.MBeanIntrospector.invokeM(MBeanIntrospector.java:237)
>	at com.sun.jmx.mbeanserver.PerInterface.invoke(PerInterface.java:138)
>	at com.sun.jmx.mbeanserver.MBeanSupport.invoke(MBeanSupport.java:252)
>	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.invoke(DefaultMBeanServerInterceptor.java:819)
>	at com.sun.jmx.mbeanserver.JmxMBeanServer.invoke(JmxMBeanServer.java:801)
>	at org.jboss.as.jmx.PluggableMBeanServerImpl$TcclMBeanServer.invoke(PluggableMBeanServerImpl.java:1453)
>	at org.jboss.as.jmx.PluggableMBeanServerImpl.invoke(PluggableMBeanServerImpl.java:731)
>	at javax.management.MBeanServerInvocationHandler.invoke(MBeanServerInvocationHandler.java:292)
>	at com.sun.proxy.$Proxy156.processRequest(Unknown Source)
>	at org.jboss.remoting.transport.servlet.web.ServerInvokerServlet.processRequest(ServerInvokerServlet.java:404)
>	at org.jboss.remoting.transport.servlet.web.ServerInvokerServlet.processRequest(ServerInvokerServlet.java:142)
>	at org.rhq.enterprise.communications.servlet.ServerInvokerServlet.processRequest(ServerInvokerServlet.java:70)
>	at org.jboss.remoting.transport.servlet.web.ServerInvokerServlet.doPost(ServerInvokerServlet.java:171)
>	at javax.servlet.http.HttpServlet.service(HttpServlet.java:754)
>	at javax.servlet.http.HttpServlet.service(HttpServlet.java:847)
>	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:295)
>	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:214)
>	at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:231)
>	at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:149)
>	at org.jboss.as.jpa.interceptor.WebNonTxEmCloserValve.invoke(WebNonTxEmCloserValve.java:50)
>	at org.jboss.as.jpa.interceptor.WebNonTxEmCloserValve.invoke(WebNonTxEmCloserValve.java:50)
>	at org.jboss.as.web.security.SecurityContextAssociationValve.invoke(SecurityContextAssociationValve.java:169)
>	at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:145)
>	at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:97)
>	at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:102)
>	at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:340)
>	at org.apache.coyote.http11.Http11NioProcessor.process(Http11NioProcessor.java:353)
>	at org.apache.coyote.http11.Http11NioProtocol$Http11ConnectionHandler.process(Http11NioProtocol.java:911)
>	at org.apache.tomcat.util.net.NioEndpoint$ChannelProcessor.run(NioEndpoint.java:920)
>	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
>	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
>	at java.lang.Thread.run(Thread.java:744)
>2014-07-08 16:12:00,402 INFO  [RHQ Agent Ping Thread-1] (org.rhq.enterprise.agent.AgentMain)- {AgentMain.ping-executor.stop-polling-resume-ping}Stopping polling and resuming pinging
>2014-07-08 16:12:03,962 INFO  [ResourceContainer.invoker.daemon-3] (org.rhq.plugins.server.JBossAS7JMXComponent)- Loading AS7 connection [service:jmx:remoting-jmx://127.0.0.1:6999] with install path [null]...
>2014-07-08 16:12:03,974 ERROR [Remoting "endpoint" read-1] (org.xnio.listener)- A channel event listener threw an exception
>java.lang.NoClassDefFoundError: Could not initialize class org.jboss.sasl.digest.DigestMD5Client
>	at org.jboss.sasl.digest.DigestMD5ClientFactory.createSaslClient(DigestMD5ClientFactory.java:41)
>	at org.jboss.remoting3.remote.ClientConnectionOpenListener$Capabilities$1.run(ClientConnectionOpenListener.java:428)
>	at org.jboss.remoting3.remote.ClientConnectionOpenListener$Capabilities$1.run(ClientConnectionOpenListener.java:426)
>	at java.security.AccessController.doPrivileged(Native Method)
>	at org.jboss.remoting3.remote.ClientConnectionOpenListener$Capabilities.handleEvent(ClientConnectionOpenListener.java:426)
>	at org.jboss.remoting3.remote.ClientConnectionOpenListener$Capabilities.handleEvent(ClientConnectionOpenListener.java:242)
>	at org.xnio.ChannelListeners.invokeChannelListener(ChannelListeners.java:72)
>	at org.xnio.channels.TranslatingSuspendableChannel.handleReadable(TranslatingSuspendableChannel.java:189)
>	at org.xnio.channels.TranslatingSuspendableChannel$1.handleEvent(TranslatingSuspendableChannel.java:103)
>	at org.xnio.ChannelListeners.invokeChannelListener(ChannelListeners.java:72)
>	at org.xnio.channels.TranslatingSuspendableChannel.handleReadable(TranslatingSuspendableChannel.java:189)
>	at org.xnio.ssl.JsseConnectedSslStreamChannel.handleReadable(JsseConnectedSslStreamChannel.java:180)
>	at org.xnio.channels.TranslatingSuspendableChannel$1.handleEvent(TranslatingSuspendableChannel.java:103)
>	at org.xnio.ChannelListeners.invokeChannelListener(ChannelListeners.java:72)
>	at org.xnio.nio.NioHandle.run(NioHandle.java:90)
>	at org.xnio.nio.WorkerThread.run(WorkerThread.java:198)
>2014-07-08 16:12:08,971 WARN  [InventoryManager.availability-1] (rhq.core.pc.inventory.AvailabilityExecutor)- Availability collection failed with exception on Resource[id=10067, uuid=87547a87-9469-4db5-baf4-a23100e3531f, type={RHQServer}RHQ Server Subsystem Services, key=RHQServerSubsystems, name=RHQ Server Subsystems, parent=EAP (127.0.0.1:6990) RHQ Server, version=EAP 6.3.0.Alpha1], availability will be reported as DOWN, reason=Failed to start component for Resource[id=10067, uuid=87547a87-9469-4db5-baf4-a23100e3531f, type={RHQServer}RHQ Server Subsystem Services, key=RHQServerSubsystems, name=RHQ Server Subsystems, parent=EAP (127.0.0.1:6990) RHQ Server, version=EAP 6.3.0.Alpha1] - Could not connect [service:jmx:remoting-jmx://127.0.0.1:6999] java.lang.RuntimeException: Operation failed with status WAITING - Cause: org.mc4j.ems.connection.EmsConnectException: Operation failed with status WAITING
>2014-07-08 16:12:08,983 INFO  [ClientCommandSenderTask Timer Thread #21] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.changing-endpoint}Communicator is changing endpoint from [InvokerLocator [servlet://uninventory.bc.jonqe.lab.eng.bos.redhat.com:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]] to [InvokerLocator [servlet://uninventory.bc.jonqe.lab.eng.bos.redhat.com:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]]
>2014-07-08 16:12:08,983 INFO  [ClientCommandSenderTask Timer Thread #21] (org.rhq.enterprise.agent.AgentMain)- {AgentMain.not-sending-dup-connect}Not sending another connect message since one was recently sent: [servlet://uninventory.bc.jonqe.lab.eng.bos.redhat.com:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet@Tue Jul 08 16:12:00 EDT 2014]
>2014-07-08 16:12:08,984 INFO  [ClientCommandSenderTask Timer Thread #21] (org.rhq.enterprise.agent.AgentMain)- {AgentMain.failed-over-to-server}The agent has triggered its failover mechanism and switched to server [servlet://uninventory.bc.jonqe.lab.eng.bos.redhat.com:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]
>2014-07-08 16:12:08,989 WARN  [ClientCommandSenderTask Timer Thread #21] (org.rhq.enterprise.agent.FailoverFailureCallback)- {AgentMain.too-many-failover-attempts}Too many failover attempts have been made [1]. Exception that triggered the failover: [org.rhq.enterprise.communications.util.NotProcessedException]
>2014-07-08 16:12:09,035 INFO  [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Sending availability report to Server...
>2014-07-08 16:12:09,035 INFO  [ClientCommandSenderTask Timer Thread #21] (org.rhq.enterprise.agent.AgentMain)- {AgentMain.not-sending-dup-connect}Not sending another connect message since one was recently sent: [servlet://uninventory.bc.jonqe.lab.eng.bos.redhat.com:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet@Tue Jul 08 16:12:00 EDT 2014]
>2014-07-08 16:12:09,045 WARN  [ClientCommandSenderTask Timer Thread #21] (org.rhq.enterprise.agent.FailoverFailureCallback)- {AgentMain.too-many-failover-attempts}Too many failover attempts have been made [1]. Exception that triggered the failover: [org.rhq.enterprise.communications.util.NotProcessedException]
>2014-07-08 16:12:09,045 WARN  [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Could not transmit availability report to server
>org.rhq.enterprise.communications.util.NotProcessedException
>	at org.rhq.enterprise.communications.GlobalSuspendCommandListener.receivedCommand(GlobalSuspendCommandListener.java:55)
>	at org.rhq.enterprise.communications.command.server.CommandProcessor.notifyListenersOfReceivedCommand(CommandProcessor.java:402)
>	at org.rhq.enterprise.communications.command.server.CommandProcessor.handleIncomingInvocationRequest(CommandProcessor.java:242)
>	at org.rhq.enterprise.communications.command.server.CommandProcessor.invoke(CommandProcessor.java:184)
>	at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:967)
>	at org.jboss.remoting.transport.servlet.ServletServerInvoker.processRequest(ServletServerInvoker.java:416)
>	at sun.reflect.GeneratedMethodAccessor143.invoke(Unknown Source)
>	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>	at java.lang.reflect.Method.invoke(Method.java:606)
>	at sun.reflect.misc.Trampoline.invoke(MethodUtil.java:75)
>	at sun.reflect.GeneratedMethodAccessor124.invoke(Unknown Source)
>	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>	at java.lang.reflect.Method.invoke(Method.java:606)
>	at sun.reflect.misc.MethodUtil.invoke(MethodUtil.java:279)
>	at com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:112)
>	at com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:46)
>	at com.sun.jmx.mbeanserver.MBeanIntrospector.invokeM(MBeanIntrospector.java:237)
>	at com.sun.jmx.mbeanserver.PerInterface.invoke(PerInterface.java:138)
>	at com.sun.jmx.mbeanserver.MBeanSupport.invoke(MBeanSupport.java:252)
>	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.invoke(DefaultMBeanServerInterceptor.java:819)
>	at com.sun.jmx.mbeanserver.JmxMBeanServer.invoke(JmxMBeanServer.java:801)
>	at org.jboss.as.jmx.PluggableMBeanServerImpl$TcclMBeanServer.invoke(PluggableMBeanServerImpl.java:1453)
>	at org.jboss.as.jmx.PluggableMBeanServerImpl.invoke(PluggableMBeanServerImpl.java:731)
>	at javax.management.MBeanServerInvocationHandler.invoke(MBeanServerInvocationHandler.java:292)
>	at com.sun.proxy.$Proxy156.processRequest(Unknown Source)
>	at org.jboss.remoting.transport.servlet.web.ServerInvokerServlet.processRequest(ServerInvokerServlet.java:404)
>	at org.jboss.remoting.transport.servlet.web.ServerInvokerServlet.processRequest(ServerInvokerServlet.java:142)
>	at org.rhq.enterprise.communications.servlet.ServerInvokerServlet.processRequest(ServerInvokerServlet.java:70)
>	at org.jboss.remoting.transport.servlet.web.ServerInvokerServlet.doPost(ServerInvokerServlet.java:171)
>	at javax.servlet.http.HttpServlet.service(HttpServlet.java:754)
>	at javax.servlet.http.HttpServlet.service(HttpServlet.java:847)
>	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:295)
>	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:214)
>	at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:231)
>	at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:149)
>	at org.jboss.as.jpa.interceptor.WebNonTxEmCloserValve.invoke(WebNonTxEmCloserValve.java:50)
>	at org.jboss.as.jpa.interceptor.WebNonTxEmCloserValve.invoke(WebNonTxEmCloserValve.java:50)
>	at org.jboss.as.web.security.SecurityContextAssociationValve.invoke(SecurityContextAssociationValve.java:169)
>	at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:145)
>	at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:97)
>	at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:102)
>	at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:340)
>	at org.apache.coyote.http11.Http11NioProcessor.process(Http11NioProcessor.java:353)
>	at org.apache.coyote.http11.Http11NioProtocol$Http11ConnectionHandler.process(Http11NioProtocol.java:911)
>	at org.apache.tomcat.util.net.NioEndpoint$ChannelProcessor.run(NioEndpoint.java:920)
>	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
>	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
>	at java.lang.Thread.run(Thread.java:744)
>2014-07-08 16:12:29,925 INFO  [ClientCommandSenderTask Timer Thread #21] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.changing-endpoint}Communicator is changing endpoint from [InvokerLocator [servlet://uninventory.bc.jonqe.lab.eng.bos.redhat.com:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]] to [InvokerLocator [servlet://uninventory.bc.jonqe.lab.eng.bos.redhat.com:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]]
>2014-07-08 16:12:29,925 INFO  [ClientCommandSenderTask Timer Thread #21] (org.rhq.enterprise.agent.AgentMain)- {AgentMain.not-sending-dup-connect}Not sending another connect message since one was recently sent: [servlet://uninventory.bc.jonqe.lab.eng.bos.redhat.com:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet@Tue Jul 08 16:12:00 EDT 2014]
>2014-07-08 16:12:29,925 INFO  [ClientCommandSenderTask Timer Thread #21] (org.rhq.enterprise.agent.AgentMain)- {AgentMain.failed-over-to-server}The agent has triggered its failover mechanism and switched to server [servlet://uninventory.bc.jonqe.lab.eng.bos.redhat.com:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]
>2014-07-08 16:12:29,930 WARN  [ClientCommandSenderTask Timer Thread #21] (org.rhq.enterprise.agent.FailoverFailureCallback)- {AgentMain.too-many-failover-attempts}Too many failover attempts have been made [1]. Exception that triggered the failover: [org.rhq.enterprise.communications.util.NotProcessedException]
>2014-07-08 16:12:39,048 INFO  [ResourceContainer.invoker.daemon-3] (org.rhq.plugins.server.JBossAS7JMXComponent)- Loading AS7 connection [service:jmx:remoting-jmx://127.0.0.1:6999] with install path [null]...
>2014-07-08 16:12:39,065 ERROR [Remoting "endpoint" read-1] (org.xnio.listener)- A channel event listener threw an exception
>java.lang.NoClassDefFoundError: Could not initialize class org.jboss.sasl.digest.DigestMD5Client
>	at org.jboss.sasl.digest.DigestMD5ClientFactory.createSaslClient(DigestMD5ClientFactory.java:41)
>	at org.jboss.remoting3.remote.ClientConnectionOpenListener$Capabilities$1.run(ClientConnectionOpenListener.java:428)
>	at org.jboss.remoting3.remote.ClientConnectionOpenListener$Capabilities$1.run(ClientConnectionOpenListener.java:426)
>	at java.security.AccessController.doPrivileged(Native Method)
>	at org.jboss.remoting3.remote.ClientConnectionOpenListener$Capabilities.handleEvent(ClientConnectionOpenListener.java:426)
>	at org.jboss.remoting3.remote.ClientConnectionOpenListener$Capabilities.handleEvent(ClientConnectionOpenListener.java:242)
>	at org.xnio.ChannelListeners.invokeChannelListener(ChannelListeners.java:72)
>	at org.xnio.channels.TranslatingSuspendableChannel.handleReadable(TranslatingSuspendableChannel.java:189)
>	at org.xnio.channels.TranslatingSuspendableChannel$1.handleEvent(TranslatingSuspendableChannel.java:103)
>	at org.xnio.ChannelListeners.invokeChannelListener(ChannelListeners.java:72)
>	at org.xnio.channels.TranslatingSuspendableChannel.handleReadable(TranslatingSuspendableChannel.java:189)
>	at org.xnio.ssl.JsseConnectedSslStreamChannel.handleReadable(JsseConnectedSslStreamChannel.java:180)
>	at org.xnio.channels.TranslatingSuspendableChannel$1.handleEvent(TranslatingSuspendableChannel.java:103)
>	at org.xnio.ChannelListeners.invokeChannelListener(ChannelListeners.java:72)
>	at org.xnio.nio.NioHandle.run(NioHandle.java:90)
>	at org.xnio.nio.WorkerThread.run(WorkerThread.java:198)
>2014-07-08 16:12:44,066 ERROR [Remoting "endpoint" read-1] (org.jboss.remoting.remote.connection)- JBREM000200: Remote connection failed: java.io.IOException: JBREM000202: Abrupt close on Remoting connection 15b4233e to localhost/127.0.0.1:6999
>2014-07-08 16:12:44,068 WARN  [InventoryManager.availability-1] (rhq.core.pc.inventory.AvailabilityExecutor)- Availability collection failed with exception on Resource[id=10067, uuid=87547a87-9469-4db5-baf4-a23100e3531f, type={RHQServer}RHQ Server Subsystem Services, key=RHQServerSubsystems, name=RHQ Server Subsystems, parent=EAP (127.0.0.1:6990) RHQ Server, version=EAP 6.3.0.Alpha1], availability will be reported as DOWN, reason=Failed to start component for Resource[id=10067, uuid=87547a87-9469-4db5-baf4-a23100e3531f, type={RHQServer}RHQ Server Subsystem Services, key=RHQServerSubsystems, name=RHQ Server Subsystems, parent=EAP (127.0.0.1:6990) RHQ Server, version=EAP 6.3.0.Alpha1] - Could not connect [service:jmx:remoting-jmx://127.0.0.1:6999] java.lang.RuntimeException: Operation failed with status WAITING - Cause: org.mc4j.ems.connection.EmsConnectException: Operation failed with status WAITING
>2014-07-08 16:12:44,079 INFO  [ClientCommandSenderTask Timer Thread #21] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.changing-endpoint}Communicator is changing endpoint from [InvokerLocator [servlet://uninventory.bc.jonqe.lab.eng.bos.redhat.com:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]] to [InvokerLocator [servlet://uninventory.bc.jonqe.lab.eng.bos.redhat.com:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]]
>2014-07-08 16:12:44,085 WARN  [ClientCommandSenderTask Timer Thread #21] (org.rhq.enterprise.agent.AgentMain)- {AgentMain.failover-failed}Failed to failover to another server. Cause: org.rhq.enterprise.communications.util.NotProcessedException
>2014-07-08 16:12:44,088 WARN  [ClientCommandSenderTask Timer Thread #21] (org.rhq.enterprise.agent.FailoverFailureCallback)- {AgentMain.too-many-failover-attempts}Too many failover attempts have been made [1]. Exception that triggered the failover: [org.rhq.enterprise.communications.util.NotProcessedException]
>2014-07-08 16:12:44,088 ERROR [ClientCommandSenderTask Timer Thread #21] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.init-callback-failed}The initialize callback has failed. It will be tried again. Cause: org.rhq.enterprise.communications.util.NotProcessedException:null. Cause: org.rhq.enterprise.communications.util.NotProcessedException
>2014-07-08 16:12:44,213 INFO  [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Sending availability report to Server...
>2014-07-08 16:12:44,222 INFO  [ClientCommandSenderTask Timer Thread #21] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.changing-endpoint}Communicator is changing endpoint from [InvokerLocator [servlet://uninventory.bc.jonqe.lab.eng.bos.redhat.com:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]] to [InvokerLocator [servlet://uninventory.bc.jonqe.lab.eng.bos.redhat.com:7080/jboss-remoting-servlet-invoker/ServerInvokerServlet]]
>2014-07-08 16:12:44,227 WARN  [ClientCommandSenderTask Timer Thread #21] (org.rhq.enterprise.agent.AgentMain)- {AgentMain.failover-failed}Failed to failover to another server. Cause: org.rhq.enterprise.communications.util.NotProcessedException
>2014-07-08 16:12:44,230 WARN  [ClientCommandSenderTask Timer Thread #21] (org.rhq.enterprise.agent.FailoverFailureCallback)- {AgentMain.too-many-failover-attempts}Too many failover attempts have been made [1]. Exception that triggered the failover: [org.rhq.enterprise.communications.util.NotProcessedException]
>2014-07-08 16:12:44,230 ERROR [ClientCommandSenderTask Timer Thread #21] (enterprise.communications.command.client.JBossRemotingRemoteCommunicator)- {JBossRemotingRemoteCommunicator.init-callback-failed}The initialize callback has failed. It will be tried again. Cause: org.rhq.enterprise.communications.util.NotProcessedException:null. Cause: org.rhq.enterprise.communications.util.NotProcessedException
>2014-07-08 16:12:44,232 WARN  [InventoryManager.availability-1] (rhq.core.pc.inventory.InventoryManager)- Could not transmit availability report to server
>org.rhq.enterprise.communications.util.NotProcessedException
>	at org.rhq.enterprise.communications.GlobalSuspendCommandListener.receivedCommand(GlobalSuspendCommandListener.java:55)
>	at org.rhq.enterprise.communications.command.server.CommandProcessor.notifyListenersOfReceivedCommand(CommandProcessor.java:402)
>	at org.rhq.enterprise.communications.command.server.CommandProcessor.handleIncomingInvocationRequest(CommandProcessor.java:242)
>	at org.rhq.enterprise.communications.command.server.CommandProcessor.invoke(CommandProcessor.java:184)
>	at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:967)
>	at org.jboss.remoting.transport.servlet.ServletServerInvoker.processRequest(ServletServerInvoker.java:416)
>	at sun.reflect.GeneratedMethodAccessor143.invoke(Unknown Source)
>	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>	at java.lang.reflect.Method.invoke(Method.java:606)
>	at sun.reflect.misc.Trampoline.invoke(MethodUtil.java:75)
>	at sun.reflect.GeneratedMethodAccessor124.invoke(Unknown Source)
>	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>	at java.lang.reflect.Method.invoke(Method.java:606)
>	at sun.reflect.misc.MethodUtil.invoke(MethodUtil.java:279)
>	at com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:112)
>	at com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:46)
>	at com.sun.jmx.mbeanserver.MBeanIntrospector.invokeM(MBeanIntrospector.java:237)
>	at com.sun.jmx.mbeanserver.PerInterface.invoke(PerInterface.java:138)
>	at com.sun.jmx.mbeanserver.MBeanSupport.invoke(MBeanSupport.java:252)
>	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.invoke(DefaultMBeanServerInterceptor.java:819)
>	at com.sun.jmx.mbeanserver.JmxMBeanServer.invoke(JmxMBeanServer.java:801)
>	at org.jboss.as.jmx.PluggableMBeanServerImpl$TcclMBeanServer.invoke(PluggableMBeanServerImpl.java:1453)
>	at org.jboss.as.jmx.PluggableMBeanServerImpl.invoke(PluggableMBeanServerImpl.java:731)
>	at javax.management.MBeanServerInvocationHandler.invoke(MBeanServerInvocationHandler.java:292)
>	at com.sun.proxy.$Proxy156.processRequest(Unknown Source)
>	at org.jboss.remoting.transport.servlet.web.ServerInvokerServlet.processRequest(ServerInvokerServlet.java:404)
>	at org.jboss.remoting.transport.servlet.web.ServerInvokerServlet.processRequest(ServerInvokerServlet.java:142)
>	at org.rhq.enterprise.communications.servlet.ServerInvokerServlet.processRequest(ServerInvokerServlet.java:70)
>	at org.jboss.remoting.transport.servlet.web.ServerInvokerServlet.doPost(ServerInvokerServlet.java:171)
>	at javax.servlet.http.HttpServlet.service(HttpServlet.java:754)
>	at javax.servlet.http.HttpServlet.service(HttpServlet.java:847)
>	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:295)
>	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:214)
>	at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:231)
>	at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:149)
>	at org.jboss.as.jpa.interceptor.WebNonTxEmCloserValve.invoke(WebNonTxEmCloserValve.java:50)
>	at org.jboss.as.jpa.interceptor.WebNonTxEmCloserValve.invoke(WebNonTxEmCloserValve.java:50)
>	at org.jboss.as.web.security.SecurityContextAssociationValve.invoke(SecurityContextAssociationValve.java:169)
>	at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:145)
>	at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:97)
>	at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:102)
>	at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:340)
>	at org.apache.coyote.http11.Http11NioProcessor.process(Http11NioProcessor.java:353)
>	at org.apache.coyote.http11.Http11NioProtocol$Http11ConnectionHandler.process(Http11NioProtocol.java:911)
>	at org.apache.tomcat.util.net.NioEndpoint$ChannelProcessor.run(NioEndpoint.java:920)
>	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
>	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
>	at java.lang.Thread.run(Thread.java:744)
>

Actions: View

Attachments on bug 1117505: 916549