Attachment 1129191 Details for Bug 1310575 – Engine Log

Engine Log

engine.txt (text/plain), 59.46 KB, created by erick on 2016-02-22 09:51:08 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: erick

Created: 2016-02-22 09:51:08 UTC

Size: 59.46 KB

patch

obsolete

>2016-02-22 04:01:15,055 INFO  [org.ovirt.engine.core.vdsbroker.gluster.GlusterVolumesListVDSCommand] (DefaultQuartzScheduler_Worker-44) [] FINISH, GlusterVolumesListVDSCommand, return: {}, log id: 1077fe01
>2016-02-22 04:01:17,049 INFO  [org.ovirt.engine.core.bll.MaintenanceNumberOfVdssCommand] (org.ovirt.thread.pool-8-thread-43) [406a784c] Running command: MaintenanceNumberOfVdssCommand internal: false. Entities affected :  ID: 50ad0574-f9f0-473b-8991-9a60f6a5265e Type: VDSAction group MANIPULATE_HOST with role type ADMIN
>2016-02-22 04:01:17,119 INFO  [org.ovirt.engine.core.bll.MaintenanceVdsCommand] (org.ovirt.thread.pool-8-thread-43) [406a784c] Running command: MaintenanceVdsCommand internal: true. Entities affected :  ID: 50ad0574-f9f0-473b-8991-9a60f6a5265e Type: VDS
>2016-02-22 04:01:17,127 INFO  [org.ovirt.engine.core.vdsbroker.SetVdsStatusVDSCommand] (org.ovirt.thread.pool-8-thread-43) [406a784c] START, SetVdsStatusVDSCommand(HostName = S1US, SetVdsStatusVDSCommandParameters:{runAsync='true', hostId='50ad0574-f9f0-473b-8991-9a60f6a5265e', status='Maintenance', nonOperationalReason='NONE', stopSpmFailureLogged='false', maintenanceReason='null'}), log id: 42a4c856
>2016-02-22 04:01:18,547 ERROR [org.ovirt.engine.core.vdsbroker.vdsbroker.GetCapabilitiesVDSCommand] (DefaultQuartzScheduler_Worker-19) [] Command 'GetCapabilitiesVDSCommand(HostName = S1US, VdsIdAndVdsVDSCommandParametersBase:{runAsync='true', hostId='50ad0574-f9f0-473b-8991-9a60f6a5265e', vds='Host[S1US,50ad0574-f9f0-473b-8991-9a60f6a5265e]'})' execution failed: java.net.SocketTimeoutException: connect timed out
>2016-02-22 04:01:18,547 ERROR [org.ovirt.engine.core.vdsbroker.HostMonitoring] (DefaultQuartzScheduler_Worker-19) [] Failure to refresh Vds runtime info: java.net.SocketTimeoutException: connect timed out
>2016-02-22 04:01:18,547 ERROR [org.ovirt.engine.core.vdsbroker.HostMonitoring] (DefaultQuartzScheduler_Worker-19) [] Exception: org.ovirt.engine.core.vdsbroker.vdsbroker.VDSNetworkException: java.net.SocketTimeoutException: connect timed out
>	at org.ovirt.engine.core.vdsbroker.vdsbroker.VdsBrokerCommand.createNetworkException(VdsBrokerCommand.java:157) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.vdsbroker.VdsBrokerCommand.executeVDSCommand(VdsBrokerCommand.java:120) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.VDSCommandBase.executeCommand(VDSCommandBase.java:65) [vdsbroker.jar:]
>	at org.ovirt.engine.core.dal.VdcCommandBase.execute(VdcCommandBase.java:33) [dal.jar:]
>	at org.ovirt.engine.core.vdsbroker.ResourceManager.runVdsCommand(ResourceManager.java:467) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.VdsManager.refreshCapabilities(VdsManager.java:647) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.HostMonitoring.refreshVdsRunTimeInfo(HostMonitoring.java:119) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.HostMonitoring.refresh(HostMonitoring.java:84) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.VdsManager.onTimer(VdsManager.java:227) [vdsbroker.jar:]
>	at sun.reflect.GeneratedMethodAccessor126.invoke(Unknown Source) [:1.8.0_71]
>	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [rt.jar:1.8.0_71]
>	at java.lang.reflect.Method.invoke(Method.java:497) [rt.jar:1.8.0_71]
>	at org.ovirt.engine.core.utils.timer.JobWrapper.invokeMethod(JobWrapper.java:81) [scheduler.jar:]
>	at org.ovirt.engine.core.utils.timer.JobWrapper.execute(JobWrapper.java:52) [scheduler.jar:]
>	at org.quartz.core.JobRunShell.run(JobRunShell.java:213) [quartz.jar:]
>	at org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:557) [quartz.jar:]
>Caused by: java.net.SocketTimeoutException: connect timed out
>	at java.net.PlainSocketImpl.socketConnect(Native Method) [rt.jar:1.8.0_71]
>	at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) [rt.jar:1.8.0_71]
>	at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) [rt.jar:1.8.0_71]
>	at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) [rt.jar:1.8.0_71]
>	at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) [rt.jar:1.8.0_71]
>	at java.net.Socket.connect(Socket.java:589) [rt.jar:1.8.0_71]
>	at sun.security.ssl.SSLSocketImpl.connect(SSLSocketImpl.java:668) [jsse.jar:1.8.0_71]
>	at org.ovirt.engine.core.utils.ssl.AuthSSLProtocolSocketFactory.createSocket(AuthSSLProtocolSocketFactory.java:293) [utils.jar:]
>	at org.apache.commons.httpclient.HttpConnection.open(HttpConnection.java:707) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.MultiThreadedHttpConnectionManager$HttpConnectionAdapter.open(MultiThreadedHttpConnectionManager.java:1361) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.HttpMethodDirector.executeWithRetry(HttpMethodDirector.java:387) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.HttpMethodDirector.executeMethod(HttpMethodDirector.java:171) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.HttpClient.executeMethod(HttpClient.java:397) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.HttpClient.executeMethod(HttpClient.java:323) [commons-httpclient.jar:]
>	at org.apache.xmlrpc.client.XmlRpcCommonsTransport.writeRequest(XmlRpcCommonsTransport.java:227) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcStreamTransport.sendRequest(XmlRpcStreamTransport.java:151) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcHttpTransport.sendRequest(XmlRpcHttpTransport.java:143) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcClientWorker.execute(XmlRpcClientWorker.java:56) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcClient.execute(XmlRpcClient.java:167) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcClient.execute(XmlRpcClient.java:137) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcClient.execute(XmlRpcClient.java:126) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.util.ClientFactory$1.invoke(ClientFactory.java:140) [xmlrpc-client.jar:3.1.3]
>	at com.sun.proxy.$Proxy167.getVdsCapabilities(Unknown Source)
>	at sun.reflect.GeneratedMethodAccessor344.invoke(Unknown Source) [:1.8.0_71]
>	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [rt.jar:1.8.0_71]
>	at java.lang.reflect.Method.invoke(Method.java:497) [rt.jar:1.8.0_71]
>	at org.ovirt.engine.core.vdsbroker.xmlrpc.XmlRpcUtils$AsyncProxy$InternalCallable.call(XmlRpcUtils.java:263) [vdsbroker.jar:]
>	at java.util.concurrent.FutureTask.run(FutureTask.java:266) [rt.jar:1.8.0_71]
>	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [rt.jar:1.8.0_71]
>	at java.util.concurrent.FutureTask.run(FutureTask.java:266) [rt.jar:1.8.0_71]
>	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [rt.jar:1.8.0_71]
>	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [rt.jar:1.8.0_71]
>	at java.lang.Thread.run(Thread.java:745) [rt.jar:1.8.0_71]
>
>2016-02-22 04:01:18,568 INFO  [org.ovirt.engine.core.vdsbroker.SetVdsStatusVDSCommand] (org.ovirt.thread.pool-8-thread-43) [406a784c] FINISH, SetVdsStatusVDSCommand, log id: 42a4c856
>2016-02-22 04:01:18,574 INFO  [org.ovirt.engine.core.dal.dbbroker.auditloghandling.AuditLogDirector] (org.ovirt.thread.pool-8-thread-43) [406a784c] Correlation ID: 406a784c, Job ID: 8d773f0a-9893-42d2-a2d2-af9ea065421b, Call Stack: null, Custom Event ID: -1, Message: Host S1US was switched to Maintenance mode by admin@internal (Reason:  No reason was returned for this operation failure. See logs for further details.).
>2016-02-22 04:01:18,790 ERROR [org.ovirt.engine.core.vdsbroker.vdsbroker.ListVDSCommand] (DefaultQuartzScheduler_Worker-46) [d87c278] Command 'ListVDSCommand(HostName = S1US, VdsIdAndVdsVDSCommandParametersBase:{runAsync='true', hostId='50ad0574-f9f0-473b-8991-9a60f6a5265e', vds='Host[S1US,50ad0574-f9f0-473b-8991-9a60f6a5265e]'})' execution failed: java.net.SocketTimeoutException: connect timed out
>2016-02-22 04:01:18,790 INFO  [org.ovirt.engine.core.vdsbroker.PollVmStatsRefresher] (DefaultQuartzScheduler_Worker-46) [d87c278] Failed to fetch vms info for host 'S1US' - skipping VMs monitoring.
>2016-02-22 04:01:20,068 INFO  [org.ovirt.engine.core.vdsbroker.gluster.GlusterVolumesListVDSCommand] (DefaultQuartzScheduler_Worker-62) [4a444ed8] START, GlusterVolumesListVDSCommand(HostName = S1CA, GlusterVolumesListVDSParameters:{runAsync='true', hostId='37775af7-7b73-4a7c-8a78-a2507c951361'}), log id: 817239
>2016-02-22 04:01:21,222 INFO  [org.ovirt.engine.core.vdsbroker.gluster.GlusterVolumesListVDSCommand] (DefaultQuartzScheduler_Worker-62) [4a444ed8] FINISH, GlusterVolumesListVDSCommand, return: {}, log id: 817239
>2016-02-22 04:01:26,236 INFO  [org.ovirt.engine.core.vdsbroker.gluster.GlusterVolumesListVDSCommand] (DefaultQuartzScheduler_Worker-1) [6c672839] START, GlusterVolumesListVDSCommand(HostName = S1CA, GlusterVolumesListVDSParameters:{runAsync='true', hostId='37775af7-7b73-4a7c-8a78-a2507c951361'}), log id: 19c06360
>2016-02-22 04:01:27,384 INFO  [org.ovirt.engine.core.vdsbroker.gluster.GlusterVolumesListVDSCommand] (DefaultQuartzScheduler_Worker-1) [6c672839] FINISH, GlusterVolumesListVDSCommand, return: {}, log id: 19c06360
>2016-02-22 04:01:29,326 INFO  [org.ovirt.engine.core.bll.ActivateVdsCommand] (default task-16) [63d9367c] Lock Acquired to object 'EngineLock:{exclusiveLocks='[50ad0574-f9f0-473b-8991-9a60f6a5265e=<VDS, ACTION_TYPE_FAILED_OBJECT_LOCKED>]', sharedLocks='null'}'
>2016-02-22 04:01:29,358 INFO  [org.ovirt.engine.core.bll.ActivateVdsCommand] (org.ovirt.thread.pool-8-thread-38) [63d9367c] Running command: ActivateVdsCommand internal: false. Entities affected :  ID: 50ad0574-f9f0-473b-8991-9a60f6a5265e Type: VDSAction group MANIPULATE_HOST with role type ADMIN
>2016-02-22 04:01:29,360 INFO  [org.ovirt.engine.core.bll.ActivateVdsCommand] (org.ovirt.thread.pool-8-thread-38) [63d9367c] Before acquiring lock in order to prevent monitoring for host 'S1US' from data-center 'USA'
>2016-02-22 04:01:29,360 INFO  [org.ovirt.engine.core.bll.ActivateVdsCommand] (org.ovirt.thread.pool-8-thread-38) [63d9367c] Lock acquired, from now a monitoring of host will be skipped for host 'S1US' from data-center 'USA'
>2016-02-22 04:01:29,379 INFO  [org.ovirt.engine.core.vdsbroker.SetVdsStatusVDSCommand] (org.ovirt.thread.pool-8-thread-38) [63d9367c] START, SetVdsStatusVDSCommand(HostName = S1US, SetVdsStatusVDSCommandParameters:{runAsync='true', hostId='50ad0574-f9f0-473b-8991-9a60f6a5265e', status='Unassigned', nonOperationalReason='NONE', stopSpmFailureLogged='false', maintenanceReason='null'}), log id: 71613fa9
>2016-02-22 04:01:29,386 INFO  [org.ovirt.engine.core.vdsbroker.SetVdsStatusVDSCommand] (org.ovirt.thread.pool-8-thread-38) [63d9367c] FINISH, SetVdsStatusVDSCommand, log id: 71613fa9
>2016-02-22 04:01:29,390 INFO  [org.ovirt.engine.core.vdsbroker.gluster.ManageGlusterServiceVDSCommand] (org.ovirt.thread.pool-8-thread-38) [63d9367c] START, ManageGlusterServiceVDSCommand(HostName = S1US, GlusterServiceVDSParameters:{runAsync='true', hostId='50ad0574-f9f0-473b-8991-9a60f6a5265e'}), log id: 79060dd5
>2016-02-22 04:01:31,395 ERROR [org.ovirt.engine.core.vdsbroker.gluster.ManageGlusterServiceVDSCommand] (org.ovirt.thread.pool-8-thread-38) [63d9367c] Command 'ManageGlusterServiceVDSCommand(HostName = S1US, GlusterServiceVDSParameters:{runAsync='true', hostId='50ad0574-f9f0-473b-8991-9a60f6a5265e'})' execution failed: java.net.SocketTimeoutException: connect timed out
>2016-02-22 04:01:31,395 INFO  [org.ovirt.engine.core.vdsbroker.gluster.ManageGlusterServiceVDSCommand] (org.ovirt.thread.pool-8-thread-38) [63d9367c] FINISH, ManageGlusterServiceVDSCommand, log id: 79060dd5
>2016-02-22 04:01:31,396 ERROR [org.ovirt.engine.core.bll.ActivateVdsCommand] (org.ovirt.thread.pool-8-thread-38) [63d9367c] Command 'org.ovirt.engine.core.bll.ActivateVdsCommand' failed: EngineException: org.ovirt.engine.core.vdsbroker.vdsbroker.VDSNetworkException: java.net.SocketTimeoutException: connect timed out (Failed with error VDS_NETWORK_ERROR and code 5022)
>2016-02-22 04:01:31,419 ERROR [org.ovirt.engine.core.dal.dbbroker.auditloghandling.AuditLogDirector] (org.ovirt.thread.pool-8-thread-38) [63d9367c] Correlation ID: 63d9367c, Job ID: 1a378127-358d-43f9-b668-310a4e4a9298, Call Stack: null, Custom Event ID: -1, Message: Failed to activate Host S1US.(User: admin@internal).
>2016-02-22 04:01:31,440 INFO  [org.ovirt.engine.core.bll.ActivateVdsCommand] (org.ovirt.thread.pool-8-thread-38) [63d9367c] Lock freed to object 'EngineLock:{exclusiveLocks='[50ad0574-f9f0-473b-8991-9a60f6a5265e=<VDS, ACTION_TYPE_FAILED_OBJECT_LOCKED>]', sharedLocks='null'}'
>2016-02-22 04:01:31,481 INFO  [org.ovirt.engine.core.vdsbroker.VdsManager] (org.ovirt.thread.pool-8-thread-23) [63d9367c] Server failed to respond, vds_id='50ad0574-f9f0-473b-8991-9a60f6a5265e', vds_name='S1US', vm_count=0, spm_status='None', non-responsive_timeout (seconds)=60, error: java.net.SocketTimeoutException: connect timed out
>2016-02-22 04:01:31,491 ERROR [org.ovirt.engine.core.dal.dbbroker.auditloghandling.AuditLogDirector] (org.ovirt.thread.pool-8-thread-23) [63d9367c] Correlation ID: null, Call Stack: org.ovirt.engine.core.vdsbroker.vdsbroker.VDSNetworkException: java.net.SocketTimeoutException: connect timed out
>	at org.ovirt.engine.core.vdsbroker.vdsbroker.VdsBrokerCommand.createNetworkException(VdsBrokerCommand.java:157)
>	at org.ovirt.engine.core.vdsbroker.vdsbroker.VdsBrokerCommand.executeVDSCommand(VdsBrokerCommand.java:120)
>	at org.ovirt.engine.core.vdsbroker.VDSCommandBase.executeCommand(VDSCommandBase.java:65)
>	at org.ovirt.engine.core.dal.VdcCommandBase.execute(VdcCommandBase.java:33)
>	at org.ovirt.engine.core.vdsbroker.ResourceManager.runVdsCommand(ResourceManager.java:467)
>	at org.ovirt.engine.core.bll.VDSBrokerFrontendImpl.RunVdsCommand(VDSBrokerFrontendImpl.java:33)
>	at org.ovirt.engine.core.bll.CommandBase.runVdsCommand(CommandBase.java:2094)
>	at org.ovirt.engine.core.bll.ActivateVdsCommand.executeCommand(ActivateVdsCommand.java:93)
>	at org.ovirt.engine.core.bll.CommandBase.executeWithoutTransaction(CommandBase.java:1215)
>	at org.ovirt.engine.core.bll.CommandBase.executeActionInTransactionScope(CommandBase.java:1359)
>	at org.ovirt.engine.core.bll.CommandBase.runInTransaction(CommandBase.java:1982)
>	at org.ovirt.engine.core.utils.transaction.TransactionSupport.executeInSuppressed(TransactionSupport.java:174)
>	at org.ovirt.engine.core.utils.transaction.TransactionSupport.executeInScope(TransactionSupport.java:116)
>	at org.ovirt.engine.core.bll.CommandBase.execute(CommandBase.java:1396)
>	at org.ovirt.engine.core.bll.CommandBase.executeAction(CommandBase.java:378)
>	at org.ovirt.engine.core.bll.MultipleActionsRunner.executeValidatedCommand(MultipleActionsRunner.java:202)
>	at org.ovirt.engine.core.bll.MultipleActionsRunner.runCommands(MultipleActionsRunner.java:170)
>	at org.ovirt.engine.core.bll.MultipleActionsRunner$2.run(MultipleActionsRunner.java:179)
>	at org.ovirt.engine.core.utils.threadpool.ThreadPoolUtil$InternalWrapperRunnable.run(ThreadPoolUtil.java:92)
>	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
>	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
>	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>	at java.lang.Thread.run(Thread.java:745)
>Caused by: java.net.SocketTimeoutException: connect timed out
>	at java.net.PlainSocketImpl.socketConnect(Native Method)
>	at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
>	at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
>	at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
>	at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
>	at java.net.Socket.connect(Socket.java:589)
>	at sun.security.ssl.SSLSocketImpl.connect(SSLSocketImpl.java:668)
>	at org.ovirt.engine.core.utils.ssl.AuthSSLProtocolSocketFactory.createSocket(AuthSSLProtocolSocketFactory.java:293)
>	at org.apache.commons.httpclient.HttpConnection.open(HttpConnection.java:707)
>	at org.apache.commons.httpclient.MultiThreadedHttpConnectionManager$HttpConnectionAdapter.open(MultiThreadedHttpConnectionManager.java:1361)
>	at org.apache.commons.httpclient.HttpMethodDirector.executeWithRetry(HttpMethodDirector.java:387)
>	at org.apache.commons.httpclient.HttpMethodDirector.executeMethod(HttpMethodDirector.java:171)
>	at org.apache.commons.httpclient.HttpClient.executeMethod(HttpClient.java:397)
>	at org.apache.commons.httpclient.HttpClient.executeMethod(HttpClient.java:323)
>	at org.apache.xmlrpc.client.XmlRpcCommonsTransport.writeRequest(XmlRpcCommonsTransport.java:227)
>	at org.apache.xmlrpc.client.XmlRpcStreamTransport.sendRequest(XmlRpcStreamTransport.java:151)
>	at org.apache.xmlrpc.client.XmlRpcHttpTransport.sendRequest(XmlRpcHttpTransport.java:143)
>	at org.apache.xmlrpc.client.XmlRpcClientWorker.execute(XmlRpcClientWorker.java:56)
>	at org.apache.xmlrpc.client.XmlRpcClient.execute(XmlRpcClient.java:167)
>	at org.apache.xmlrpc.client.XmlRpcClient.execute(XmlRpcClient.java:137)
>	at org.apache.xmlrpc.client.XmlRpcClient.execute(XmlRpcClient.java:126)
>	at org.apache.xmlrpc.client.util.ClientFactory$1.invoke(ClientFactory.java:140)
>	at com.sun.proxy.$Proxy167.glusterServicesAction(Unknown Source)
>	at sun.reflect.GeneratedMethodAccessor661.invoke(Unknown Source)
>	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>	at java.lang.reflect.Method.invoke(Method.java:497)
>	at org.ovirt.engine.core.vdsbroker.xmlrpc.XmlRpcUtils$AsyncProxy$InternalCallable.call(XmlRpcUtils.java:263)
>	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
>	... 5 more
>, Custom Event ID: -1, Message: Host S1US is non responsive.
>2016-02-22 04:01:31,501 INFO  [org.ovirt.engine.core.bll.VdsEventListener] (org.ovirt.thread.pool-8-thread-28) [63d9367c] ResourceManager::vdsNotResponding entered for Host '50ad0574-f9f0-473b-8991-9a60f6a5265e', '192.75.197.3'
>2016-02-22 04:01:31,520 INFO  [org.ovirt.engine.core.bll.pm.VdsNotRespondingTreatmentCommand] (org.ovirt.thread.pool-8-thread-28) [5d2a56b2] Failed to Acquire Lock to object 'EngineLock:{exclusiveLocks='[50ad0574-f9f0-473b-8991-9a60f6a5265e=<VDS_FENCE, POWER_MANAGEMENT_ACTION_ON_ENTITY_ALREADY_IN_PROGRESS>]', sharedLocks='null'}'
>2016-02-22 04:01:31,520 WARN  [org.ovirt.engine.core.bll.pm.VdsNotRespondingTreatmentCommand] (org.ovirt.thread.pool-8-thread-28) [5d2a56b2] CanDoAction of action 'VdsNotRespondingTreatment' failed for user SYSTEM. Reasons: VAR__ACTION__RESTART,POWER_MANAGEMENT_ACTION_ON_ENTITY_ALREADY_IN_PROGRESS
>2016-02-22 04:01:32,397 INFO  [org.ovirt.engine.core.vdsbroker.gluster.GlusterVolumesListVDSCommand] (DefaultQuartzScheduler_Worker-68) [1f2a1e7d] START, GlusterVolumesListVDSCommand(HostName = S1CA, GlusterVolumesListVDSParameters:{runAsync='true', hostId='37775af7-7b73-4a7c-8a78-a2507c951361'}), log id: 64bfed12
>2016-02-22 04:01:33,249 INFO  [org.ovirt.engine.core.vdsbroker.gluster.GlusterVolumesListVDSCommand] (DefaultQuartzScheduler_Worker-68) [1f2a1e7d] FINISH, GlusterVolumesListVDSCommand, return: {}, log id: 64bfed12
>2016-02-22 04:01:34,667 INFO  [org.ovirt.engine.core.bll.tasks.AsyncTaskManager] (DefaultQuartzScheduler_Worker-79) [7366cc47] Setting new tasks map. The map contains now 0 tasks
>2016-02-22 04:01:34,667 INFO  [org.ovirt.engine.core.bll.tasks.AsyncTaskManager] (DefaultQuartzScheduler_Worker-79) [7366cc47] Cleared all tasks of pool 'acacbc53-79ef-42d6-8fbd-b01da345befc'.
>2016-02-22 04:01:35,570 ERROR [org.ovirt.engine.core.vdsbroker.vdsbroker.GetCapabilitiesVDSCommand] (DefaultQuartzScheduler_Worker-33) [] Command 'GetCapabilitiesVDSCommand(HostName = S1US, VdsIdAndVdsVDSCommandParametersBase:{runAsync='true', hostId='50ad0574-f9f0-473b-8991-9a60f6a5265e', vds='Host[S1US,50ad0574-f9f0-473b-8991-9a60f6a5265e]'})' execution failed: java.net.SocketTimeoutException: connect timed out
>2016-02-22 04:01:35,571 ERROR [org.ovirt.engine.core.vdsbroker.HostMonitoring] (DefaultQuartzScheduler_Worker-33) [] Failure to refresh Vds runtime info: java.net.SocketTimeoutException: connect timed out
>2016-02-22 04:01:35,571 ERROR [org.ovirt.engine.core.vdsbroker.HostMonitoring] (DefaultQuartzScheduler_Worker-33) [] Exception: org.ovirt.engine.core.vdsbroker.vdsbroker.VDSNetworkException: java.net.SocketTimeoutException: connect timed out
>	at org.ovirt.engine.core.vdsbroker.vdsbroker.VdsBrokerCommand.createNetworkException(VdsBrokerCommand.java:157) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.vdsbroker.VdsBrokerCommand.executeVDSCommand(VdsBrokerCommand.java:120) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.VDSCommandBase.executeCommand(VDSCommandBase.java:65) [vdsbroker.jar:]
>	at org.ovirt.engine.core.dal.VdcCommandBase.execute(VdcCommandBase.java:33) [dal.jar:]
>	at org.ovirt.engine.core.vdsbroker.ResourceManager.runVdsCommand(ResourceManager.java:467) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.VdsManager.refreshCapabilities(VdsManager.java:647) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.HostMonitoring.refreshVdsRunTimeInfo(HostMonitoring.java:119) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.HostMonitoring.refresh(HostMonitoring.java:84) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.VdsManager.onTimer(VdsManager.java:227) [vdsbroker.jar:]
>	at sun.reflect.GeneratedMethodAccessor126.invoke(Unknown Source) [:1.8.0_71]
>	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [rt.jar:1.8.0_71]
>	at java.lang.reflect.Method.invoke(Method.java:497) [rt.jar:1.8.0_71]
>	at org.ovirt.engine.core.utils.timer.JobWrapper.invokeMethod(JobWrapper.java:81) [scheduler.jar:]
>	at org.ovirt.engine.core.utils.timer.JobWrapper.execute(JobWrapper.java:52) [scheduler.jar:]
>	at org.quartz.core.JobRunShell.run(JobRunShell.java:213) [quartz.jar:]
>	at org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:557) [quartz.jar:]
>Caused by: java.net.SocketTimeoutException: connect timed out
>	at java.net.PlainSocketImpl.socketConnect(Native Method) [rt.jar:1.8.0_71]
>	at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) [rt.jar:1.8.0_71]
>	at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) [rt.jar:1.8.0_71]
>	at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) [rt.jar:1.8.0_71]
>	at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) [rt.jar:1.8.0_71]
>	at java.net.Socket.connect(Socket.java:589) [rt.jar:1.8.0_71]
>	at sun.security.ssl.SSLSocketImpl.connect(SSLSocketImpl.java:668) [jsse.jar:1.8.0_71]
>	at org.ovirt.engine.core.utils.ssl.AuthSSLProtocolSocketFactory.createSocket(AuthSSLProtocolSocketFactory.java:293) [utils.jar:]
>	at org.apache.commons.httpclient.HttpConnection.open(HttpConnection.java:707) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.MultiThreadedHttpConnectionManager$HttpConnectionAdapter.open(MultiThreadedHttpConnectionManager.java:1361) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.HttpMethodDirector.executeWithRetry(HttpMethodDirector.java:387) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.HttpMethodDirector.executeMethod(HttpMethodDirector.java:171) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.HttpClient.executeMethod(HttpClient.java:397) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.HttpClient.executeMethod(HttpClient.java:323) [commons-httpclient.jar:]
>	at org.apache.xmlrpc.client.XmlRpcCommonsTransport.writeRequest(XmlRpcCommonsTransport.java:227) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcStreamTransport.sendRequest(XmlRpcStreamTransport.java:151) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcHttpTransport.sendRequest(XmlRpcHttpTransport.java:143) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcClientWorker.execute(XmlRpcClientWorker.java:56) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcClient.execute(XmlRpcClient.java:167) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcClient.execute(XmlRpcClient.java:137) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcClient.execute(XmlRpcClient.java:126) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.util.ClientFactory$1.invoke(ClientFactory.java:140) [xmlrpc-client.jar:3.1.3]
>	at com.sun.proxy.$Proxy167.getVdsCapabilities(Unknown Source)
>	at sun.reflect.GeneratedMethodAccessor344.invoke(Unknown Source) [:1.8.0_71]
>	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [rt.jar:1.8.0_71]
>	at java.lang.reflect.Method.invoke(Method.java:497) [rt.jar:1.8.0_71]
>	at org.ovirt.engine.core.vdsbroker.xmlrpc.XmlRpcUtils$AsyncProxy$InternalCallable.call(XmlRpcUtils.java:263) [vdsbroker.jar:]
>	at java.util.concurrent.FutureTask.run(FutureTask.java:266) [rt.jar:1.8.0_71]
>	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [rt.jar:1.8.0_71]
>	at java.util.concurrent.FutureTask.run(FutureTask.java:266) [rt.jar:1.8.0_71]
>	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [rt.jar:1.8.0_71]
>	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [rt.jar:1.8.0_71]
>	at java.lang.Thread.run(Thread.java:745) [rt.jar:1.8.0_71]
>
>2016-02-22 04:01:35,798 ERROR [org.ovirt.engine.core.vdsbroker.vdsbroker.ListVDSCommand] (DefaultQuartzScheduler_Worker-53) [5d90a9ec] Command 'ListVDSCommand(HostName = S1US, VdsIdAndVdsVDSCommandParametersBase:{runAsync='true', hostId='50ad0574-f9f0-473b-8991-9a60f6a5265e', vds='Host[S1US,50ad0574-f9f0-473b-8991-9a60f6a5265e]'})' execution failed: java.net.SocketTimeoutException: connect timed out
>2016-02-22 04:01:35,798 INFO  [org.ovirt.engine.core.vdsbroker.PollVmStatsRefresher] (DefaultQuartzScheduler_Worker-53) [5d90a9ec] Failed to fetch vms info for host 'S1US' - skipping VMs monitoring.
>2016-02-22 04:01:38,261 INFO  [org.ovirt.engine.core.vdsbroker.gluster.GlusterVolumesListVDSCommand] (DefaultQuartzScheduler_Worker-42) [35b429a0] START, GlusterVolumesListVDSCommand(HostName = S1CA, GlusterVolumesListVDSParameters:{runAsync='true', hostId='37775af7-7b73-4a7c-8a78-a2507c951361'}), log id: 1c1ff87e
>2016-02-22 04:01:39,219 INFO  [org.ovirt.engine.core.vdsbroker.gluster.GlusterVolumesListVDSCommand] (DefaultQuartzScheduler_Worker-42) [35b429a0] FINISH, GlusterVolumesListVDSCommand, return: {}, log id: 1c1ff87e
>2016-02-22 04:01:40,582 ERROR [org.ovirt.engine.core.vdsbroker.vdsbroker.GetCapabilitiesVDSCommand] (DefaultQuartzScheduler_Worker-41) [] Command 'GetCapabilitiesVDSCommand(HostName = S1US, VdsIdAndVdsVDSCommandParametersBase:{runAsync='true', hostId='50ad0574-f9f0-473b-8991-9a60f6a5265e', vds='Host[S1US,50ad0574-f9f0-473b-8991-9a60f6a5265e]'})' execution failed: java.net.SocketTimeoutException: connect timed out
>2016-02-22 04:01:40,583 ERROR [org.ovirt.engine.core.vdsbroker.HostMonitoring] (DefaultQuartzScheduler_Worker-41) [] Failure to refresh Vds runtime info: java.net.SocketTimeoutException: connect timed out
>2016-02-22 04:01:40,583 ERROR [org.ovirt.engine.core.vdsbroker.HostMonitoring] (DefaultQuartzScheduler_Worker-41) [] Exception: org.ovirt.engine.core.vdsbroker.vdsbroker.VDSNetworkException: java.net.SocketTimeoutException: connect timed out
>	at org.ovirt.engine.core.vdsbroker.vdsbroker.VdsBrokerCommand.createNetworkException(VdsBrokerCommand.java:157) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.vdsbroker.VdsBrokerCommand.executeVDSCommand(VdsBrokerCommand.java:120) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.VDSCommandBase.executeCommand(VDSCommandBase.java:65) [vdsbroker.jar:]
>	at org.ovirt.engine.core.dal.VdcCommandBase.execute(VdcCommandBase.java:33) [dal.jar:]
>	at org.ovirt.engine.core.vdsbroker.ResourceManager.runVdsCommand(ResourceManager.java:467) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.VdsManager.refreshCapabilities(VdsManager.java:647) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.HostMonitoring.refreshVdsRunTimeInfo(HostMonitoring.java:119) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.HostMonitoring.refresh(HostMonitoring.java:84) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.VdsManager.onTimer(VdsManager.java:227) [vdsbroker.jar:]
>	at sun.reflect.GeneratedMethodAccessor126.invoke(Unknown Source) [:1.8.0_71]
>	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [rt.jar:1.8.0_71]
>	at java.lang.reflect.Method.invoke(Method.java:497) [rt.jar:1.8.0_71]
>	at org.ovirt.engine.core.utils.timer.JobWrapper.invokeMethod(JobWrapper.java:81) [scheduler.jar:]
>	at org.ovirt.engine.core.utils.timer.JobWrapper.execute(JobWrapper.java:52) [scheduler.jar:]
>	at org.quartz.core.JobRunShell.run(JobRunShell.java:213) [quartz.jar:]
>	at org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:557) [quartz.jar:]
>Caused by: java.net.SocketTimeoutException: connect timed out
>	at java.net.PlainSocketImpl.socketConnect(Native Method) [rt.jar:1.8.0_71]
>	at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) [rt.jar:1.8.0_71]
>	at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) [rt.jar:1.8.0_71]
>	at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) [rt.jar:1.8.0_71]
>	at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) [rt.jar:1.8.0_71]
>	at java.net.Socket.connect(Socket.java:589) [rt.jar:1.8.0_71]
>	at sun.security.ssl.SSLSocketImpl.connect(SSLSocketImpl.java:668) [jsse.jar:1.8.0_71]
>	at org.ovirt.engine.core.utils.ssl.AuthSSLProtocolSocketFactory.createSocket(AuthSSLProtocolSocketFactory.java:293) [utils.jar:]
>	at org.apache.commons.httpclient.HttpConnection.open(HttpConnection.java:707) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.MultiThreadedHttpConnectionManager$HttpConnectionAdapter.open(MultiThreadedHttpConnectionManager.java:1361) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.HttpMethodDirector.executeWithRetry(HttpMethodDirector.java:387) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.HttpMethodDirector.executeMethod(HttpMethodDirector.java:171) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.HttpClient.executeMethod(HttpClient.java:397) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.HttpClient.executeMethod(HttpClient.java:323) [commons-httpclient.jar:]
>	at org.apache.xmlrpc.client.XmlRpcCommonsTransport.writeRequest(XmlRpcCommonsTransport.java:227) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcStreamTransport.sendRequest(XmlRpcStreamTransport.java:151) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcHttpTransport.sendRequest(XmlRpcHttpTransport.java:143) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcClientWorker.execute(XmlRpcClientWorker.java:56) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcClient.execute(XmlRpcClient.java:167) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcClient.execute(XmlRpcClient.java:137) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcClient.execute(XmlRpcClient.java:126) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.util.ClientFactory$1.invoke(ClientFactory.java:140) [xmlrpc-client.jar:3.1.3]
>	at com.sun.proxy.$Proxy167.getVdsCapabilities(Unknown Source)
>	at sun.reflect.GeneratedMethodAccessor344.invoke(Unknown Source) [:1.8.0_71]
>	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [rt.jar:1.8.0_71]
>	at java.lang.reflect.Method.invoke(Method.java:497) [rt.jar:1.8.0_71]
>	at org.ovirt.engine.core.vdsbroker.xmlrpc.XmlRpcUtils$AsyncProxy$InternalCallable.call(XmlRpcUtils.java:263) [vdsbroker.jar:]
>	at java.util.concurrent.FutureTask.run(FutureTask.java:266) [rt.jar:1.8.0_71]
>	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [rt.jar:1.8.0_71]
>	at java.util.concurrent.FutureTask.run(FutureTask.java:266) [rt.jar:1.8.0_71]
>	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [rt.jar:1.8.0_71]
>	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [rt.jar:1.8.0_71]
>	at java.lang.Thread.run(Thread.java:745) [rt.jar:1.8.0_71]
>
>2016-02-22 04:01:40,805 ERROR [org.ovirt.engine.core.vdsbroker.vdsbroker.ListVDSCommand] (DefaultQuartzScheduler_Worker-49) [241c456] Command 'ListVDSCommand(HostName = S1US, VdsIdAndVdsVDSCommandParametersBase:{runAsync='true', hostId='50ad0574-f9f0-473b-8991-9a60f6a5265e', vds='Host[S1US,50ad0574-f9f0-473b-8991-9a60f6a5265e]'})' execution failed: java.net.SocketTimeoutException: connect timed out
>2016-02-22 04:01:40,806 INFO  [org.ovirt.engine.core.vdsbroker.PollVmStatsRefresher] (DefaultQuartzScheduler_Worker-49) [241c456] Failed to fetch vms info for host 'S1US' - skipping VMs monitoring.
>2016-02-22 04:01:44,231 INFO  [org.ovirt.engine.core.vdsbroker.gluster.GlusterVolumesListVDSCommand] (DefaultQuartzScheduler_Worker-58) [6ccdf0e0] START, GlusterVolumesListVDSCommand(HostName = S1CA, GlusterVolumesListVDSParameters:{runAsync='true', hostId='37775af7-7b73-4a7c-8a78-a2507c951361'}), log id: 3e570a93
>2016-02-22 04:01:45,394 INFO  [org.ovirt.engine.core.vdsbroker.gluster.GlusterVolumesListVDSCommand] (DefaultQuartzScheduler_Worker-58) [6ccdf0e0] FINISH, GlusterVolumesListVDSCommand, return: {}, log id: 3e570a93
>2016-02-22 04:01:45,593 ERROR [org.ovirt.engine.core.vdsbroker.vdsbroker.GetCapabilitiesVDSCommand] (DefaultQuartzScheduler_Worker-72) [] Command 'GetCapabilitiesVDSCommand(HostName = S1US, VdsIdAndVdsVDSCommandParametersBase:{runAsync='true', hostId='50ad0574-f9f0-473b-8991-9a60f6a5265e', vds='Host[S1US,50ad0574-f9f0-473b-8991-9a60f6a5265e]'})' execution failed: java.net.SocketTimeoutException: connect timed out
>2016-02-22 04:01:45,593 ERROR [org.ovirt.engine.core.vdsbroker.HostMonitoring] (DefaultQuartzScheduler_Worker-72) [] Failure to refresh Vds runtime info: java.net.SocketTimeoutException: connect timed out
>2016-02-22 04:01:45,593 ERROR [org.ovirt.engine.core.vdsbroker.HostMonitoring] (DefaultQuartzScheduler_Worker-72) [] Exception: org.ovirt.engine.core.vdsbroker.vdsbroker.VDSNetworkException: java.net.SocketTimeoutException: connect timed out
>	at org.ovirt.engine.core.vdsbroker.vdsbroker.VdsBrokerCommand.createNetworkException(VdsBrokerCommand.java:157) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.vdsbroker.VdsBrokerCommand.executeVDSCommand(VdsBrokerCommand.java:120) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.VDSCommandBase.executeCommand(VDSCommandBase.java:65) [vdsbroker.jar:]
>	at org.ovirt.engine.core.dal.VdcCommandBase.execute(VdcCommandBase.java:33) [dal.jar:]
>	at org.ovirt.engine.core.vdsbroker.ResourceManager.runVdsCommand(ResourceManager.java:467) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.VdsManager.refreshCapabilities(VdsManager.java:647) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.HostMonitoring.refreshVdsRunTimeInfo(HostMonitoring.java:119) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.HostMonitoring.refresh(HostMonitoring.java:84) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.VdsManager.onTimer(VdsManager.java:227) [vdsbroker.jar:]
>	at sun.reflect.GeneratedMethodAccessor126.invoke(Unknown Source) [:1.8.0_71]
>	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [rt.jar:1.8.0_71]
>	at java.lang.reflect.Method.invoke(Method.java:497) [rt.jar:1.8.0_71]
>	at org.ovirt.engine.core.utils.timer.JobWrapper.invokeMethod(JobWrapper.java:81) [scheduler.jar:]
>	at org.ovirt.engine.core.utils.timer.JobWrapper.execute(JobWrapper.java:52) [scheduler.jar:]
>	at org.quartz.core.JobRunShell.run(JobRunShell.java:213) [quartz.jar:]
>	at org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:557) [quartz.jar:]
>Caused by: java.net.SocketTimeoutException: connect timed out
>	at java.net.PlainSocketImpl.socketConnect(Native Method) [rt.jar:1.8.0_71]
>	at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) [rt.jar:1.8.0_71]
>	at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) [rt.jar:1.8.0_71]
>	at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) [rt.jar:1.8.0_71]
>	at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) [rt.jar:1.8.0_71]
>	at java.net.Socket.connect(Socket.java:589) [rt.jar:1.8.0_71]
>	at sun.security.ssl.SSLSocketImpl.connect(SSLSocketImpl.java:668) [jsse.jar:1.8.0_71]
>	at org.ovirt.engine.core.utils.ssl.AuthSSLProtocolSocketFactory.createSocket(AuthSSLProtocolSocketFactory.java:293) [utils.jar:]
>	at org.apache.commons.httpclient.HttpConnection.open(HttpConnection.java:707) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.MultiThreadedHttpConnectionManager$HttpConnectionAdapter.open(MultiThreadedHttpConnectionManager.java:1361) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.HttpMethodDirector.executeWithRetry(HttpMethodDirector.java:387) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.HttpMethodDirector.executeMethod(HttpMethodDirector.java:171) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.HttpClient.executeMethod(HttpClient.java:397) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.HttpClient.executeMethod(HttpClient.java:323) [commons-httpclient.jar:]
>	at org.apache.xmlrpc.client.XmlRpcCommonsTransport.writeRequest(XmlRpcCommonsTransport.java:227) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcStreamTransport.sendRequest(XmlRpcStreamTransport.java:151) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcHttpTransport.sendRequest(XmlRpcHttpTransport.java:143) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcClientWorker.execute(XmlRpcClientWorker.java:56) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcClient.execute(XmlRpcClient.java:167) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcClient.execute(XmlRpcClient.java:137) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcClient.execute(XmlRpcClient.java:126) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.util.ClientFactory$1.invoke(ClientFactory.java:140) [xmlrpc-client.jar:3.1.3]
>	at com.sun.proxy.$Proxy167.getVdsCapabilities(Unknown Source)
>	at sun.reflect.GeneratedMethodAccessor344.invoke(Unknown Source) [:1.8.0_71]
>	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [rt.jar:1.8.0_71]
>	at java.lang.reflect.Method.invoke(Method.java:497) [rt.jar:1.8.0_71]
>	at org.ovirt.engine.core.vdsbroker.xmlrpc.XmlRpcUtils$AsyncProxy$InternalCallable.call(XmlRpcUtils.java:263) [vdsbroker.jar:]
>	at java.util.concurrent.FutureTask.run(FutureTask.java:266) [rt.jar:1.8.0_71]
>	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [rt.jar:1.8.0_71]
>	at java.util.concurrent.FutureTask.run(FutureTask.java:266) [rt.jar:1.8.0_71]
>	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [rt.jar:1.8.0_71]
>	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [rt.jar:1.8.0_71]
>	at java.lang.Thread.run(Thread.java:745) [rt.jar:1.8.0_71]
>
>2016-02-22 04:01:45,813 ERROR [org.ovirt.engine.core.vdsbroker.vdsbroker.ListVDSCommand] (DefaultQuartzScheduler_Worker-89) [] Command 'ListVDSCommand(HostName = S1US, VdsIdAndVdsVDSCommandParametersBase:{runAsync='true', hostId='50ad0574-f9f0-473b-8991-9a60f6a5265e', vds='Host[S1US,50ad0574-f9f0-473b-8991-9a60f6a5265e]'})' execution failed: java.net.SocketTimeoutException: connect timed out
>2016-02-22 04:01:45,813 INFO  [org.ovirt.engine.core.vdsbroker.PollVmStatsRefresher] (DefaultQuartzScheduler_Worker-89) [] Failed to fetch vms info for host 'S1US' - skipping VMs monitoring.
>2016-02-22 04:01:50,409 INFO  [org.ovirt.engine.core.vdsbroker.gluster.GlusterVolumesListVDSCommand] (DefaultQuartzScheduler_Worker-9) [] START, GlusterVolumesListVDSCommand(HostName = S1CA, GlusterVolumesListVDSParameters:{runAsync='true', hostId='37775af7-7b73-4a7c-8a78-a2507c951361'}), log id: 7e77cf63
>2016-02-22 04:01:50,606 ERROR [org.ovirt.engine.core.vdsbroker.vdsbroker.GetCapabilitiesVDSCommand] (DefaultQuartzScheduler_Worker-38) [b08fab0] Command 'GetCapabilitiesVDSCommand(HostName = S1US, VdsIdAndVdsVDSCommandParametersBase:{runAsync='true', hostId='50ad0574-f9f0-473b-8991-9a60f6a5265e', vds='Host[S1US,50ad0574-f9f0-473b-8991-9a60f6a5265e]'})' execution failed: java.net.SocketTimeoutException: connect timed out
>2016-02-22 04:01:50,606 ERROR [org.ovirt.engine.core.vdsbroker.HostMonitoring] (DefaultQuartzScheduler_Worker-38) [b08fab0] Failure to refresh Vds runtime info: java.net.SocketTimeoutException: connect timed out
>2016-02-22 04:01:50,607 ERROR [org.ovirt.engine.core.vdsbroker.HostMonitoring] (DefaultQuartzScheduler_Worker-38) [b08fab0] Exception: org.ovirt.engine.core.vdsbroker.vdsbroker.VDSNetworkException: java.net.SocketTimeoutException: connect timed out
>	at org.ovirt.engine.core.vdsbroker.vdsbroker.VdsBrokerCommand.createNetworkException(VdsBrokerCommand.java:157) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.vdsbroker.VdsBrokerCommand.executeVDSCommand(VdsBrokerCommand.java:120) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.VDSCommandBase.executeCommand(VDSCommandBase.java:65) [vdsbroker.jar:]
>	at org.ovirt.engine.core.dal.VdcCommandBase.execute(VdcCommandBase.java:33) [dal.jar:]
>	at org.ovirt.engine.core.vdsbroker.ResourceManager.runVdsCommand(ResourceManager.java:467) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.VdsManager.refreshCapabilities(VdsManager.java:647) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.HostMonitoring.refreshVdsRunTimeInfo(HostMonitoring.java:119) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.HostMonitoring.refresh(HostMonitoring.java:84) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.VdsManager.onTimer(VdsManager.java:227) [vdsbroker.jar:]
>	at sun.reflect.GeneratedMethodAccessor126.invoke(Unknown Source) [:1.8.0_71]
>	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [rt.jar:1.8.0_71]
>	at java.lang.reflect.Method.invoke(Method.java:497) [rt.jar:1.8.0_71]
>	at org.ovirt.engine.core.utils.timer.JobWrapper.invokeMethod(JobWrapper.java:81) [scheduler.jar:]
>	at org.ovirt.engine.core.utils.timer.JobWrapper.execute(JobWrapper.java:52) [scheduler.jar:]
>	at org.quartz.core.JobRunShell.run(JobRunShell.java:213) [quartz.jar:]
>	at org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:557) [quartz.jar:]
>Caused by: java.net.SocketTimeoutException: connect timed out
>	at java.net.PlainSocketImpl.socketConnect(Native Method) [rt.jar:1.8.0_71]
>	at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) [rt.jar:1.8.0_71]
>	at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) [rt.jar:1.8.0_71]
>	at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) [rt.jar:1.8.0_71]
>	at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) [rt.jar:1.8.0_71]
>	at java.net.Socket.connect(Socket.java:589) [rt.jar:1.8.0_71]
>	at sun.security.ssl.SSLSocketImpl.connect(SSLSocketImpl.java:668) [jsse.jar:1.8.0_71]
>	at org.ovirt.engine.core.utils.ssl.AuthSSLProtocolSocketFactory.createSocket(AuthSSLProtocolSocketFactory.java:293) [utils.jar:]
>	at org.apache.commons.httpclient.HttpConnection.open(HttpConnection.java:707) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.MultiThreadedHttpConnectionManager$HttpConnectionAdapter.open(MultiThreadedHttpConnectionManager.java:1361) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.HttpMethodDirector.executeWithRetry(HttpMethodDirector.java:387) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.HttpMethodDirector.executeMethod(HttpMethodDirector.java:171) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.HttpClient.executeMethod(HttpClient.java:397) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.HttpClient.executeMethod(HttpClient.java:323) [commons-httpclient.jar:]
>	at org.apache.xmlrpc.client.XmlRpcCommonsTransport.writeRequest(XmlRpcCommonsTransport.java:227) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcStreamTransport.sendRequest(XmlRpcStreamTransport.java:151) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcHttpTransport.sendRequest(XmlRpcHttpTransport.java:143) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcClientWorker.execute(XmlRpcClientWorker.java:56) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcClient.execute(XmlRpcClient.java:167) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcClient.execute(XmlRpcClient.java:137) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcClient.execute(XmlRpcClient.java:126) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.util.ClientFactory$1.invoke(ClientFactory.java:140) [xmlrpc-client.jar:3.1.3]
>	at com.sun.proxy.$Proxy167.getVdsCapabilities(Unknown Source)
>	at sun.reflect.GeneratedMethodAccessor344.invoke(Unknown Source) [:1.8.0_71]
>	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [rt.jar:1.8.0_71]
>	at java.lang.reflect.Method.invoke(Method.java:497) [rt.jar:1.8.0_71]
>	at org.ovirt.engine.core.vdsbroker.xmlrpc.XmlRpcUtils$AsyncProxy$InternalCallable.call(XmlRpcUtils.java:263) [vdsbroker.jar:]
>	at java.util.concurrent.FutureTask.run(FutureTask.java:266) [rt.jar:1.8.0_71]
>	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [rt.jar:1.8.0_71]
>	at java.util.concurrent.FutureTask.run(FutureTask.java:266) [rt.jar:1.8.0_71]
>	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [rt.jar:1.8.0_71]
>	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [rt.jar:1.8.0_71]
>	at java.lang.Thread.run(Thread.java:745) [rt.jar:1.8.0_71]
>
>2016-02-22 04:01:50,821 ERROR [org.ovirt.engine.core.vdsbroker.vdsbroker.GetAllVmStatsVDSCommand] (DefaultQuartzScheduler_Worker-59) [6537c9f7] Command 'GetAllVmStatsVDSCommand(HostName = S1US, VdsIdAndVdsVDSCommandParametersBase:{runAsync='true', hostId='50ad0574-f9f0-473b-8991-9a60f6a5265e', vds='Host[S1US,50ad0574-f9f0-473b-8991-9a60f6a5265e]'})' execution failed: java.net.SocketTimeoutException: connect timed out
>2016-02-22 04:01:50,822 INFO  [org.ovirt.engine.core.vdsbroker.PollVmStatsRefresher] (DefaultQuartzScheduler_Worker-59) [6537c9f7] Failed to fetch vms info for host 'S1US' - skipping VMs monitoring.
>2016-02-22 04:01:51,572 INFO  [org.ovirt.engine.core.vdsbroker.gluster.GlusterVolumesListVDSCommand] (DefaultQuartzScheduler_Worker-9) [] FINISH, GlusterVolumesListVDSCommand, return: {}, log id: 7e77cf63
>2016-02-22 04:01:55,621 ERROR [org.ovirt.engine.core.vdsbroker.vdsbroker.GetCapabilitiesVDSCommand] (DefaultQuartzScheduler_Worker-61) [5e282aea] Command 'GetCapabilitiesVDSCommand(HostName = S1US, VdsIdAndVdsVDSCommandParametersBase:{runAsync='true', hostId='50ad0574-f9f0-473b-8991-9a60f6a5265e', vds='Host[S1US,50ad0574-f9f0-473b-8991-9a60f6a5265e]'})' execution failed: java.net.SocketTimeoutException: connect timed out
>2016-02-22 04:01:55,621 ERROR [org.ovirt.engine.core.vdsbroker.HostMonitoring] (DefaultQuartzScheduler_Worker-61) [5e282aea] Failure to refresh Vds runtime info: java.net.SocketTimeoutException: connect timed out
>2016-02-22 04:01:55,621 ERROR [org.ovirt.engine.core.vdsbroker.HostMonitoring] (DefaultQuartzScheduler_Worker-61) [5e282aea] Exception: org.ovirt.engine.core.vdsbroker.vdsbroker.VDSNetworkException: java.net.SocketTimeoutException: connect timed out
>	at org.ovirt.engine.core.vdsbroker.vdsbroker.VdsBrokerCommand.createNetworkException(VdsBrokerCommand.java:157) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.vdsbroker.VdsBrokerCommand.executeVDSCommand(VdsBrokerCommand.java:120) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.VDSCommandBase.executeCommand(VDSCommandBase.java:65) [vdsbroker.jar:]
>	at org.ovirt.engine.core.dal.VdcCommandBase.execute(VdcCommandBase.java:33) [dal.jar:]
>	at org.ovirt.engine.core.vdsbroker.ResourceManager.runVdsCommand(ResourceManager.java:467) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.VdsManager.refreshCapabilities(VdsManager.java:647) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.HostMonitoring.refreshVdsRunTimeInfo(HostMonitoring.java:119) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.HostMonitoring.refresh(HostMonitoring.java:84) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.VdsManager.onTimer(VdsManager.java:227) [vdsbroker.jar:]
>	at sun.reflect.GeneratedMethodAccessor126.invoke(Unknown Source) [:1.8.0_71]
>	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [rt.jar:1.8.0_71]
>	at java.lang.reflect.Method.invoke(Method.java:497) [rt.jar:1.8.0_71]
>	at org.ovirt.engine.core.utils.timer.JobWrapper.invokeMethod(JobWrapper.java:81) [scheduler.jar:]
>	at org.ovirt.engine.core.utils.timer.JobWrapper.execute(JobWrapper.java:52) [scheduler.jar:]
>	at org.quartz.core.JobRunShell.run(JobRunShell.java:213) [quartz.jar:]
>	at org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:557) [quartz.jar:]
>Caused by: java.net.SocketTimeoutException: connect timed out
>	at java.net.PlainSocketImpl.socketConnect(Native Method) [rt.jar:1.8.0_71]
>	at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) [rt.jar:1.8.0_71]
>	at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) [rt.jar:1.8.0_71]
>	at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) [rt.jar:1.8.0_71]
>	at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) [rt.jar:1.8.0_71]
>	at java.net.Socket.connect(Socket.java:589) [rt.jar:1.8.0_71]
>	at sun.security.ssl.SSLSocketImpl.connect(SSLSocketImpl.java:668) [jsse.jar:1.8.0_71]
>	at org.ovirt.engine.core.utils.ssl.AuthSSLProtocolSocketFactory.createSocket(AuthSSLProtocolSocketFactory.java:293) [utils.jar:]
>	at org.apache.commons.httpclient.HttpConnection.open(HttpConnection.java:707) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.MultiThreadedHttpConnectionManager$HttpConnectionAdapter.open(MultiThreadedHttpConnectionManager.java:1361) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.HttpMethodDirector.executeWithRetry(HttpMethodDirector.java:387) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.HttpMethodDirector.executeMethod(HttpMethodDirector.java:171) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.HttpClient.executeMethod(HttpClient.java:397) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.HttpClient.executeMethod(HttpClient.java:323) [commons-httpclient.jar:]
>	at org.apache.xmlrpc.client.XmlRpcCommonsTransport.writeRequest(XmlRpcCommonsTransport.java:227) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcStreamTransport.sendRequest(XmlRpcStreamTransport.java:151) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcHttpTransport.sendRequest(XmlRpcHttpTransport.java:143) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcClientWorker.execute(XmlRpcClientWorker.java:56) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcClient.execute(XmlRpcClient.java:167) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcClient.execute(XmlRpcClient.java:137) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcClient.execute(XmlRpcClient.java:126) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.util.ClientFactory$1.invoke(ClientFactory.java:140) [xmlrpc-client.jar:3.1.3]
>	at com.sun.proxy.$Proxy167.getVdsCapabilities(Unknown Source)
>	at sun.reflect.GeneratedMethodAccessor344.invoke(Unknown Source) [:1.8.0_71]
>	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [rt.jar:1.8.0_71]
>	at java.lang.reflect.Method.invoke(Method.java:497) [rt.jar:1.8.0_71]
>	at org.ovirt.engine.core.vdsbroker.xmlrpc.XmlRpcUtils$AsyncProxy$InternalCallable.call(XmlRpcUtils.java:263) [vdsbroker.jar:]
>	at java.util.concurrent.FutureTask.run(FutureTask.java:266) [rt.jar:1.8.0_71]
>	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [rt.jar:1.8.0_71]
>	at java.util.concurrent.FutureTask.run(FutureTask.java:266) [rt.jar:1.8.0_71]
>	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [rt.jar:1.8.0_71]
>	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [rt.jar:1.8.0_71]
>	at java.lang.Thread.run(Thread.java:745) [rt.jar:1.8.0_71]
>
>2016-02-22 04:01:55,827 ERROR [org.ovirt.engine.core.vdsbroker.vdsbroker.ListVDSCommand] (DefaultQuartzScheduler_Worker-4) [17f99b9e] Command 'ListVDSCommand(HostName = S1US, VdsIdAndVdsVDSCommandParametersBase:{runAsync='true', hostId='50ad0574-f9f0-473b-8991-9a60f6a5265e', vds='Host[S1US,50ad0574-f9f0-473b-8991-9a60f6a5265e]'})' execution failed: java.net.SocketTimeoutException: connect timed out
>2016-02-22 04:01:55,827 INFO  [org.ovirt.engine.core.vdsbroker.PollVmStatsRefresher] (DefaultQuartzScheduler_Worker-4) [17f99b9e] Failed to fetch vms info for host 'S1US' - skipping VMs monitoring.
>2016-02-22 04:01:56,587 INFO  [org.ovirt.engine.core.vdsbroker.gluster.GlusterVolumesListVDSCommand] (DefaultQuartzScheduler_Worker-26) [] START, GlusterVolumesListVDSCommand(HostName = S1CA, GlusterVolumesListVDSParameters:{runAsync='true', hostId='37775af7-7b73-4a7c-8a78-a2507c951361'}), log id: 34480b18
>2016-02-22 04:01:57,722 INFO  [org.ovirt.engine.core.vdsbroker.gluster.GlusterVolumesListVDSCommand] (DefaultQuartzScheduler_Worker-26) [] FINISH, GlusterVolumesListVDSCommand, return: {}, log id: 34480b18
>2016-02-22 04:02:00,633 ERROR [org.ovirt.engine.core.vdsbroker.vdsbroker.GetCapabilitiesVDSCommand] (DefaultQuartzScheduler_Worker-84) [7eb00d5c] Command 'GetCapabilitiesVDSCommand(HostName = S1US, VdsIdAndVdsVDSCommandParametersBase:{runAsync='true', hostId='50ad0574-f9f0-473b-8991-9a60f6a5265e', vds='Host[S1US,50ad0574-f9f0-473b-8991-9a60f6a5265e]'})' execution failed: java.net.SocketTimeoutException: connect timed out
>2016-02-22 04:02:00,633 ERROR [org.ovirt.engine.core.vdsbroker.HostMonitoring] (DefaultQuartzScheduler_Worker-84) [7eb00d5c] Failure to refresh Vds runtime info: java.net.SocketTimeoutException: connect timed out
>2016-02-22 04:02:00,633 ERROR [org.ovirt.engine.core.vdsbroker.HostMonitoring] (DefaultQuartzScheduler_Worker-84) [7eb00d5c] Exception: org.ovirt.engine.core.vdsbroker.vdsbroker.VDSNetworkException: java.net.SocketTimeoutException: connect timed out
>	at org.ovirt.engine.core.vdsbroker.vdsbroker.VdsBrokerCommand.createNetworkException(VdsBrokerCommand.java:157) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.vdsbroker.VdsBrokerCommand.executeVDSCommand(VdsBrokerCommand.java:120) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.VDSCommandBase.executeCommand(VDSCommandBase.java:65) [vdsbroker.jar:]
>	at org.ovirt.engine.core.dal.VdcCommandBase.execute(VdcCommandBase.java:33) [dal.jar:]
>	at org.ovirt.engine.core.vdsbroker.ResourceManager.runVdsCommand(ResourceManager.java:467) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.VdsManager.refreshCapabilities(VdsManager.java:647) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.HostMonitoring.refreshVdsRunTimeInfo(HostMonitoring.java:119) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.HostMonitoring.refresh(HostMonitoring.java:84) [vdsbroker.jar:]
>	at org.ovirt.engine.core.vdsbroker.VdsManager.onTimer(VdsManager.java:227) [vdsbroker.jar:]
>	at sun.reflect.GeneratedMethodAccessor126.invoke(Unknown Source) [:1.8.0_71]
>	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [rt.jar:1.8.0_71]
>	at java.lang.reflect.Method.invoke(Method.java:497) [rt.jar:1.8.0_71]
>	at org.ovirt.engine.core.utils.timer.JobWrapper.invokeMethod(JobWrapper.java:81) [scheduler.jar:]
>	at org.ovirt.engine.core.utils.timer.JobWrapper.execute(JobWrapper.java:52) [scheduler.jar:]
>	at org.quartz.core.JobRunShell.run(JobRunShell.java:213) [quartz.jar:]
>	at org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:557) [quartz.jar:]
>Caused by: java.net.SocketTimeoutException: connect timed out
>	at java.net.PlainSocketImpl.socketConnect(Native Method) [rt.jar:1.8.0_71]
>	at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) [rt.jar:1.8.0_71]
>	at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) [rt.jar:1.8.0_71]
>	at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) [rt.jar:1.8.0_71]
>	at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) [rt.jar:1.8.0_71]
>	at java.net.Socket.connect(Socket.java:589) [rt.jar:1.8.0_71]
>	at sun.security.ssl.SSLSocketImpl.connect(SSLSocketImpl.java:668) [jsse.jar:1.8.0_71]
>	at org.ovirt.engine.core.utils.ssl.AuthSSLProtocolSocketFactory.createSocket(AuthSSLProtocolSocketFactory.java:293) [utils.jar:]
>	at org.apache.commons.httpclient.HttpConnection.open(HttpConnection.java:707) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.MultiThreadedHttpConnectionManager$HttpConnectionAdapter.open(MultiThreadedHttpConnectionManager.java:1361) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.HttpMethodDirector.executeWithRetry(HttpMethodDirector.java:387) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.HttpMethodDirector.executeMethod(HttpMethodDirector.java:171) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.HttpClient.executeMethod(HttpClient.java:397) [commons-httpclient.jar:]
>	at org.apache.commons.httpclient.HttpClient.executeMethod(HttpClient.java:323) [commons-httpclient.jar:]
>	at org.apache.xmlrpc.client.XmlRpcCommonsTransport.writeRequest(XmlRpcCommonsTransport.java:227) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcStreamTransport.sendRequest(XmlRpcStreamTransport.java:151) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcHttpTransport.sendRequest(XmlRpcHttpTransport.java:143) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcClientWorker.execute(XmlRpcClientWorker.java:56) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcClient.execute(XmlRpcClient.java:167) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcClient.execute(XmlRpcClient.java:137) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.XmlRpcClient.execute(XmlRpcClient.java:126) [xmlrpc-client.jar:3.1.3]
>	at org.apache.xmlrpc.client.util.ClientFactory$1.invoke(ClientFactory.java:140) [xmlrpc-client.jar:3.1.3]
>	at com.sun.proxy.$Proxy167.getVdsCapabilities(Unknown Source)
>	at sun.reflect.GeneratedMethodAccessor344.invoke(Unknown Source) [:1.8.0_71]
>	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [rt.jar:1.8.0_71]
>	at java.lang.reflect.Method.invoke(Method.java:497) [rt.jar:1.8.0_71]
>	at org.ovirt.engine.core.vdsbroker.xmlrpc.XmlRpcUtils$AsyncProxy$InternalCallable.call(XmlRpcUtils.java:263) [vdsbroker.jar:]
>	at java.util.concurrent.FutureTask.run(FutureTask.java:266) [rt.jar:1.8.0_71]
>	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [rt.jar:1.8.0_71]
>	at java.util.concurrent.FutureTask.run(FutureTask.java:266) [rt.jar:1.8.0_71]
>	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [rt.jar:1.8.0_71]
>	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [rt.jar:1.8.0_71]
>	at java.lang.Thread.run(Thread.java:745) [rt.jar:1.8.0_71]
>
>2016-02-22 04:02:00,834 ERROR [org.ovirt.engine.core.vdsbroker.vdsbroker.ListVDSCommand] (DefaultQuartzScheduler_Worker-17) [1009dafa] Command 'ListVDSCommand(HostName = S1US, VdsIdAndVdsVDSCommandParametersBase:{runAsync='true', hostId='50ad0574-f9f0-473b-8991-9a60f6a5265e', vds='Host[S1US,50ad0574-f9f0-473b-8991-9a60f6a5265e]'})' execution failed: java.net.SocketTimeoutException: connect timed out
>2016-02-22 04:02:00,834 INFO  [org.ovirt.engine.core.vdsbroker.PollVmStatsRefresher] (DefaultQuartzScheduler_Worker-17) [1009dafa] Failed to fetch vms info for host 'S1US' - skipping VMs monitoring.

Actions: View

Attachments on bug 1310575: 1129191