+++ This bug was initially created as a clone of Bug #1459985 +++ Description of problem: Refresh pools jobs are being created but not scheduled, and are stuck in a CREATED state indefinitely Version-Release number of selected component (if applicable): 2.0.33, 2.0.36 How reproducible: Jobs are still being processed, but there are some that end up in this state. It is unknown why the job scheduler is throwing an error, though the root cause on the stack trace shows Lock Wait Timeout. Steps to Reproduce: 1.Issue a refresh pools job 2.check the status of the job 3. Actual results: Jobs are stuck in CREATED state for days Expected results: Jobs that error are put in a FAILED state immediately Additional info: Stack Trace when scheduler fails: 2017-06-02 11:37:21,450 [thread=http-8080-9] [req=a741764d-92dd-48f6-948c-2b5e8f6cdced, org=, csid=] ERROR org.candlepin.pinsetter.core.PinsetterKernel - There was a problem scheduling refresh_pools_bff1d406-45ce-41c7-9d0d-d24c9ade446f java.lang.reflect.InvocationTargetException: null at sun.reflect.GeneratedMethodAccessor632.invoke(Unknown Source) ~[na:na] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_101] at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_101] at org.candlepin.pinsetter.core.PinsetterKernel.scheduleJob(PinsetterKernel.java:383) [PinsetterKernel.class:na] at org.candlepin.pinsetter.core.PinsetterKernel.scheduleSingleJob(PinsetterKernel.java:428) [PinsetterKernel.class:na] at org.candlepin.resteasy.filter.PinsetterAsyncFilter.scheduleJob(PinsetterAsyncFilter.java:91) [PinsetterAsyncFilter.class:na] at org.candlepin.resteasy.filter.PinsetterAsyncFilter.postProcess(PinsetterAsyncFilter.java:69) [PinsetterAsyncFilter.class:na] at org.jboss.resteasy.core.interception.ContainerResponseFilterRegistry$ContainerResponseFilterFacade.filter(ContainerResponseFilterRegistry.java:47) [resteasy-jaxrs-3.0.10.Final.jar:na] at org.jboss.resteasy.core.ServerResponseWriter.executeFilters(ServerResponseWriter.java:122) [resteasy-jaxrs-3.0.10.Final.jar:na] at org.jboss.resteasy.core.ServerResponseWriter.writeNomapResponse(ServerResponseWriter.java:49) [resteasy-jaxrs-3.0.10.Final.jar:na] at org.jboss.resteasy.core.SynchronousDispatcher.writeResponse(SynchronousDispatcher.java:427) [resteasy-jaxrs-3.0.10.Final.jar:na] at org.jboss.resteasy.core.SynchronousDispatcher.invoke(SynchronousDispatcher.java:376) [resteasy-jaxrs-3.0.10.Final.jar:na] at org.jboss.resteasy.core.SynchronousDispatcher.invoke(SynchronousDispatcher.java:179) [resteasy-jaxrs-3.0.10.Final.jar:na] at org.jboss.resteasy.plugins.server.servlet.ServletContainerDispatcher.service(ServletContainerDispatcher.java:220) [resteasy-jaxrs-3.0.10.Final.jar:na] at org.jboss.resteasy.plugins.server.servlet.HttpServletDispatcher.service(HttpServletDispatcher.java:56) [resteasy-jaxrs-3.0.10.Final.jar:na] at org.jboss.resteasy.plugins.server.servlet.HttpServletDispatcher.service(HttpServletDispatcher.java:51) [resteasy-jaxrs-3.0.10.Final.jar:na] at javax.servlet.http.HttpServlet.service(HttpServlet.java:723) [servlet-api.jar:na] at com.google.inject.servlet.ServletDefinition.doService(ServletDefinition.java:263) [guice-servlet-3.0.jar:na] at com.google.inject.servlet.ServletDefinition.service(ServletDefinition.java:178) [guice-servlet-3.0.jar:na] at com.google.inject.servlet.ManagedServletPipeline.service(ManagedServletPipeline.java:91) [guice-servlet-3.0.jar:na] at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:62) [guice-servlet-3.0.jar:na] at org.candlepin.servlet.filter.EventFilter.doFilter(EventFilter.java:61) [EventFilter.class:na] at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:163) [guice-servlet-3.0.jar:na] at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) [guice-servlet-3.0.jar:na] at org.candlepin.servlet.filter.ContentTypeHackFilter.doFilter(ContentTypeHackFilter.java:58) [ContentTypeHackFilter.class:na] at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:163) [guice-servlet-3.0.jar:na] at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) [guice-servlet-3.0.jar:na] at org.candlepin.common.filter.LoggingFilter.doFilter(LoggingFilter.java:112) [candlepin-common-2.0.3.jar:na] at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:163) [guice-servlet-3.0.jar:na] at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) [guice-servlet-3.0.jar:na] at org.candlepin.servlet.filter.CandlepinPersistFilter.doFilter(CandlepinPersistFilter.java:48) [CandlepinPersistFilter.class:na] at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:163) [guice-servlet-3.0.jar:na] at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) [guice-servlet-3.0.jar:na] at org.candlepin.servlet.filter.CandlepinScopeFilter.doFilter(CandlepinScopeFilter.java:68) [CandlepinScopeFilter.class:na] at com.google.inject.servlet.FilterDefinition.doFilter(FilterDefinition.java:163) [guice-servlet-3.0.jar:na] at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:58) [guice-servlet-3.0.jar:na] at com.google.inject.servlet.ManagedFilterPipeline.dispatch(ManagedFilterPipeline.java:118) [guice-servlet-3.0.jar:na] at com.google.inject.servlet.GuiceFilter.doFilter(GuiceFilter.java:113) [guice-servlet-3.0.jar:na] at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235) [catalina.jar:6.0.37] at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) [catalina.jar:6.0.37] at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233) [catalina.jar:6.0.37] at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191) [catalina.jar:6.0.37] at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127) [catalina.jar:6.0.37] at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103) [catalina.jar:6.0.37] at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:615) [catalina.jar:6.0.37] at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109) [catalina.jar:6.0.37] at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:293) [catalina.jar:6.0.37] at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:861) [tomcat-coyote.jar:6.0.37] at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:606) [tomcat-coyote.jar:6.0.37] at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:489) [tomcat-coyote.jar:6.0.37] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_101] Caused by: org.quartz.impl.jdbcjobstore.LockException: Failure obtaining db row lock: Lock wait timeout exceeded; try restarting transaction at org.quartz.impl.jdbcjobstore.StdRowLockSemaphore.executeSQL(StdRowLockSemaphore.java:157) ~[quartz-2.2.1.jar:na] at org.quartz.impl.jdbcjobstore.DBSemaphore.obtainLock(DBSemaphore.java:113) ~[quartz-2.2.1.jar:na] at org.quartz.impl.jdbcjobstore.JobStoreSupport.executeInNonManagedTXLock(JobStoreSupport.java:3780) ~[quartz-2.2.1.jar:na] at org.quartz.impl.jdbcjobstore.JobStoreTX.executeInLock(JobStoreTX.java:93) ~[quartz-2.2.1.jar:na] at org.quartz.impl.jdbcjobstore.JobStoreSupport.storeJobAndTrigger(JobStoreSupport.java:1058) ~[quartz-2.2.1.jar:na] at org.quartz.core.QuartzScheduler.scheduleJob(QuartzScheduler.java:886) ~[quartz-2.2.1.jar:na] at org.quartz.impl.StdScheduler.scheduleJob(StdScheduler.java:249) ~[quartz-2.2.1.jar:na] at org.candlepin.pinsetter.tasks.KingpinJob.scheduleJob(KingpinJob.java:164) ~[KingpinJob.class:na] at org.candlepin.pinsetter.tasks.UniqueByEntityJob.scheduleJob(UniqueByEntityJob.java:46) ~[UniqueByEntityJob.class:na] ... 51 common frames omitted Caused by: java.sql.SQLException: Lock wait timeout exceeded; try restarting transaction at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:1078) ~[mysql-connector-java-5.1.26.jar:na] at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:4190) ~[mysql-connector-java-5.1.26.jar:na] at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:4122) ~[mysql-connector-java-5.1.26.jar:na] at com.mysql.jdbc.MysqlIO.sendCommand(MysqlIO.java:2570) ~[mysql-connector-java-5.1.26.jar:na] at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:2731) ~[mysql-connector-java-5.1.26.jar:na] at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2818) ~[mysql-connector-java-5.1.26.jar:na] at com.mysql.jdbc.PreparedStatement.executeInternal(PreparedStatement.java:2157) ~[mysql-connector-java-5.1.26.jar:na] at com.mysql.jdbc.PreparedStatement.executeQuery(PreparedStatement.java:2315) ~[mysql-connector-java-5.1.26.jar:na] at com.mchange.v2.c3p0.impl.NewProxyPreparedStatement.executeQuery(NewProxyPreparedStatement.java:353) ~[c3p0-0.9.5.2.jar:0.9.5.2] at org.quartz.impl.jdbcjobstore.StdRowLockSemaphore.executeSQL(StdRowLockSemaphore.java:96) ~[quartz-2.2.1.jar:na] ... 59 common frames omitted