Description:
I'm getting error messages like:
java.lang.RuntimeException: jdbc:mysql://db-hostname:3306/mem?createDatabaseIfNotExist=true&elideSetAutoCommits=true&useServerPrepStmts=false&cachePrepStmts=true&cacheCallableStmts=true&noAccessToProcedureBodies=true&characterEncoding=UTF-8&includeInnodbStatusInDeadlockExceptions=true&cacheServerConfiguration=true&useDynamicCharsetInfo=false&useLocalSessionState=true&alwaysSendSetIsolation=false&rewriteBatchedStatements=true&prepStmtCacheSize=500&prepStmtCacheSqlLimit=4096&connectionLifecycleInterceptors=com.mysql.etools.monitor.pom.NoAutocommitConnectionInterceptor&sessionVariables=sql_mode='STRICT_TRANS_TABLES,NO_ENGINE_SUBSTITUTION'
at com.mysql.etools.monitor.pom.DBCPConnectionProvider.getConnection(DBCPConnectionProvider.java:469)
at com.mysql.etools.monitor.pom.ConfigTimeConnectionProviderHolder.getConnection(ConfigTimeConnectionProviderHolder.java:79)
at org.hibernate.jdbc.ConnectionManager.openConnection(ConnectionManager.java:446)
at org.hibernate.jdbc.ConnectionManager.getConnection(ConnectionManager.java:167)
at org.hibernate.jdbc.JDBCContext.connection(JDBCContext.java:142)
at org.hibernate.transaction.JDBCTransaction.begin(JDBCTransaction.java:85)
at org.hibernate.impl.SessionImpl.beginTransaction(SessionImpl.java:1354)
at sun.reflect.GeneratedMethodAccessor45.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
at java.lang.reflect.Method.invoke(Unknown Source)
at org.hibernate.context.ThreadLocalSessionContext$TransactionProtectionWrapper.invoke(ThreadLocalSessionContext.java:342)
at $Proxy8.beginTransaction(Unknown Source)
at com.mysql.etools.monitor.pom.hib.HibernateExecutor.asTransaction(HibernateExecutor.java:220)
at com.mysql.etools.monitor.pom.hib.HibernateExecutor.asTransaction(HibernateExecutor.java:182)
at com.mysql.etools.monitor.pom.hib.SchemaMaintenanceExecutor.asTransaction(SchemaMaintenanceExecutor.java:204)
at com.mysql.etools.monitor.pom.hib.HibPersistence.getObject(HibPersistence.java:480)
at com.mysql.etools.monitor.pom.hib.HibInventoryObject.findParent(HibInventoryObject.java:133)
at com.mysql.etools.monitor.pom.hib.HibInventoryObject.findParent(HibInventoryObject.java:61)
at com.mysql.etools.monitor.pom.InvObj.findParent(InvObj.java:454)
at com.mysql.etools.monitor.bo.Manager.ensureHostForServerInventory(Manager.java:312)
at com.mysql.etools.monitor.bo.Manager.ensureServer(Manager.java:292)
at com.mysql.etools.monitor.bo.Manager.ensureServer(Manager.java:284)
at com.mysql.etools.monitor.bo.Manager.ensureServer(Manager.java:280)
at com.mysql.etools.monitor.bo.ManagerRestApi.put(ManagerRestApi.java:143)
at com.mysql.etools.monitor.bo.RestAdaptor$4.apply(RestAdaptor.java:253)
at com.mysql.etools.monitor.bo.RestAdaptor$4.apply(RestAdaptor.java:248)
at com.mysql.etools.monitor.bo.RestAdaptor$7.apply(RestAdaptor.java:290)
at com.mysql.etools.monitor.bo.RestAdaptor$7.apply(RestAdaptor.java:288)
at com.mysql.etools.monitor.bo.RestAdaptor.foreach(RestAdaptor.java:303)
at com.mysql.etools.monitor.bo.RestAdaptor.foreachInstance(RestAdaptor.java:287)
at com.mysql.etools.monitor.bo.RestAdaptor.putMatching(RestAdaptor.java:247)
at com.mysql.etools.monitor.bo.Agent.processTasks(Agent.java:588)
at com.mysql.etools.monitor.bo.Agent.lockedAcceptHeartbeat(Agent.java:425)
at com.mysql.etools.monitor.bo.Agent.lockedAcceptHeartbeat(Agent.java:402)
at com.mysql.etools.monitor.bo.Agent.access$100(Agent.java:80)
at com.mysql.etools.monitor.bo.Agent$3.call(Agent.java:391)
at com.mysql.etools.monitor.bo.Agent$3.call(Agent.java:390)
at com.mysql.etools.concurrent.LockingExecutor.execute(LockingExecutor.java:39)
at com.mysql.etools.monitor.bo.Agent.acceptHeartbeat(Agent.java:389)
at com.mysql.etools.monitor.bo.HeartbeatAcceptor.acceptHeartbeat(HeartbeatAcceptor.java:60)
at com.mysql.etools.monitor.bo.HeartBeatCommandProcessor.processRequest(HeartBeatCommandProcessor.java:102)
at com.mysql.merlin.server.MerlinServlet$1.processRequest(MerlinServlet.java:321)
at com.mysql.merlin.server.agent.HeartbeatServlet$1.processRequest(HeartbeatServlet.java:59)
at com.mysql.merlin.server.MerlinServlet.doCommon(MerlinServlet.java:290)
at com.mysql.merlin.server.MerlinServlet.doPost(MerlinServlet.java:412)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:637)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at com.mysql.merlin.ui.filters.UiActionMarkerFilter.doFilter(UiActionMarkerFilter.java:48)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at com.mysql.merlin.server.RequestCounterFilter.doFilter(RequestCounterFilter.java:133)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at com.mysql.etools.monitor.rest.AuthFilter.doFilter(AuthFilter.java:93)
at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:236)
at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:167)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:298)
at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:859)
at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:588)
at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:489)
at java.lang.Thread.run(Unknown Source)
Caused by: org.apache.commons.dbcp.SQLNestedException: Cannot get a connection, pool exhausted
and I find on the mysql site: http://dev.mysql.com/doc/mysql-monitor/2.3/en/mem-config-properties-file-location.html yet this does not document the settings in this file, including the new setting which relates to db pools (their names and sizes).
How to repeat:
Startup merlin monitoring a large number of mysqld instances (say > 300).
Suggested fix:
* Fix the documentation so that all possible parameters are documented.
* Make the error message specify which pool is affected, and possibly confirm the maximum setting value that's being used.