| Bug #49882 | dissociating an orphan'd mysql instance throws exceptions at when rebinding rule | ||
|---|---|---|---|
| Submitted: | 22 Dec 2009 16:37 | Modified: | 18 Jan 2010 14:44 |
| Reporter: | Marcos Palacios | Email Updates: | |
| Status: | Closed | Impact on me: | |
| Category: | MySQL Enterprise Monitor: Server | Severity: | S3 (Non-critical) |
| Version: | 2.1.1.1131 | OS: | Any |
| Assigned to: | Darren Oldag | CPU Architecture: | Any |
[22 Dec 2009 16:38]
Marcos Palacios
2009-12-22 10:19:16,833 ERROR [http-18443-4:com.mysql.infra.com.mysql.merlin.server.MerlinServlet] <?xml version='1.0'?><exceptions><error><![CDATA[Internal error: Transport is of type null. No Agent instance... marcos-palacioss-macbook-pro.local:13306 %agent.reachable% mysql.Agent.agent.reachable local]]></error></exceptions> java.lang.RuntimeException: Transport is of type null. No Agent instance... marcos-palacioss-macbook-pro.local:13306 %agent.reachable% mysql.Agent.agent.reachable local at com.mysql.etools.monitor.bo.InstanceFinder.getAgentFromServer(InstanceFinder.java:138) at com.mysql.etools.monitor.bo.InstanceFinder.findInstance(InstanceFinder.java:56) at com.mysql.etools.monitor.bo.VariableBinder.getVariableInstance(VariableBinder.java:121) at com.mysql.etools.monitor.bo.VariableBinder.bindCollections(VariableBinder.java:58) at com.mysql.etools.monitor.bo.RuleScheduleBuilder.bindCollections(RuleScheduleBuilder.java:56) at com.mysql.etools.monitor.bo.rule.Schedule.rebindCollections(Schedule.java:355) at com.mysql.etools.monitor.bo.Server.rebindRuleScheduleVariables(Server.java:982) at com.mysql.etools.monitor.bo.Server.rebindScheduleVariables(Server.java:922) at com.mysql.etools.monitor.bo.Server.setTransport(Server.java:913) at com.mysql.etools.monitor.bo.Agent.dissociateUnreportedServers(Agent.java:579) at com.mysql.etools.monitor.bo.Agent.processTasks(Agent.java:562) at com.mysql.etools.monitor.bo.Agent.lockedAcceptHeartbeat(Agent.java:394) at com.mysql.etools.monitor.bo.Agent.lockedAcceptHeartbeat(Agent.java:371) at com.mysql.etools.monitor.bo.Agent.access$100(Agent.java:74) at com.mysql.etools.monitor.bo.Agent$2.call(Agent.java:360) at com.mysql.etools.monitor.bo.Agent$2.call(Agent.java:359) at com.mysql.util.LockingExecutor.execute(LockingExecutor.java:39) at com.mysql.etools.monitor.bo.Agent.acceptHeartbeat(Agent.java:358) at com.mysql.etools.monitor.bo.HeartbeatAcceptor.acceptHeartbeat(HeartbeatAcceptor.java:60) at com.mysql.etools.monitor.bo.HeartBeatCommandProcessor.processRequest(HeartBeatCommandProcessor.java:102) at com.mysql.merlin.server.MerlinServlet$1.processRequest(MerlinServlet.java:328) at com.mysql.merlin.server.agent.HeartbeatServlet$1.processRequest(HeartbeatServlet.java:62) at com.mysql.merlin.server.MerlinServlet.doCommon(MerlinServlet.java:297) at com.mysql.merlin.server.MerlinServlet.doPost(MerlinServlet.java:419) at javax.servlet.http.HttpServlet.service(HttpServlet.java:710) at javax.servlet.http.HttpServlet.service(HttpServlet.java:803) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at com.mysql.util.RequestCounterFilter.doFilter(RequestCounterFilter.java:117) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at com.mysql.etools.monitor.rest.AuthFilter.doFilter(AuthFilter.java:98) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:183) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:138) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233) at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:175) at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:128) at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102) at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109) at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:263) at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:844) at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:584) at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:447) at java.lang.Thread.run(Thread.java:637)
[22 Dec 2009 16:40]
Marcos Palacios
2009-12-22 10:19:58,887 ERROR [http-18080-9:com.mysql.infra.com.mysql.etools.monitor.pom.hib.HibernateExecutor] asTransaction java.lang.NullPointerException at com.mysql.etools.monitor.pom.hib.HibInventoryObject.saveAttribute(HibInventoryObject.java:225) at com.mysql.etools.monitor.pom.InvObj.saveAttribute(InvObj.java:207) at com.mysql.etools.monitor.pom.InvObj.save(InvObj.java:212) at com.mysql.etools.monitor.pom.InvObj.saveAttribute(InvObj.java:220) at com.mysql.etools.monitor.bo.Server.setCachedDisplayName(Server.java:1006) at com.mysql.etools.monitor.bo.Server.cacheDisplaynameIfHostnameValid(Server.java:466) at com.mysql.etools.monitor.bo.Server.getDisplayName(Server.java:424) at com.mysql.merlin.ui.containers.ServerContainer.<init>(ServerContainer.java:119) at com.mysql.merlin.ui.helper.UiManager.fromCache(UiManager.java:98) at com.mysql.merlin.ui.helper.UiManager.fromCache(UiManager.java:88) at com.mysql.merlin.ui.containers.GroupContainer.<init>(GroupContainer.java:69) at com.mysql.merlin.ui.helper.UiManager.fromCache(UiManager.java:109) at com.mysql.merlin.ui.actions.BaseAction.getDefaultGroup(BaseAction.java:972) at com.mysql.merlin.ui.actions.BaseAction.getSelectedGroup(BaseAction.java:1338) at com.mysql.merlin.ui.actions.monitor.dashboard.Dashboard.updateAlarms(Dashboard.java:203) at com.mysql.merlin.ui.actions.monitor.dashboard.Dashboard.executeInner(Dashboard.java:181) at com.mysql.merlin.ui.actions.BaseViewAction.timeLogExecInner(BaseViewAction.java:70) at com.mysql.merlin.ui.actions.BaseViewAction.access$000(BaseViewAction.java:29) at com.mysql.merlin.ui.actions.BaseViewAction$1.call(BaseViewAction.java:62) at com.mysql.merlin.ui.actions.BaseViewAction$1.call(BaseViewAction.java:61) at com.mysql.etools.monitor.pom.hib.HibernateExecutor$2.execute(HibernateExecutor.java:142) at com.mysql.etools.monitor.pom.hib.HibernateExecutor.asTransaction(HibernateExecutor.java:188) at com.mysql.etools.monitor.pom.hib.HibernateExecutor.asReadOnlyTransaction(HibernateExecutor.java:148) at com.mysql.etools.monitor.pom.hib.SchemaMaintenanceExecutor.asReadOnlyTransaction(SchemaMaintenanceExecutor.java:214) at com.mysql.etools.monitor.pom.hib.HibPersistence.asReadOnlyTransaction(HibPersistence.java:1397) at com.mysql.merlin.ui.actions.BaseViewAction.openSessionInViewTxn(BaseViewAction.java:60) at com.mysql.merlin.ui.actions.BaseViewAction.exec(BaseViewAction.java:46) at com.mysql.merlin.ui.actions.BaseAction$1.call(BaseAction.java:1920) at com.mysql.merlin.ui.actions.BaseAction$1.call(BaseAction.java:1919) at com.mysql.merlin.ui.actions.BaseAction.execute(BaseAction.java:1903) at com.mysql.merlin.ui.actions.BaseAction.execute(BaseAction.java:1918) at sun.reflect.GeneratedMethodAccessor171.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at com.opensymphony.xwork2.DefaultActionInvocation.invokeAction(DefaultActionInvocation.java:404) at com.opensymphony.xwork2.DefaultActionInvocation.invokeActionOnly(DefaultActionInvocation.java:267) at com.opensymphony.xwork2.DefaultActionInvocation.invoke(DefaultActionInvocation.java:229) at com.opensymphony.xwork2.interceptor.ParametersInterceptor.doIntercept(ParametersInterceptor.java:167) at com.opensymphony.xwork2.interceptor.MethodFilterInterceptor.intercept(MethodFilterInterceptor.java:86) at com.opensymphony.xwork2.DefaultActionInvocation$2.doProfiling(DefaultActionInvocation.java:224) at com.opensymphony.xwork2.DefaultActionInvocation$2.doProfiling(DefaultActionInvocation.java:1) at com.opensymphony.xwork2.util.profiling.UtilTimerStack.profile(UtilTimerStack.java:455) at com.opensymphony.xwork2.DefaultActionInvocation.invoke(DefaultActionInvocation.java:221) at org.apache.struts2.interceptor.ServletConfigInterceptor.intercept(ServletConfigInterceptor.java:170) at com.opensymphony.xwork2.DefaultActionInvocation$2.doProfiling(DefaultActionInvocation.java:224) at com.opensymphony.xwork2.DefaultActionInvocation$2.doProfiling(DefaultActionInvocation.java:1) at com.opensymphony.xwork2.util.profiling.UtilTimerStack.profile(UtilTimerStack.java:455) at com.opensymphony.xwork2.DefaultActionInvocation.invoke(DefaultActionInvocation.java:221) at com.mysql.merlin.ui.interceptors.InitializeInterceptor.intercept(InitializeInterceptor.java:82) at com.opensymphony.xwork2.DefaultActionInvocation$2.doProfiling(DefaultActionInvocation.java:224) at com.opensymphony.xwork2.DefaultActionInvocation$2.doProfiling(DefaultActionInvocation.java:1) at com.opensymphony.xwork2.util.profiling.UtilTimerStack.profile(UtilTimerStack.java:455) at com.opensymphony.xwork2.DefaultActionInvocation.invoke(DefaultActionInvocation.java:221) at com.opensymphony.xwork2.interceptor.ExceptionMappingInterceptor.intercept(ExceptionMappingInterceptor.java:176) at com.opensymphony.xwork2.DefaultActionInvocation$2.doProfiling(DefaultActionInvocation.java:224) at com.opensymphony.xwork2.DefaultActionInvocation$2.doProfiling(DefaultActionInvocation.java:1) at com.opensymphony.xwork2.util.profiling.UtilTimerStack.profile(UtilTimerStack.java:455) at com.opensymphony.xwork2.DefaultActionInvocation.invoke(DefaultActionInvocation.java:221) at com.mysql.merlin.ui.interceptors.AuthenticationInterceptor.intercept(AuthenticationInterceptor.java:118) at com.opensymphony.xwork2.DefaultActionInvocation$2.doProfiling(DefaultActionInvocation.java:224) at com.opensymphony.xwork2.DefaultActionInvocation$2.doProfiling(DefaultActionInvocation.java:1) at com.opensymphony.xwork2.util.profiling.UtilTimerStack.profile(UtilTimerStack.java:455) at com.opensymphony.xwork2.DefaultActionInvocation.invoke(DefaultActionInvocation.java:221) at org.apache.struts2.impl.StrutsActionProxy.execute(StrutsActionProxy.java:50) at org.apache.struts2.dispatcher.Dispatcher.serviceAction(Dispatcher.java:507) at org.apache.struts2.dispatcher.FilterDispatcher.doFilter(FilterDispatcher.java:421) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at net.sf.ehcache.constructs.web.filter.GzipFilter.doFilter(GzipFilter.java:81) at net.sf.ehcache.constructs.web.filter.Filter.doFilter(Filter.java:92) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at com.mysql.util.RequestCounterFilter.doFilter(RequestCounterFilter.java:117) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at com.mysql.merlin.ui.filters.AccessLogFilter.doFilter(AccessLogFilter.java:56) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233) at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:175) at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:128) at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102) at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109) at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:263) at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:844) at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:584) at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:447) at java.lang.Thread.run(Thread.java:637)
[14 Jan 2010 1:03]
Enterprise Tools JIRA Robot
Keith Russell writes: Patch available in builds => 2.1.1.1138.
[15 Jan 2010 18:16]
Enterprise Tools JIRA Robot
Marcos Palacios writes: Verified fixed in Monitor build 2.1.1.1138.
[18 Jan 2010 14:44]
MC Brown
A note has been added to the 2.1.1 and 2.2.0 changelog: When creating a new instance by copying an existing agent configuration, it is possible to create an orphaned agent. The recommended advice is to <literal>TRUNCATE</literal> the <literal>mysql.inventory</literal> table. However, doing this could lead to additional errors and an exception when the scheduled data updates on the now orphaned agent are executed.
Description: When the agent gets a message like: "(critical) agent_mysqld.c:611: [mysql] the hostid from mysql.inventory doesn't match our agent's host-id (mac:{001ec2c3f5840000} != mac:{001f5bf604680000}) We assume that this is a cloned host and shutdown now. Please TRUNCATE TABLE mysql.inventory on this mysql-instance and restart the agent. If this is a master for replication, please also run SET SQL_LOG_BIN = 0; first." If we do the TRUNCATE, then when we start the agent again, we get: 2009-12-22 10:19:16: (critical) exception received from server: Internal error: Transport is of type null. No Agent instance... marcos-palacioss-macbook-pro.local:13306 %agent.reachable% mysql.Agent.agent.reachable local and the monitor has: 2009-12-22 10:19:16,795 WARN [http-18443-4:com.mysql.agentTask] Client IP: 127.0.0.1 java.lang.RuntimeException: Transport is of type null. No Agent instance... marcos-palacioss-macbook-pro.local:13306 %agent.reachable% mysql.Agent.agent.reachable local at com.mysql.etools.monitor.bo.InstanceFinder.getAgentFromServer(InstanceFinder.java:138) at com.mysql.etools.monitor.bo.InstanceFinder.findInstance(InstanceFinder.java:56) at com.mysql.etools.monitor.bo.VariableBinder.getVariableInstance(VariableBinder.java:121) at com.mysql.etools.monitor.bo.VariableBinder.bindCollections(VariableBinder.java:58) at com.mysql.etools.monitor.bo.RuleScheduleBuilder.bindCollections(RuleScheduleBuilder.java:56) at com.mysql.etools.monitor.bo.rule.Schedule.rebindCollections(Schedule.java:355) at com.mysql.etools.monitor.bo.Server.rebindRuleScheduleVariables(Server.java:982) at com.mysql.etools.monitor.bo.Server.rebindScheduleVariables(Server.java:922) at com.mysql.etools.monitor.bo.Server.setTransport(Server.java:913) at com.mysql.etools.monitor.bo.Agent.dissociateUnreportedServers(Agent.java:579) at com.mysql.etools.monitor.bo.Agent.processTasks(Agent.java:562) at com.mysql.etools.monitor.bo.Agent.lockedAcceptHeartbeat(Agent.java:394) at com.mysql.etools.monitor.bo.Agent.lockedAcceptHeartbeat(Agent.java:371) at com.mysql.etools.monitor.bo.Agent.access$100(Agent.java:74) at com.mysql.etools.monitor.bo.Agent$2.call(Agent.java:360) at com.mysql.etools.monitor.bo.Agent$2.call(Agent.java:359) at com.mysql.util.LockingExecutor.execute(LockingExecutor.java:39) at com.mysql.etools.monitor.bo.Agent.acceptHeartbeat(Agent.java:358) at com.mysql.etools.monitor.bo.HeartbeatAcceptor.acceptHeartbeat(HeartbeatAcceptor.java:60) at com.mysql.etools.monitor.bo.HeartBeatCommandProcessor.processRequest(HeartBeatCommandProcessor.java:102) at com.mysql.merlin.server.MerlinServlet$1.processRequest(MerlinServlet.java:328) at com.mysql.merlin.server.agent.HeartbeatServlet$1.processRequest(HeartbeatServlet.java:62) at com.mysql.merlin.server.MerlinServlet.doCommon(MerlinServlet.java:297) at com.mysql.merlin.server.MerlinServlet.doPost(MerlinServlet.java:419) at javax.servlet.http.HttpServlet.service(HttpServlet.java:710) at javax.servlet.http.HttpServlet.service(HttpServlet.java:803) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at com.mysql.util.RequestCounterFilter.doFilter(RequestCounterFilter.java:117) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at com.mysql.etools.monitor.rest.AuthFilter.doFilter(AuthFilter.java:98) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:183) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:138) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233) at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:175) at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:128) at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102) at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109) at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:263) at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:844) at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:584) at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:447) at java.lang.Thread.run(Thread.java:637) to be continued in a comment... How to repeat: - install monitor and agent - let them run a bit - stop the agent - do TRUNCATE TABLE mysql.inventory on this mysql-instance - restart the agent - check the logs