Getting continuos error message of Failed to find instance to delegate to for agent in Rancher master logs with Rancher v1.2.2. Agents are showing connected and containers and stacks are running properly it seems.
Infra: Amazon EC2
Rancher: v1.2.2
OS: Amazon Linux 2016.09 for agents and master
Docker: v1.11.2 for agents and master
Database: MySQL 5.7 in AWS RDS
Orchestration: Cattle
Sample exception:
2016-12-29 10:38:19,560 ERROR [:] [] [] [] [ecutorService-1] [i.c.p.agent.impl.AgentLocatorImpl ] Failed to find instance to delegate to for agent [6823] uri [delegate:///?vnetId=236&networkServiceProviderId=9]
2016-12-29 10:38:19,578 ERROR [:] [] [] [] [ecutorService-1] [i.c.p.agent.impl.AgentLocatorImpl ] Failed to find instance to delegate to for agent [187] uri [delegate:///?vnetId=14&networkServiceProviderId=2]
2016-12-29 10:38:19,595 ERROR [:] [] [] [] [ecutorService-1] [i.c.p.agent.impl.AgentLocatorImpl ] Failed to find instance to delegate to for agent [4779] uri [delegate:///?vnetId=145&networkServiceProviderId=9]
2016-12-29 10:38:19,612 ERROR [:] [] [] [] [ecutorService-1] [i.c.p.agent.impl.AgentLocatorImpl ] Failed to find instance to delegate to for agent [6828] uri [delegate:///?vnetId=237&networkServiceProviderId=9]
2016-12-29 10:38:19,629 ERROR [:] [] [] [] [ecutorService-1] [i.c.p.agent.impl.AgentLocatorImpl ] Failed to find instance to delegate to for agent [6880] uri [delegate:///?vnetId=250&networkServiceProviderId=9]
2016-12-29 10:38:19,646 ERROR [:] [] [] [] [ecutorService-1] [i.c.p.agent.impl.AgentLocatorImpl ] Failed to find instance to delegate to for agent [6705] uri [delegate:///?vnetId=211&networkServiceProviderId=9]
2016-12-29 10:38:19,650 ERROR [:] [] [] [] [ecutorService-1] [.p.c.v.i.ConfigItemStatusManagerImpl] Error updating item(s) [ipsec-hosts, ipsec, node-services, agent-instance-startup, monit, agent-instance-scripts, healthcheck, services, iptables, configscripts, hosts] on [agent:6823] com.google.common.util.concurrent.UncheckedExecutionException: java.lang.IllegalStateException: Delegate [delegate:///?vnetId=236&networkServiceProviderId=9] has no instance associated
at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2199) ~[guava-15.0.jar:na]
at com.google.common.cache.LocalCache.get(LocalCache.java:3932) ~[guava-15.0.jar:na]
at com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:3936) ~[guava-15.0.jar:na]
at com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4806) ~[guava-15.0.jar:na]
at com.google.common.cache.LocalCache$LocalLoadingCache.getUnchecked(LocalCache.java:4812) ~[guava-15.0.jar:na]
at io.cattle.platform.agent.impl.AgentLocatorImpl.lookupAgent(AgentLocatorImpl.java:85) ~[cattle-iaas-agent-0.5.0-SNAPSHOT.jar:na]
at io.cattle.platform.configitem.version.impl.ConfigUpdatePublisher.call(ConfigUpdatePublisher.java:327) [cattle-config-item-common-0.5.0-SNAPSHOT.jar:na]
at io.cattle.platform.configitem.version.impl.ConfigUpdatePublisher.publishUpdate(ConfigUpdatePublisher.java:231) [cattle-config-item-common-0.5.0-SNAPSHOT.jar:na]
at io.cattle.platform.configitem.version.impl.ConfigUpdatePublisher.publish(ConfigUpdatePublisher.java:201) [cattle-config-item-common-0.5.0-SNAPSHOT.jar:na]
at io.cattle.platform.configitem.version.impl.ConfigUpdatePublisher.loopBody(ConfigUpdatePublisher.java:127) [cattle-config-item-common-0.5.0-SNAPSHOT.jar:na]
at io.cattle.platform.configitem.version.impl.ConfigUpdatePublisher.doRun(ConfigUpdatePublisher.java:100) [cattle-config-item-common-0.5.0-SNAPSHOT.jar:na]
at org.apache.cloudstack.managed.context.NoExceptionRunnable.runInContext(NoExceptionRunnable.java:15) [cattle-framework-managed-context-0.5.0-SNAPSHOT.jar:na]
at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49) [cattle-framework-managed-context-0.5.0-SNAPSHOT.jar:na]
at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:55) [cattle-framework-managed-context-0.5.0-SNAPSHOT.jar:na]
at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:108) [cattle-framework-managed-context-0.5.0-SNAPSHOT.jar:na]
at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:52) [cattle-framework-managed-context-0.5.0-SNAPSHOT.jar:na]
at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46) [cattle-framework-managed-context-0.5.0-SNAPSHOT.jar:na]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_72]
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) [na:1.8.0_72]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) [na:1.8.0_72]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) [na:1.8.0_72]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [na:1.8.0_72]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [na:1.8.0_72]
at java.lang.Thread.run(Thread.java:745) [na:1.8.0_72]
Caused by: java.lang.IllegalStateException: Delegate [delegate:///?vnetId=236&networkServiceProviderId=9] has no instance associated
at io.cattle.platform.agent.impl.AgentLocatorImpl.getAgentForDelegate(AgentLocatorImpl.java:132) ~[cattle-iaas-agent-0.5.0-SNAPSHOT.jar:na]
at io.cattle.platform.agent.impl.AgentLocatorImpl.buildDelegate(AgentLocatorImpl.java:104) ~[cattle-iaas-agent-0.5.0-SNAPSHOT.jar:na]
at io.cattle.platform.agent.impl.AgentLocatorImpl$1.load(AgentLocatorImpl.java:52) ~[cattle-iaas-agent-0.5.0-SNAPSHOT.jar:na]
at io.cattle.platform.agent.impl.AgentLocatorImpl$1.load(AgentLocatorImpl.java:47) ~[cattle-iaas-agent-0.5.0-SNAPSHOT.jar:na]
at com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3522) ~[guava-15.0.jar:na]
at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2315) ~[guava-15.0.jar:na]
at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2278) ~[guava-15.0.jar:na]
at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2193) ~[guava-15.0.jar:na]
... 23 common frames omitted
2016-12-30 05:48:27,128 ERROR [:] [] [] [] [ecutorService-1] [.p.c.v.i.ConfigItemStatusManagerImpl] Error updating item(s) [ipsec-hosts, ipsec, node-services, agent-instance-startup, monit, healthcheck, agent-instance-scripts, iptables, services, configscripts, hosts] on [agent:6823] com.google.common.util.concurrent.UncheckedExecutionException: java.lang.IllegalStateException: Delegate [delegate:///?vnetId=236&networkServiceProviderId=9] has no instance associated
at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2199) ~[guava-15.0.jar:na]
at com.google.common.cache.LocalCache.get(LocalCache.java:3932) ~[guava-15.0.jar:na]
at com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:3936) ~[guava-15.0.jar:na]
at com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4806) ~[guava-15.0.jar:na]
at com.google.common.cache.LocalCache$LocalLoadingCache.getUnchecked(LocalCache.java:4812) ~[guava-15.0.jar:na]
at io.cattle.platform.agent.impl.AgentLocatorImpl.lookupAgent(AgentLocatorImpl.java:85) ~[cattle-iaas-agent-0.5.0-SNAPSHOT.jar:na]
at io.cattle.platform.configitem.version.impl.ConfigUpdatePublisher.call(ConfigUpdatePublisher.java:327) [cattle-config-item-common-0.5.0-SNAPSHOT.jar:na]
at io.cattle.platform.configitem.version.impl.ConfigUpdatePublisher.publishUpdate(ConfigUpdatePublisher.java:231) [cattle-config-item-common-0.5.0-SNAPSHOT.jar:na]
at io.cattle.platform.configitem.version.impl.ConfigUpdatePublisher.publish(ConfigUpdatePublisher.java:201) [cattle-config-item-common-0.5.0-SNAPSHOT.jar:na]
at io.cattle.platform.configitem.version.impl.ConfigUpdatePublisher.loopBody(ConfigUpdatePublisher.java:127) [cattle-config-item-common-0.5.0-SNAPSHOT.jar:na]
at io.cattle.platform.configitem.version.impl.ConfigUpdatePublisher.doRun(ConfigUpdatePublisher.java:100) [cattle-config-item-common-0.5.0-SNAPSHOT.jar:na]
at org.apache.cloudstack.managed.context.NoExceptionRunnable.runInContext(NoExceptionRunnable.java:15) [cattle-framework-managed-context-0.5.0-SNAPSHOT.jar:na]
at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49) [cattle-framework-managed-context-0.5.0-SNAPSHOT.jar:na]
at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:55) [cattle-framework-managed-context-0.5.0-SNAPSHOT.jar:na]
at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:108) [cattle-framework-managed-context-0.5.0-SNAPSHOT.jar:na]
at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:52) [cattle-framework-managed-context-0.5.0-SNAPSHOT.jar:na]
at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46) [cattle-framework-managed-context-0.5.0-SNAPSHOT.jar:na]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_72]
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) [na:1.8.0_72]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) [na:1.8.0_72]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) [na:1.8.0_72]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [na:1.8.0_72]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [na:1.8.0_72]
at java.lang.Thread.run(Thread.java:745) [na:1.8.0_72]
Caused by: java.lang.IllegalStateException: Delegate [delegate:///?vnetId=236&networkServiceProviderId=9] has no instance associated
at io.cattle.platform.agent.impl.AgentLocatorImpl.getAgentForDelegate(AgentLocatorImpl.java:132) ~[cattle-iaas-agent-0.5.0-SNAPSHOT.jar:na]
at io.cattle.platform.agent.impl.AgentLocatorImpl.buildDelegate(AgentLocatorImpl.java:104) ~[cattle-iaas-agent-0.5.0-SNAPSHOT.jar:na]
at io.cattle.platform.agent.impl.AgentLocatorImpl$1.load(AgentLocatorImpl.java:52) ~[cattle-iaas-agent-0.5.0-SNAPSHOT.jar:na]
at io.cattle.platform.agent.impl.AgentLocatorImpl$1.load(AgentLocatorImpl.java:47) ~[cattle-iaas-agent-0.5.0-SNAPSHOT.jar:na]
at com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3522) ~[guava-15.0.jar:na]
at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2315) ~[guava-15.0.jar:na]
at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2278) ~[guava-15.0.jar:na]
at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2193) ~[guava-15.0.jar:na]
... 23 common frames omitted
2017-01-04 09:50:54,463 ERROR [:] [] [] [] [ecutorService-1] [i.c.p.agent.impl.AgentLocatorImpl ] Failed to find instance to delegate to for agent [317] uri [delegate:///?vnetId=11&networkServiceProviderId=5]
2017-01-04 09:50:54,479 ERROR [:] [] [] [] [ecutorService-1] [i.c.p.agent.impl.AgentLocatorImpl ] Failed to find instance to delegate to for agent [314] uri [delegate:///?vnetId=10&networkServiceProviderId=5]
2017-01-04 09:50:54,484 ERROR [:] [] [] [] [ecutorService-1] [.p.c.v.i.ConfigItemStatusManagerImpl] Error updating item(s) [services, iptables, hosts, ipsec-hosts, ipsec, node-services, agent-instance-startup, monit, agent-instance-scripts, healthcheck] on [agent:317] com.google.common.util.concurrent.UncheckedExecutionException: java.lang.IllegalStateException: Delegate [delegate:///?vnetId=11&networkServiceProviderId=5] has no instance associated
at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2199) ~[guava-15.0.jar:na]
at com.google.common.cache.LocalCache.get(LocalCache.java:3932) ~[guava-15.0.jar:na]
at com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:3936) ~[guava-15.0.jar:na]
at com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4806) ~[guava-15.0.jar:na]
at com.google.common.cache.LocalCache$LocalLoadingCache.getUnchecked(LocalCache.java:4812) ~[guava-15.0.jar:na]
at io.cattle.platform.agent.impl.AgentLocatorImpl.lookupAgent(AgentLocatorImpl.java:85) ~[cattle-iaas-agent-0.5.0-SNAPSHOT.jar:na]
at io.cattle.platform.configitem.version.impl.ConfigUpdatePublisher.call(ConfigUpdatePublisher.java:327) [cattle-config-item-common-0.5.0-SNAPSHOT.jar:na]
at io.cattle.platform.configitem.version.impl.ConfigUpdatePublisher.publishUpdate(ConfigUpdatePublisher.java:231) [cattle-config-item-common-0.5.0-SNAPSHOT.jar:na]
at io.cattle.platform.configitem.version.impl.ConfigUpdatePublisher.publish(ConfigUpdatePublisher.java:201) [cattle-config-item-common-0.5.0-SNAPSHOT.jar:na]
at io.cattle.platform.configitem.version.impl.ConfigUpdatePublisher.loopBody(ConfigUpdatePublisher.java:127) [cattle-config-item-common-0.5.0-SNAPSHOT.jar:na]
at io.cattle.platform.configitem.version.impl.ConfigUpdatePublisher.doRun(ConfigUpdatePublisher.java:100) [cattle-config-item-common-0.5.0-SNAPSHOT.jar:na]
at org.apache.cloudstack.managed.context.NoExceptionRunnable.runInContext(NoExceptionRunnable.java:15) [cattle-framework-managed-context-0.5.0-SNAPSHOT.jar:na]
at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49) [cattle-framework-managed-context-0.5.0-SNAPSHOT.jar:na]
at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:55) [cattle-framework-managed-context-0.5.0-SNAPSHOT.jar:na]
at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:108) [cattle-framework-managed-context-0.5.0-SNAPSHOT.jar:na]
at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:52) [cattle-framework-managed-context-0.5.0-SNAPSHOT.jar:na]
at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46) [cattle-framework-managed-context-0.5.0-SNAPSHOT.jar:na]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_72]
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) [na:1.8.0_72]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) [na:1.8.0_72]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) [na:1.8.0_72]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [na:1.8.0_72]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [na:1.8.0_72]
at java.lang.Thread.run(Thread.java:745) [na:1.8.0_72]
Caused by: java.lang.IllegalStateException: Delegate [delegate:///?vnetId=11&networkServiceProviderId=5] has no instance associated
at io.cattle.platform.agent.impl.AgentLocatorImpl.getAgentForDelegate(AgentLocatorImpl.java:132) ~[cattle-iaas-agent-0.5.0-SNAPSHOT.jar:na]
at io.cattle.platform.agent.impl.AgentLocatorImpl.buildDelegate(AgentLocatorImpl.java:104) ~[cattle-iaas-agent-0.5.0-SNAPSHOT.jar:na]
at io.cattle.platform.agent.impl.AgentLocatorImpl$1.load(AgentLocatorImpl.java:52) ~[cattle-iaas-agent-0.5.0-SNAPSHOT.jar:na]
at io.cattle.platform.agent.impl.AgentLocatorImpl$1.load(AgentLocatorImpl.java:47) ~[cattle-iaas-agent-0.5.0-SNAPSHOT.jar:na]
at com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3522) ~[guava-15.0.jar:na]
at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2315) ~[guava-15.0.jar:na]
at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2278) ~[guava-15.0.jar:na]
at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2193) ~[guava-15.0.jar:na]
... 23 common frames omitted
Besides this error output in the logs everything seems to work properly. If i start a fresh rancher-server instance no error occurs.