I am trying to set up a consul cluster in my Rancher setup.
My Rancher version is 1.5.7. I have 3 Ubuntu 16.04.2 hosts running Docker 17.03.1-ce as a cattle cluster.
When I create the cluster, in the stack I see 3 consul containers (running), and 3 consul-base containers (running), 3 consul-data containers (started-once) and one consul-lb container. The consul-lb container keeps starting, and stopping over and over.
The logs for the lb container show the following:
5/4/2017 12:53:02 PMruntime.cgocall(0x1bd7c30, 0xc82002ebd8, 0x0)
5/4/2017 12:53:02 PM /usr/local/go/src/runtime/cgocall.go:123 +0x11b fp=0xc82002eb78 sp=0xc82002eb48
5/4/2017 12:53:02 PMnet._C2func_getaddrinfo(0x7fa4dc0008c0, 0x0, 0xc8204c1740, 0xc8200ea508, 0x0, 0x0, 0x0)
5/4/2017 12:53:02 PM ??:0 +0x55 fp=0xc82002ebd8 sp=0xc82002eb78
5/4/2017 12:53:02 PMnet.cgoLookupIPCNAME(0xc8204c12f0, 0x22, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x296bb00)
5/4/2017 12:53:02 PM /usr/local/go/src/net/cgo_unix.go:111 +0x448 fp=0xc82002ed50 sp=0xc82002ebd8
5/4/2017 12:53:02 PMnet.cgoLookupIP(0xc8204c12f0, 0x22, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
5/4/2017 12:53:02 PM /usr/local/go/src/net/cgo_unix.go:163 +0x56 fp=0xc82002eda8 sp=0xc82002ed50
5/4/2017 12:53:02 PMnet.lookupIP(0xc8204c12f0, 0x22, 0x0, 0x0, 0x0, 0x0, 0x0)
5/4/2017 12:53:02 PM /usr/local/go/src/net/lookup_unix.go:67 +0x94 fp=0xc82002ee18 sp=0xc82002eda8
5/4/2017 12:53:02 PMnet.glob.func16(0x296bb70, 0xc8204c12f0, 0x22, 0x0, 0x0, 0x0, 0x0, 0x0)
5/4/2017 12:53:02 PM /usr/local/go/src/net/hook.go:10 +0x4d fp=0xc82002ee58 sp=0xc82002ee18
5/4/2017 12:53:02 PMnet.lookupIPDeadline.func1(0x0, 0x0, 0x0, 0x0)
5/4/2017 12:53:02 PM /usr/local/go/src/net/lookup.go:106 +0x71 fp=0xc82002eed8 sp=0xc82002ee58
5/4/2017 12:53:02 PMinternal/singleflight.(*Group).doCall(0x36674d0, 0xc82000a3c0, 0xc8204c12f0, 0x22, 0xc82000e280)
5/4/2017 12:53:02 PM /usr/local/go/src/internal/singleflight/singleflight.go:93 +0x2c fp=0xc82002ef88 sp=0xc82002eed8
5/4/2017 12:53:02 PMruntime.goexit()
5/4/2017 12:53:02 PM /usr/local/go/src/runtime/asm_amd64.s:1998 +0x1 fp=0xc82002ef90 sp=0xc82002ef88
5/4/2017 12:53:02 PMcreated by internal/singleflight.(*Group).DoChan
5/4/2017 12:53:02 PM /usr/local/go/src/internal/singleflight/singleflight.go:86 +0x3ee
5/4/2017 12:53:02 PM
5/4/2017 12:53:02 PMgoroutine 1 [select, locked to thread]:
5/4/2017 12:53:02 PMnet/http.(*Transport).getConn(0xc8200bc000, 0xc8204e20e0, 0x0, 0xc8200e8500, 0x5, 0xc8204c12f0, 0x26, 0x0, 0x0, 0x0)
5/4/2017 12:53:02 PM /usr/local/go/src/net/http/transport.go:711 +0x4ef
5/4/2017 12:53:02 PMnet/http.(*Transport).RoundTrip(0xc8200bc000, 0xc8204e20e0, 0xc8200bc000, 0x0, 0x0)
5/4/2017 12:53:02 PM /usr/local/go/src/net/http/transport.go:311 +0x7e9
5/4/2017 12:53:02 PMnet/http.send(0xc8204e2000, 0x7fa4e3c98578, 0xc8200bc000, 0xed09d7da8, 0x38464e28, 0x3668ba0, 0xc8200e85c0, 0x0, 0x0)
5/4/2017 12:53:02 PM /usr/local/go/src/net/http/client.go:260 +0x6b7
5/4/2017 12:53:02 PMnet/http.(*Client).send(0xc8204c11a0, 0xc8204e2000, 0xed09d7da8, 0x38464e28, 0x3668ba0, 0xc82022b790, 0x0, 0x0)
5/4/2017 12:53:02 PM /usr/local/go/src/net/http/client.go:155 +0x185
5/4/2017 12:53:02 PMnet/http.(*Client).doFollowingRedirects(0xc8204c11a0, 0xc8204e2000, 0x296bd98, 0x0, 0x0, 0x0)
5/4/2017 12:53:02 PM /usr/local/go/src/net/http/client.go:475 +0x8a4
5/4/2017 12:53:02 PMnet/http.(*Client).Do(0xc8204c11a0, 0xc8204e2000, 0x14, 0x0, 0x0)
5/4/2017 12:53:02 PM /usr/local/go/src/net/http/client.go:188 +0xff
5/4/2017 12:53:02 PMgithub.com/rancher/lb-controller/vendor/github.com/rancher/go-rancher/v2.setupRancherBaseClient(0xc8200e66c0, 0xc8200e84c0, 0x0, 0x0)
5/4/2017 12:53:02 PM /go/src/github.com/rancher/lb-controller/vendor/github.com/rancher/go-rancher/v2/common.go:156 +0x2a3
5/4/2017 12:53:02 PMgithub.com/rancher/lb-controller/vendor/github.com/rancher/go-rancher/v2.NewRancherClient(0xc8200e84c0, 0xc82000c092, 0x0, 0x0)
5/4/2017 12:53:02 PM /go/src/github.com/rancher/lb-controller/vendor/github.com/rancher/go-rancher/v2/generated_client.go:319 +0xcb
5/4/2017 12:53:02 PMgithub.com/rancher/lb-controller/provider/rancher.init.1()
5/4/2017 12:53:02 PM /go/src/github.com/rancher/lb-controller/provider/rancher/rancher.go:65 +0x49d
5/4/2017 12:53:02 PMgithub.com/rancher/lb-controller/provider/rancher.init()
5/4/2017 12:53:02 PM /go/src/github.com/rancher/lb-controller/provider/rancher/rancher.go:889 +0x77
5/4/2017 12:53:02 PMmain.init()
5/4/2017 12:53:02 PM /go/src/github.com/rancher/lb-controller/plugins.go:12 +0x8d
5/4/2017 12:53:02 PM
5/4/2017 12:53:02 PMgoroutine 17 [syscall, locked to thread]:
5/4/2017 12:53:02 PMruntime.goexit()
5/4/2017 12:53:02 PM /usr/local/go/src/runtime/asm_amd64.s:1998 +0x1
5/4/2017 12:53:02 PM
5/4/2017 12:53:02 PMgoroutine 5 [chan receive]:
5/4/2017 12:53:02 PMgithub.com/rancher/lb-controller/vendor/github.com/golang/glog.(*loggingT).flushDaemon(0x3668fe0)
5/4/2017 12:53:02 PM /go/src/github.com/rancher/lb-controller/vendor/github.com/golang/glog/glog.go:882 +0x67
5/4/2017 12:53:02 PMcreated by github.com/rancher/lb-controller/vendor/github.com/golang/glog.init.1
5/4/2017 12:53:02 PM /go/src/github.com/rancher/lb-controller/vendor/github.com/golang/glog/glog.go:410 +0x297
5/4/2017 12:53:02 PM
5/4/2017 12:53:02 PMgoroutine 9 [syscall]:
5/4/2017 12:53:02 PMos/signal.signal_recv(0x17)
5/4/2017 12:53:02 PM /usr/local/go/src/runtime/sigqueue.go:116 +0x132
5/4/2017 12:53:02 PMos/signal.loop()
5/4/2017 12:53:02 PM /usr/local/go/src/os/signal/signal_unix.go:22 +0x18
5/4/2017 12:53:02 PMcreated by os/signal.init.1
5/4/2017 12:53:02 PM /usr/local/go/src/os/signal/signal_unix.go:28 +0x37
5/4/2017 12:53:02 PM
5/4/2017 12:53:02 PMgoroutine 19 [runnable]:
5/4/2017 12:53:02 PMgithub.com/rancher/lb-controller/vendor/github.com/patrickmn/go-cache.(*janitor).Run(0xc820498ba0, 0xc8200e8440)
5/4/2017 12:53:02 PM /go/src/github.com/rancher/lb-controller/vendor/github.com/patrickmn/go-cache/cache.go:1034 +0x38
5/4/2017 12:53:02 PMcreated by github.com/rancher/lb-controller/vendor/github.com/patrickmn/go-cache.runJanitor
5/4/2017 12:53:02 PM /go/src/github.com/rancher/lb-controller/vendor/github.com/patrickmn/go-cache/cache.go:1056 +0x81
5/4/2017 12:53:02 PM
5/4/2017 12:53:02 PMgoroutine 20 [chan receive]:
5/4/2017 12:53:02 PMgithub.com/rancher/lb-controller/vendor/github.com/rancher/event-subscriber/locks.locker()
5/4/2017 12:53:02 PM /go/src/github.com/rancher/lb-controller/vendor/github.com/rancher/event-subscriber/locks/locks.go:61 +0xdf
5/4/2017 12:53:02 PMcreated by github.com/rancher/lb-controller/vendor/github.com/rancher/event-subscriber/locks.init.1
5/4/2017 12:53:02 PM /go/src/github.com/rancher/lb-controller/vendor/github.com/rancher/event-subscriber/locks/locks.go:7 +0x59
5/4/2017 12:53:02 PM
5/4/2017 12:53:02 PMgoroutine 21 [select]:
5/4/2017 12:53:02 PMnet/http.setRequestCancel.func4(0x0, 0xc8204c1200, 0xc8200e8600, 0xc8200e5320)
5/4/2017 12:53:02 PM /usr/local/go/src/net/http/client.go:333 +0x12e
5/4/2017 12:53:02 PMcreated by net/http.setRequestCancel
5/4/2017 12:53:02 PM /usr/local/go/src/net/http/client.go:341 +0x2f7
5/4/2017 12:53:02 PM
5/4/2017 12:53:02 PMgoroutine 22 [select]:
5/4/2017 12:53:02 PMnet.lookupIPDeadline(0xc8204c12f0, 0x22, 0xed09d7dbc, 0x38570061, 0x3668ba0, 0x0, 0x0, 0x0, 0x0, 0x0)
5/4/2017 12:53:02 PM /usr/local/go/src/net/lookup.go:109 +0x6a6
5/4/2017 12:53:02 PMnet.internetAddrList(0x25569d0, 0x3, 0xc8204c12f0, 0x26, 0xed09d7dbc, 0x38570061, 0x3668ba0, 0x0, 0x0, 0x0, ...)
5/4/2017 12:53:02 PM /usr/local/go/src/net/ipsock.go:252 +0x6ee
5/4/2017 12:53:02 PMnet.resolveAddrList(0x2551dc0, 0x4, 0x25569d0, 0x3, 0xc8204c12f0, 0x26, 0xed09d7dbc, 0x38570061, 0x3668ba0, 0x0, ...)
5/4/2017 12:53:02 PM /usr/local/go/src/net/dial.go:158 +0x466
5/4/2017 12:53:02 PMnet.(*Dialer).Dial(0xc82000a690, 0x25569d0, 0x3, 0xc8204c12f0, 0x26, 0x0, 0x0, 0x0, 0x0)
5/4/2017 12:53:02 PM /usr/local/go/src/net/dial.go:216 +0x124
5/4/2017 12:53:02 PMnet.(*Dialer).Dial-fm(0x25569d0, 0x3, 0xc8204c12f0, 0x26, 0x0, 0x0, 0x0, 0x0)
5/4/2017 12:53:02 PM /usr/local/go/src/net/http/transport.go:38 +0x6e
5/4/2017 12:53:02 PMnet/http.(*Transport).dial(0xc8200bc000, 0x25569d0, 0x3, 0xc8204c12f0, 0x26, 0x0, 0x0, 0x0, 0x0)
5/4/2017 12:53:02 PM /usr/local/go/src/net/http/transport.go:659 +0x96
5/4/2017 12:53:02 PMnet/http.(*Transport).dialConn(0xc8200bc000, 0x0, 0xc8200e8500, 0x5, 0xc8204c12f0, 0x26, 0x0, 0x0, 0x0)
5/4/2017 12:53:02 PM /usr/local/go/src/net/http/transport.go:762 +0x1e3d
5/4/2017 12:53:02 PMnet/http.(*Transport).getConn.func4(0xc8200bc000, 0x0, 0xc8200e8500, 0x5, 0xc8204c12f0, 0x26, 0xc8200e53e0)
5/4/2017 12:53:02 PM /usr/local/go/src/net/http/transport.go:706 +0x66
5/4/2017 12:53:02 PMcreated by net/http.(*Transport).getConn
5/4/2017 12:53:02 PM /usr/local/go/src/net/http/transport.go:708 +0x262
Has anyone come across this beforfe?