Hi,
We are running rancher 1.1.4 on CentOS Linux release 7.3.1611 (Core).
Docker version: Docker version 1.13.0, build 49bf474
When we try to execute shell,view logs or see stats it simply disconnects.
I have read that upgrade should solve this problems.
I followed the upgrade procedure from https://docs.rancher.com/rancher/v1.3/en/upgrading/#single-container-external-database
After rancher server was upgraded i clicked upgrade in the UI. After successful upgrade i see that hosts are in “reconnecting” state and docker agent is full of errors:
INFO: Starting agent for xxxxxxxxxxxxxxxxxxxxxxxx
INFO: Access Key: xxxxxxxxxxxxxxxxxxxxxxxx
INFO: Config URL: http://mzup-rancher.xxxxx.local:8080/v1
INFO: Storage URL: http://mzup-rancher.xxxxx.local:8080/v1
INFO: API URL: http://mzup-rancher.xxxxx.local:8080/v1
INFO: IP: 192.168.6.63
INFO: Port:
INFO: Required Image: rancher/agent:v1.1.3
INFO: Current Image: rancher/agent:v1.1.3
INFO: Using image rancher/agent:v1.1.3
INFO: Downloading agent http://mzup-rancher.xxxxx.local:8080/v1/configcontent/configscripts
INFO: Updating configscripts
INFO: Downloading http://mzup-rancher.xxxxx.local:8080/v1//configcontent//configscripts current=
INFO: Running /var/lib/cattle/download/configscripts/configscripts-1-f0f3fb2e1110b5ada7c441705981f93a480313a324294321cff467f0c3e12319/apply.sh
INFO: Sending configscripts applied 1-f0f3fb2e1110b5ada7c441705981f93a480313a324294321cff467f0c3e12319
INFO: Updating pyagent
INFO: Downloading http://mzup-rancher.xxxxx.local:8080/v1//configcontent//pyagent current=
INFO: Running /var/lib/cattle/download/pyagent/pyagent-1-9e3dcab12585149a65d818082279708965cd826df1937a55b24f48428f8f0c0644fc1033b5be15c88dcf9540b1c04c67/apply.sh
INFO: Updating host-api
INFO: Downloading http://mzup-rancher.xxxxx.local:8080/v1//configcontent//host-api current=1-9608ead804f8cebbfe95b9f114701d8a5ac4244d603d196068043c902b1a0bdf
INFO: Already up to date
INFO: Sending host-api applied 1-9608ead804f8cebbfe95b9f114701d8a5ac4244d603d196068043c902b1a0bdf
INFO: Updating python-agent
INFO: Downloading http://mzup-rancher.xxxxx.local:8080/v1//configcontent//python-agent current=1-8f7ddbf845f3f27063a382644792cfa8bcbac969d38627eeafcdfb452d498458
INFO: Already up to date
INFO: Sending python-agent applied 1-8f7ddbf845f3f27063a382644792cfa8bcbac969d38627eeafcdfb452d498458
INFO: Sending pyagent applied 1-9e3dcab12585149a65d818082279708965cd826df1937a55b24f48428f8f0c0644fc1033b5be15c88dcf9540b1c04c67
INFO: Starting agent /var/lib/cattle/pyagent/apply.sh
INFO: Executing /var/lib/cattle/pyagent/agent
INFO: Updating host-config
INFO: Downloading http://mzup-rancher.xxxxx.local:8080/v1//configcontent//host-config current=host-config-1-33fa6999d14b23745fe1d3408bdfd23d13da6dca2819ca225c7b77ace8516267
INFO: Already up to date
INFO: Sending host-config applied 1-33fa6999d14b23745fe1d3408bdfd23d13da6dca2819ca225c7b77ace8516267
time="2017-02-01T11:37:23Z" level=info msg="Launching agent"
time="2017-02-01T11:37:23Z" level=info msg="http://mzup-rancher.xxxxx.local:8080/v176CE0177FB8A06EB8B29Lwp4XYmsJjCd3XRmUyrhMShxGoCV72MnvwMLfQgB"
time="2017-02-01T11:37:23Z" level=info msg="Listening for events on http://mzup-rancher.xxxxx.local:8080/v1"
time="2017-02-01T11:37:23Z" level=info msg="launching hostapi"
fatal error: unexpected signal during runtime execution
[signal SIGSEGV: segmentation violation code=0x1 addr=0x63 pc=0x7f1719ea5259]
runtime stack:
runtime.throw(0x9cf035, 0x2a)
/usr/local/go/src/runtime/panic.go:566 +0x95
runtime.sigpanic()
/usr/local/go/src/runtime/sigpanic_unix.go:12 +0x2cc
goroutine 21 [syscall, locked to thread]:
runtime.cgocall(0x7a21c0, 0xc420023df8, 0xc400000000)
/usr/local/go/src/runtime/cgocall.go:131 +0x110 fp=0xc420023db0 sp=0xc420023d70
net._C2func_getaddrinfo(0x7f170c0008c0, 0x0, 0xc4200be810, 0xc4202ba4b8, 0x0, 0x0, 0x0)
??:0 +0x68 fp=0xc420023df8 sp=0xc420023db0
net.cgoLookupIPCNAME(0xc420010112, 0x19, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/cgo_unix.go:146 +0x37c fp=0xc420023f18 sp=0xc420023df8
net.cgoIPLookup(0xc42005c720, 0xc420010112, 0x19)
/usr/local/go/src/net/cgo_unix.go:198 +0x4d fp=0xc420023fa8 sp=0xc420023f18
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc420023fb0 sp=0xc420023fa8
created by net.cgoLookupIP
/usr/local/go/src/net/cgo_unix.go:208 +0xb4
goroutine 1 [select]:
net/http.(*persistConn).roundTrip(0xc42000bc00, 0xc4202b4020, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:1840 +0x93b
net/http.(*Transport).RoundTrip(0xc4200fc960, 0xc4200fcc30, 0xc4200fc960, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:380 +0x4ee
net/http.send(0xc4200fcc30, 0xe235c0, 0xc4200fc960, 0x0, 0x0, 0x0, 0x8, 0xc42019faf8, 0xc420024588)
/usr/local/go/src/net/http/client.go:256 +0x15f
net/http.(*Client).send(0xc4202ac450, 0xc4200fcc30, 0x0, 0x0, 0x0, 0xc420024588, 0x0, 0x1)
/usr/local/go/src/net/http/client.go:146 +0x102
net/http.(*Client).doFollowingRedirects(0xc4202ac450, 0xc4200fcc30, 0xa04708, 0x3, 0xc42019fc01, 0x40e99f)
/usr/local/go/src/net/http/client.go:528 +0x5e5
net/http.(*Client).Do(0xc4202ac450, 0xc4200fcc30, 0xc42000c238, 0xc4200fcc30, 0xc42000c238)
/usr/local/go/src/net/http/client.go:184 +0x1ea
github.com/rancher/agent/vendor/golang.org/x/net/context/ctxhttp.Do(0x7f171b623118, 0xc42000c238, 0xc4202ac450, 0xc4200fcb40, 0x1, 0xc4202ac4e0, 0x0)
/go/src/github.com/rancher/agent/vendor/golang.org/x/net/context/ctxhttp/ctxhttp.go:30 +0x92
github.com/rancher/agent/vendor/github.com/docker/docker/client.(*Client).sendClientRequest(0xc42027ca80, 0x7f171b623118, 0xc42000c238, 0x9beb8b, 0x3, 0x9bf1eb, 0x5, 0xc4202ac4e0, 0x0, 0x0, ...)
/go/src/github.com/rancher/agent/vendor/github.com/docker/docker/client/request.go:110 +0x1a9
github.com/rancher/agent/vendor/github.com/docker/docker/client.(*Client).sendRequest(0xc42027ca80, 0x7f171b623118, 0xc42000c238, 0x9beb8b, 0x3, 0x9bf1eb, 0x5, 0xc4202ac4e0, 0x0, 0x0, ...)
/go/src/github.com/rancher/agent/vendor/github.com/docker/docker/client/request.go:78 +0xf1
github.com/rancher/agent/vendor/github.com/docker/docker/client.(*Client).get(0xc42027ca80, 0x7f171b623118, 0xc42000c238, 0x9bf1eb, 0x5, 0xc4202ac4e0, 0x0, 0x0, 0x0, 0x7f171b623118, ...)
/go/src/github.com/rancher/agent/vendor/github.com/docker/docker/client/request.go:36 +0xb5
github.com/rancher/agent/vendor/github.com/docker/docker/client.(*Client).Info(0xc42027ca80, 0x7f171b623118, 0xc42000c238, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/rancher/agent/vendor/github.com/docker/docker/client/info.go:15 +0x120
github.com/rancher/agent/handlers.initializeHandlers(0xc420010240)
/go/src/github.com/rancher/agent/handlers/common.go:104 +0x17f
github.com/rancher/agent/handlers.GetHandlers(0xc400000000)
/go/src/github.com/rancher/agent/handlers/common.go:29 +0x37
github.com/rancher/agent/events.Listen(0xc42001010b, 0x28, 0xc4200140d2, 0x14, 0xc4200100d2, 0x28, 0x32, 0xc4201a1e08, 0xe667f0)
/go/src/github.com/rancher/agent/events/listener.go:37 +0x30e
main.main()
/go/src/github.com/rancher/agent/main.go:55 +0x47c
goroutine 17 [syscall, locked to thread]:
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:2086 +0x1
goroutine 18 [select]:
net/http.setRequestCancel.func4(0x0, 0xc4200be2d0, 0xc42005e200, 0xc42005c240)
/usr/local/go/src/net/http/client.go:329 +0x15a
created by net/http.setRequestCancel
/usr/local/go/src/net/http/client.go:337 +0x29f
goroutine 8 [chan receive]:
github.com/rancher/agent/vendor/github.com/rancher/event-subscriber/locks.locker()
/go/src/github.com/rancher/agent/vendor/github.com/rancher/event-subscriber/locks/locks.go:95 +0xf0
created by github.com/rancher/agent/vendor/github.com/rancher/event-subscriber/locks.init.1
/go/src/github.com/rancher/agent/vendor/github.com/rancher/event-subscriber/locks/locks.go:6 +0x35
goroutine 9 [chan receive]:
github.com/rancher/agent/vendor/github.com/golang/glog.(*loggingT).flushDaemon(0xe43820)
/go/src/github.com/rancher/agent/vendor/github.com/golang/glog/glog.go:879 +0x7a
created by github.com/rancher/agent/vendor/github.com/golang/glog.init.1
/go/src/github.com/rancher/agent/vendor/github.com/golang/glog/glog.go:410 +0x21d
goroutine 10 [select]:
net/http.(*Transport).getConn(0xc4200fc000, 0xc4202ae320, 0x0, 0xc42001010b, 0x4, 0xc420010112, 0x1e, 0x0, 0x0, 0x4)
/usr/local/go/src/net/http/transport.go:890 +0x9d2
net/http.(*Transport).RoundTrip(0xc4200fc000, 0xc4202c41e0, 0xc4200fc000, 0xed023bf7d, 0x33efd64c)
/usr/local/go/src/net/http/transport.go:367 +0x307
net/http.send(0xc4202c40f0, 0xe235c0, 0xc4200fc000, 0xed023bf7d, 0x33efd64c, 0xe435a0, 0x8, 0xc420123238, 0xc4202ba490)
/usr/local/go/src/net/http/client.go:256 +0x15f
net/http.(*Client).send(0xc420123490, 0xc4202c40f0, 0xed023bf7d, 0x33efd64c, 0xe435a0, 0xc4202ba490, 0x0, 0x1)
/usr/local/go/src/net/http/client.go:146 +0x102
net/http.(*Client).doFollowingRedirects(0xc420123490, 0xc4202c40f0, 0xa04708, 0x3, 0x9c2501, 0xd)
/usr/local/go/src/net/http/client.go:528 +0x5e5
net/http.(*Client).Do(0xc420123490, 0xc4202c40f0, 0x14, 0xc4200100d2, 0x28)
/usr/local/go/src/net/http/client.go:184 +0x1ea
github.com/rancher/agent/vendor/github.com/rancher/go-rancher/client.setupRancherBaseClient(0xc4202ae240, 0xc42005e140, 0x0, 0x0)
/go/src/github.com/rancher/agent/vendor/github.com/rancher/go-rancher/client/common.go:143 +0x15e
github.com/rancher/agent/vendor/github.com/rancher/go-rancher/client.NewRancherClient(0xc42005e140, 0xc42005e140, 0xc4202c6000, 0x0)
/go/src/github.com/rancher/agent/vendor/github.com/rancher/go-rancher/client/generated_client.go:307 +0xcd
github.com/rancher/agent/service/hostapi/util.GetRancherClient(0xc4202c6000, 0x0, 0x0)
/go/src/github.com/rancher/agent/service/hostapi/util/util.go:21 +0x14a
github.com/rancher/agent/service/hostapi/events.(*DockerEventsProcessor).Process(0xc420123dc0, 0xa04338, 0x0)
/go/src/github.com/rancher/agent/service/hostapi/events/entry.go:40 +0x73
github.com/rancher/agent/service/hostapi.StartUp()
/go/src/github.com/rancher/agent/service/hostapi/hostapi.go:53 +0x12e
created by github.com/rancher/agent/events.Listen
/go/src/github.com/rancher/agent/events/listener.go:24 +0x2f1
goroutine 11 [sleep]:
time.Sleep(0x77359400)
/usr/local/go/src/runtime/time.go:59 +0xe1
github.com/rancher/agent/events.Listen.func1()
/go/src/github.com/rancher/agent/events/listener.go:33 +0x67
created by github.com/rancher/agent/events.Listen
/go/src/github.com/rancher/agent/events/listener.go:35 +0x309
goroutine 13 [IO wait]:
net.runtime_pollWait(0x7f171b6240c0, 0x72, 0x3)
/usr/local/go/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc42027cca0, 0x72, 0xc4200359d0, 0xc42000c048)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc42027cca0, 0xe24dc0, 0xc42000c048)
/usr/local/go/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc42027cc40, 0xc4202d0000, 0x1000, 0x1000, 0x0, 0xe24dc0, 0xc42000c048)
/usr/local/go/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420024590, 0xc4202d0000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:173 +0x70
net/http.(*persistConn).Read(0xc42000bc00, 0xc4202d0000, 0x1000, 0x1000, 0x64d493, 0xc42001cb78, 0xc42001cb88)
/usr/local/go/src/net/http/transport.go:1261 +0x154
bufio.(*Reader).fill(0xc4201e2d80)
/usr/local/go/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Peek(0xc4201e2d80, 0x1, 0xc4201e2f00, 0xc42001cc60, 0x0, 0x0, 0x0)
/usr/local/go/src/bufio/bufio.go:129 +0x62
net/http.(*persistConn).readLoop(0xc42000bc00)
/usr/local/go/src/net/http/transport.go:1418 +0x1a1
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1062 +0x4e9
goroutine 14 [select]:
net/http.(*persistConn).writeLoop(0xc42000bc00)
/usr/local/go/src/net/http/transport.go:1646 +0x3bd
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1063 +0x50e
goroutine 19 [select]:
net.lookupIPContext(0xe27bc0, 0xc42005c660, 0xc420010112, 0x19, 0x0, 0x0, 0x0, 0x0, 0x524fe2410f4)
/usr/local/go/src/net/lookup.go:122 +0x7bc
net.internetAddrList(0xe27bc0, 0xc42005c660, 0x9becbd, 0x3, 0xc420010112, 0x1e, 0x0, 0x0, 0x0, 0xed023bf73, ...)
/usr/local/go/src/net/ipsock.go:241 +0x5e0
net.resolveAddrList(0xe27bc0, 0xc42005c660, 0x9bef84, 0x4, 0x9becbd, 0x3, 0xc420010112, 0x1e, 0x0, 0x0, ...)
/usr/local/go/src/net/dial.go:179 +0x106
net.(*Dialer).DialContext(0xc420052910, 0xe27b80, 0xc42000c238, 0x9becbd, 0x3, 0xc420010112, 0x1e, 0x0, 0x0, 0x0, ...)
/usr/local/go/src/net/dial.go:329 +0x238
net.(*Dialer).DialContext-fm(0xe27b80, 0xc42000c238, 0x9becbd, 0x3, 0xc420010112, 0x1e, 0xc4202b6201, 0xc42005c5a0, 0xc420030a38, 0x411238)
/usr/local/go/src/net/http/transport.go:43 +0x73
net/http.(*Transport).dial(0xc4200fc000, 0xe27b80, 0xc42000c238, 0x9becbd, 0x3, 0xc420010112, 0x1e, 0x0, 0x0, 0x0, ...)
/usr/local/go/src/net/http/transport.go:826 +0x227
net/http.(*Transport).dialConn(0xc4200fc000, 0xe27b80, 0xc42000c238, 0x0, 0xc42001010b, 0x4, 0xc420010112, 0x1e, 0x0, 0x0, ...)
/usr/local/go/src/net/http/transport.go:967 +0x1a85
net/http.(*Transport).getConn.func4(0xc4200fc000, 0xe27b80, 0xc42000c238, 0xc4200be390, 0xc42005c300)
/usr/local/go/src/net/http/transport.go:885 +0x78
created by net/http.(*Transport).getConn
/usr/local/go/src/net/http/transport.go:887 +0x398
goroutine 20 [select]:
net.cgoLookupIP(0xe27bc0, 0xc42005c660, 0xc420010112, 0x19, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/cgo_unix.go:209 +0x2f5
net.lookupIP(0xe27bc0, 0xc42005c660, 0xc420010112, 0x19, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/lookup_unix.go:70 +0xf9
net.glob..func11(0xe27bc0, 0xc42005c660, 0xa04588, 0xc420010112, 0x19, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/hook.go:19 +0x52
net.lookupIPContext.func1(0xc420023f88, 0x653478, 0xc4200fc000, 0xe27b80)
/usr/local/go/src/net/lookup.go:119 +0x5c
internal/singleflight.(*Group).doCall(0xe42640, 0xc4202b02d0, 0xc420010112, 0x19, 0xc4200be4b0)
/usr/local/go/src/internal/singleflight/singleflight.go:93 +0x3c
created by internal/singleflight.(*Group).DoChan
/usr/local/go/src/internal/singleflight/singleflight.go:86 +0x339
kind regards,
Milos Zupancic