Rancher versions:
rancher/server: 1.6.12
rancher/agent: 1.2.7
Infrastructure Stack versions:
healthcheck: 0.3.3
ipsec:0.13.5
network-services:0.7.18
scheduler:0.8.2
Docker version: (docker version,docker info preferred) 17.09.1-ce
Operating system and kernel: (cat /etc/os-release, uname -r preferred)
uname -r : 4.11.0-1016-azure
debian_version : stretch/sid
Type/provider of hosts: (VirtualBox/Bare-metal/AWS/GCE/DO) Azure
Setup details: (single node rancher vs. HA rancher, internal DB vs. external DB) Rancher HA with External DB
Environment Template: (Cattle/Kubernetes/Swarm/Mesos) Cattle
Steps to Reproduce:
I have an application developed in Scala with Akka. The purpose of the app is to call external services and SQL databases (using JDBC), do some processing, and return a parsed result, on a recurrent basis. The app uses akka cluster so that it can scale horizontally.
The application is dockerized and when we deployed it manually (with a simple docker run), the app works like a charm and processed ~2500 calls/second.
But when we deployed it with Rancher, the app doesn’t work, the SourceScheduler stop working and doesn’t tick evvery second. Also, the app uses more CPU resources, and way more RAM (1.3GB in production vs ~350MB on my machine)
The production app runs in a JRE-8 alpine-based Docker container.
Results:
I have done a docker inspect on a container without rancher and one with rancher .
Without Rancher :
[
{
"Id": "6d0d3615370c3fdef717ad5d94fea543592aa130518262ba05feef4915678121",
"Created": "2017-12-15T09:40:15.170813277Z",
"Path": "bin/scala",
"Args": [],
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 60506,
"ExitCode": 0,
"Error": "",
"StartedAt": "2017-12-15T09:40:15.623127566Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Image": "sha256:c5a7ec8306c210366c4248dcfccef3cdd738d67f4923e6215507a37549a5abcc",
"ResolvConfPath": "/var/lib/docker/containers/6d0d3615370c3fdef717ad5d94fea543592aa130518262ba05feef4915678121/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/6d0d3615370c3fdef717ad5d94fea543592aa130518262ba05feef4915678121/hostname",
"HostsPath": "/var/lib/docker/containers/6d0d3615370c3fdef717ad5d94fea543592aa130518262ba05feef4915678121/hosts",
"LogPath": "/var/lib/docker/containers/6d0d3615370c3fdef717ad5d94fea543592aa130518262ba05feef4915678121/6d0d3615370c3fdef717ad5d94fea543592aa130518262ba05feef4915678121-json.log",
"Name": "/stoic_noether",
"RestartCount": 0,
"Driver": "overlay2",
"Platform": "linux",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "docker-default",
"ExecIDs": null,
"HostConfig": {
"Binds": null,
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {}
},
"NetworkMode": "default",
"PortBindings": {},
"RestartPolicy": {
"Name": "no",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"CapAdd": null,
"CapDrop": null,
"Dns": [],
"DnsOptions": [],
"DnsSearch": [],
"ExtraHosts": null,
"GroupAdd": null,
"IpcMode": "shareable",
"Cgroup": "",
"Links": null,
"OomScoreAdj": 0,
"PidMode": "",
"Privileged": false,
"PublishAllPorts": false,
"ReadonlyRootfs": false,
"SecurityOpt": null,
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 67108864,
"Runtime": "runc",
"ConsoleSize": [
0,
0
],
"Isolation": "",
"CpuShares": 0,
"Memory": 0,
"NanoCpus": 0,
"CgroupParent": "",
"BlkioWeight": 0,
"BlkioWeightDevice": [],
"BlkioDeviceReadBps": null,
"BlkioDeviceWriteBps": null,
"BlkioDeviceReadIOps": null,
"BlkioDeviceWriteIOps": null,
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": [],
"DeviceCgroupRules": null,
"DiskQuota": 0,
"KernelMemory": 0,
"MemoryReservation": 0,
"MemorySwap": 0,
"MemorySwappiness": null,
"OomKillDisable": false,
"PidsLimit": 0,
"Ulimits": null,
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0
},
"GraphDriver": {
"Data": {
"LowerDir": "/var/lib/docker/overlay2/4eee804f8c85aedaa3ac168d4d0adbc61a2cc80968718b3a08e01a9dfdd5c361-init/diff:/var/lib/docker/overlay2/223c2ba1f682dab8106da36c433b65f9f2bba54aafd65b478f703ce4197a44cc/diff:/var/lib/docker/overlay2/7c940fe22ad24ff19b89f37a7341657827ef241b612e022d654dc8abd86544af/diff:/var/lib/docker/overlay2/6a1c5ef0b6cae39e416e14124195f9930816b93ad080517895b051392fa73620/diff:/var/lib/docker/overlay2/6ce76d7177465ec613d696eccf0e2de7af5e5508a19f4ef36ffc473264f3f10d/diff:/var/lib/docker/overlay2/0e1213486133a489b7e823badbefb522356b320f454a784c0a1a02c2ed6e398d/diff:/var/lib/docker/overlay2/991da0ff1a0e6489544a47d7ca49d9dc6a721d0ae3fd181155043d98603cbe44/diff",
"MergedDir": "/var/lib/docker/overlay2/4eee804f8c85aedaa3ac168d4d0adbc61a2cc80968718b3a08e01a9dfdd5c361/merged",
"UpperDir": "/var/lib/docker/overlay2/4eee804f8c85aedaa3ac168d4d0adbc61a2cc80968718b3a08e01a9dfdd5c361/diff",
"WorkDir": "/var/lib/docker/overlay2/4eee804f8c85aedaa3ac168d4d0adbc61a2cc80968718b3a08e01a9dfdd5c361/work"
},
"Name": "overlay2"
},
"Mounts": [],
"Config": {
"Hostname": "6d0d3615370c",
"Domainname": "",
"User": "daemon",
"AttachStdin": false,
"AttachStdout": true,
"AttachStderr": true,
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/jvm/java-1.8-openjdk/jre/bin:/usr/lib/jvm/java-1.8-openjdk/bin",
"LANG=C.UTF-8",
"JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk/jre",
"JAVA_VERSION=8u151",
"JAVA_ALPINE_VERSION=8.151.12-r0"
],
"Cmd": [],
"ArgsEscaped": true,
"Image": "scala",
"Volumes": null,
"WorkingDir": "/opt/docker",
"Entrypoint": [
"bin/scala"
],
"OnBuild": null,
"Labels": {}
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "92821545d4e8122930222df5ec7e60744f3bf54d1fa4839371f2e0fd1e238a2f",
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"Ports": {},
"SandboxKey": "/var/run/docker/netns/92821545d4e8",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "113cf8351e53f5dd8ebec282c2d7de2eb06dd2a8707bbc992b173963ed87e246",
"Gateway": "172.17.0.1",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "172.17.0.3",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"MacAddress": "02:42:ac:11:00:03",
"Networks": {
"bridge": {
"IPAMConfig": null,
"Links": null,
"Aliases": null,
"NetworkID": "6fdb5635b4fb06e9c61c5b63c36427288b7ecf2b2cbe289e05d87d6a294f0f94",
"EndpointID": "113cf8351e53f5dd8ebec282c2d7de2eb06dd2a8707bbc992b173963ed87e246",
"Gateway": "172.17.0.1",
"IPAddress": "172.17.0.3",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"MacAddress": "02:42:ac:11:00:03",
"DriverOpts": null
}
}
}
}
]
With Rancher :
[
{
"Id": "6f0855bc50fa89e05b32ddfdf2edcf9e9f244e87dda8c9bc64e9950201caa168",
"Created": "2017-12-14T17:15:35.558131739Z",
"Path": "/.r/r",
"Args": [
"bin/scala"
],
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 17816,
"ExitCode": 0,
"Error": "",
"StartedAt": "2017-12-15T09:47:08.705863445Z",
"FinishedAt": "2017-12-15T09:38:43.38370028Z"
},
"Image": "sha256:2a2cc6d07ddea7e48435773ed0236a396ff1a3171aa15498de0b82786d0cf239",
"ResolvConfPath": "/var/lib/docker/containers/6f0855bc50fa89e05b32ddfdf2edcf9e9f244e87dda8c9bc64e9950201caa168/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/6f0855bc50fa89e05b32ddfdf2edcf9e9f244e87dda8c9bc64e9950201caa168/hostname",
"HostsPath": "/etc/hosts",
"LogPath": "/var/lib/docker/containers/6f0855bc50fa89e05b32ddfdf2edcf9e9f244e87dda8c9bc64e9950201caa168/6f0855bc50fa89e05b32ddfdf2edcf9e9f244e87dda8c9bc64e9950201caa168-json.log",
"Name": "/r-pingviewv3-flexo-master-1-07414ab9",
"RestartCount": 0,
"Driver": "overlay2",
"Platform": "linux",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "docker-default",
"ExecIDs": null,
"HostConfig": {
"Binds": [
"/etc/hosts:/etc/hosts:rw",
"rancher-cni:/.r:ro"
],
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {}
},
"NetworkMode": "none",
"PortBindings": {},
"RestartPolicy": {
"Name": "",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"CapAdd": [],
"CapDrop": [],
"Dns": [
"169.254.169.250"
],
"DnsOptions": null,
"DnsSearch": [
"stack.rancher.internal",
"scala-master.stack.rancher.internal",
"rancher.internal"
],
"ExtraHosts": null,
"GroupAdd": null,
"IpcMode": "shareable",
"Cgroup": "",
"Links": null,
"OomScoreAdj": 0,
"PidMode": "",
"Privileged": false,
"PublishAllPorts": false,
"ReadonlyRootfs": false,
"SecurityOpt": null,
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 67108864,
"Runtime": "runc",
"ConsoleSize": [
0,
0
],
"Isolation": "",
"CpuShares": 2,
"Memory": 0,
"NanoCpus": 0,
"CgroupParent": "",
"BlkioWeight": 0,
"BlkioWeightDevice": null,
"BlkioDeviceReadBps": null,
"BlkioDeviceWriteBps": null,
"BlkioDeviceReadIOps": null,
"BlkioDeviceWriteIOps": null,
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": [],
"DeviceCgroupRules": null,
"DiskQuota": 0,
"KernelMemory": 0,
"MemoryReservation": 0,
"MemorySwap": 0,
"MemorySwappiness": null,
"OomKillDisable": false,
"PidsLimit": 0,
"Ulimits": null,
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0,
"Init": false
},
"GraphDriver": {
"Data": {
"LowerDir": "/var/lib/docker/overlay2/c46c1c31edee775b137c940a5b901a62b6e99ce5ae4de06da5162fd2cb882dea-init/diff:/var/lib/docker/overlay2/9b072cfbb8ff0dbce01e6139f9ff285b1e27d5df8cf8b414d2b195bfa7ced15a/diff:/var/lib/docker/overlay2/f22665cd4ae9b6589b879c1d312815431b7bc9207e3a99e625a1f484beef269e/diff:/var/lib/docker/overlay2/cd56becbad4449e49248fdde20eae8c465bb213c28b86629cadbd2558a6239ae/diff:/var/lib/docker/overlay2/6ce76d7177465ec613d696eccf0e2de7af5e5508a19f4ef36ffc473264f3f10d/diff:/var/lib/docker/overlay2/0e1213486133a489b7e823badbefb522356b320f454a784c0a1a02c2ed6e398d/diff:/var/lib/docker/overlay2/991da0ff1a0e6489544a47d7ca49d9dc6a721d0ae3fd181155043d98603cbe44/diff",
"MergedDir": "/var/lib/docker/overlay2/c46c1c31edee775b137c940a5b901a62b6e99ce5ae4de06da5162fd2cb882dea/merged",
"UpperDir": "/var/lib/docker/overlay2/c46c1c31edee775b137c940a5b901a62b6e99ce5ae4de06da5162fd2cb882dea/diff",
"WorkDir": "/var/lib/docker/overlay2/c46c1c31edee775b137c940a5b901a62b6e99ce5ae4de06da5162fd2cb882dea/work"
},
"Name": "overlay2"
},
"Mounts": [
{
"Type": "volume",
"Name": "rancher-cni",
"Source": "/var/lib/docker/volumes/rancher-cni/_data",
"Destination": "/.r",
"Driver": "local",
"Mode": "ro",
"RW": false,
"Propagation": ""
},
{
"Type": "bind",
"Source": "/etc/hosts",
"Destination": "/etc/hosts",
"Mode": "rw",
"RW": true,
"Propagation": "rprivate"
}
],
"Config": {
"Hostname": "6f0855bc50fa",
"Domainname": "",
"User": "daemon",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/jvm/java-1.8-openjdk/jre/bin:/usr/lib/jvm/java-1.8-openjdk/bin",
"LANG=C.UTF-8",
"JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk/jre",
"JAVA_VERSION=8u151",
"JAVA_ALPINE_VERSION=8.151.12-r0"
],
"Cmd": null,
"Healthcheck": {},
"Image": "scala:develop",
"Volumes": {
"/etc/hosts": {}
},
"WorkingDir": "/opt/docker",
"Entrypoint": [
"/.r/r",
"bin/scala"
],
"OnBuild": null,
"Labels": {
[...]
}
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "569abc082b9a8737526399725c9745cb3be682e9e1c02dc44f89008d41eddc32",
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"Ports": {},
"SandboxKey": "/var/run/docker/netns/569abc082b9a",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "",
"Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "",
"IPPrefixLen": 0,
"IPv6Gateway": "",
"MacAddress": "",
"Networks": {
"none": {
"IPAMConfig": null,
"Links": null,
"Aliases": null,
"NetworkID": "eb9aab35225dc7863c85b7004383e919fe668376f0e552052695502c63822828",
"EndpointID": "bbf5b592bb18c1194763b0d8f31c0f3f7a550564c927ae172a74300e33a0114c",
"Gateway": "",
"IPAddress": "",
"IPPrefixLen": 0,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"MacAddress": "",
"DriverOpts": null
}
}
}
}
]
I have tried to deploy the container with rancher with the network mode : bridge but with the same result as managed.
I supposed Rancher modified the container with some configuration but I don’t know which modification is in cause.