I’m having another issue with the same stack as posted above. The application is only reachable if the LB proxy is on the same host as the service (webapp).
The HAProxy seems to be setup correctly and the LB is listening on port 80:
root@32125630f2d4:/# cat /etc/haproxy/haproxy.cfg
global
log 127.0.0.1 local0
log 127.0.0.1 local1 notice
maxconn 4096
maxpipes 1024
chroot /var/lib/haproxy
user haproxy
group haproxy
daemon
defaults
log global
mode tcp
option tcplog
option dontlognull
option redispatch
option forwardfor
retries 3
timeout connect 5000
timeout client 50000
timeout server 50000
errorfile 400 /etc/haproxy/errors/400.http
errorfile 403 /etc/haproxy/errors/403.http
errorfile 408 /etc/haproxy/errors/408.http
errorfile 500 /etc/haproxy/errors/500.http
errorfile 502 /etc/haproxy/errors/502.http
errorfile 503 /etc/haproxy/errors/503.http
errorfile 504 /etc/haproxy/errors/504.http
frontend b2405680-6fb7-418c-996b-819e6b88b926_frontend
bind 10.42.157.104:80
mode http
default_backend b2405680-6fb7-418c-996b-819e6b88b926_0_backend
backend b2405680-6fb7-418c-996b-819e6b88b926_0_backend
mode http
balance roundrobin
server e89c505b-fd79-47c4-a1e5-bb15dd7f117a 10.42.198.58:6543
IP tables save:
ubuntu@ip-172-20-0-67:~$ sudo iptables-save
# Generated by iptables-save v1.4.21 on Mon Oct 5 12:57:48 2015
*mangle
:PREROUTING ACCEPT [374102:639779789]
:INPUT ACCEPT [309133:546665524]
:FORWARD ACCEPT [64969:93114265]
:OUTPUT ACCEPT [180593:40406938]
:POSTROUTING ACCEPT [245562:133521203]
COMMIT
# Completed on Mon Oct 5 12:57:48 2015
# Generated by iptables-save v1.4.21 on Mon Oct 5 12:57:48 2015
*nat
:PREROUTING ACCEPT [54:3968]
:INPUT ACCEPT [1:84]
:OUTPUT ACCEPT [22:1651]
:POSTROUTING ACCEPT [74:4771]
:CATTLE_POSTROUTING - [0:0]
:CATTLE_PREROUTING - [0:0]
:DOCKER - [0:0]
-A PREROUTING -j CATTLE_PREROUTING
-A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER
-A OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j DOCKER
-A POSTROUTING -j CATTLE_POSTROUTING
-A POSTROUTING -s 172.17.0.0/16 ! -o docker0 -j MASQUERADE
-A POSTROUTING -s 172.17.0.4/32 -d 172.17.0.4/32 -p udp -m udp --dport 4500 -j MASQUERADE
-A POSTROUTING -s 172.17.0.4/32 -d 172.17.0.4/32 -p udp -m udp --dport 500 -j MASQUERADE
-A CATTLE_POSTROUTING -s 10.42.0.0/16 -d 169.254.169.250/32 -j ACCEPT
-A CATTLE_POSTROUTING -s 10.42.0.0/16 ! -d 10.42.0.0/16 -p tcp -j MASQUERADE --to-ports 1024-65535
-A CATTLE_POSTROUTING -s 10.42.0.0/16 ! -d 10.42.0.0/16 -p udp -j MASQUERADE --to-ports 1024-65535
-A CATTLE_POSTROUTING -s 10.42.0.0/16 ! -d 10.42.0.0/16 -j MASQUERADE
-A CATTLE_POSTROUTING -s 172.17.0.0/16 ! -o docker0 -p tcp -j MASQUERADE --to-ports 1024-65535
-A CATTLE_POSTROUTING -s 172.17.0.0/16 ! -o docker0 -p udp -j MASQUERADE --to-ports 1024-65535
-A CATTLE_PREROUTING -p udp -m addrtype --dst-type LOCAL -m udp --dport 4500 -j DNAT --to-destination 10.42.177.254:4500
-A CATTLE_PREROUTING -p udp -m addrtype --dst-type LOCAL -m udp --dport 500 -j DNAT --to-destination 10.42.177.254:500
-A CATTLE_PREROUTING -p tcp -m addrtype --dst-type LOCAL -m tcp --dport 80 -j DNAT --to-destination 10.42.157.104:80
-A DOCKER ! -i docker0 -p udp -m udp --dport 4500 -j DNAT --to-destination 172.17.0.4:4500
-A DOCKER ! -i docker0 -p udp -m udp --dport 500 -j DNAT --to-destination 172.17.0.4:500
COMMIT
# Completed on Mon Oct 5 12:57:48 2015
# Generated by iptables-save v1.4.21 on Mon Oct 5 12:57:48 2015
*filter
:INPUT ACCEPT [309142:546671377]
:FORWARD ACCEPT [12:720]
:OUTPUT ACCEPT [180615:40418887]
:DOCKER - [0:0]
-A FORWARD -o docker0 -j DOCKER
-A FORWARD -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -i docker0 ! -o docker0 -j ACCEPT
-A FORWARD -i docker0 -o docker0 -j ACCEPT
-A DOCKER -d 172.17.0.4/32 ! -i docker0 -o docker0 -p udp -m udp --dport 4500 -j ACCEPT
-A DOCKER -d 172.17.0.4/32 ! -i docker0 -o docker0 -p udp -m udp --dport 500 -j ACCEPT
COMMIT
# Completed on Mon Oct 5 12:57:48 2015
And the container running the service (webapp) on the other host (docker inspect):
[
{
"Id": "98eff28178730175b29f18804de302f43f3022b97f80a0c390a96941b50d4c20",
"Created": "2015-10-05T12:33:31.991420993Z",
"Path": "/bin/sh",
"Args": [
"-c",
"./docker/start.sh"
],
"State": {
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 24047,
"ExitCode": 0,
"Error": "",
"StartedAt": "2015-10-05T12:33:32.140555869Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Image": "75793386b8db4ba30202450b2a579c5098185db2edb5d1f73bf0d3596ccd41a4",
"NetworkSettings": {
"Bridge": "",
"EndpointID": "fe649512bc7b6940934552fb4b848c8c8ec9814339fb7cc87fbb09557fed12dc",
"Gateway": "172.17.42.1",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"HairpinMode": false,
"IPAddress": "172.17.0.10",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"MacAddress": "02:7f:75:e9:50:aa",
"NetworkID": "199bf3d928ffae11ed77e9cf42c6a429fbdf31e613e7485a918e8b57063730ca",
"PortMapping": null,
"Ports": {
"6543/tcp": null
},
"SandboxKey": "/var/run/docker/netns/98eff2817873",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null
},
"ResolvConfPath": "/var/lib/docker/containers/98eff28178730175b29f18804de302f43f3022b97f80a0c390a96941b50d4c20/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/98eff28178730175b29f18804de302f43f3022b97f80a0c390a96941b50d4c20/hostname",
"HostsPath": "/var/lib/docker/containers/98eff28178730175b29f18804de302f43f3022b97f80a0c390a96941b50d4c20/hosts",
"LogPath": "/var/lib/docker/containers/98eff28178730175b29f18804de302f43f3022b97f80a0c390a96941b50d4c20/98eff28178730175b29f18804de302f43f3022b97f80a0c390a96941b50d4c20-json.log",
"Name": "/09d02248-7380-4034-be6c-3cf57527d207",
"RestartCount": 0,
"Driver": "aufs",
"ExecDriver": "native-0.2",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "",
"ExecIDs": null,
"HostConfig": {
"Binds": null,
"ContainerIDFile": "",
"LxcConf": null,
"Memory": 0,
"MemorySwap": 0,
"CpuShares": 0,
"CpuPeriod": 0,
"CpusetCpus": "",
"CpusetMems": "",
"CpuQuota": 0,
"BlkioWeight": 0,
"OomKillDisable": false,
"MemorySwappiness": null,
"Privileged": false,
"PortBindings": null,
"Links": null,
"PublishAllPorts": false,
"Dns": null,
"DnsSearch": null,
"ExtraHosts": null,
"VolumesFrom": null,
"Devices": null,
"NetworkMode": "default",
"IpcMode": "",
"PidMode": "",
"UTSMode": "",
"CapAdd": null,
"CapDrop": null,
"GroupAdd": null,
"RestartPolicy": {
"Name": "on-failure",
"MaximumRetryCount": 5
},
"SecurityOpt": null,
"ReadonlyRootfs": false,
"Ulimits": null,
"LogConfig": {
"Type": "json-file",
"Config": {}
},
"CgroupParent": "",
"ConsoleSize": [
0,
0
]
},
"GraphDriver": {
"Name": "aufs",
"Data": null
},
"Mounts": [],
"Config": {
"Hostname": "98eff2817873",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"ExposedPorts": {
"6543/tcp": {}
},
"PublishService": "",
"Tty": true,
"OpenStdin": true,
"StdinOnce": false,
"Env": [
"INI=rancher",
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"DEBIAN_FRONTEND=noninteractive",
"PYTHON_EGG_CACHE=/pack/app/.egg-cache"
],
"Cmd": [
"/bin/sh",
"-c",
"./docker/start.sh"
],
"Image": "uberresearch/solr_webapp:rancher",
"Volumes": null,
"VolumeDriver": "",
"WorkingDir": "/pack/app",
"Entrypoint": null,
"NetworkDisabled": false,
"MacAddress": "02:7f:75:e9:50:aa",
"OnBuild": null,
"Labels": {
"io.rancher.container.ip": "10.42.198.58/16",
"io.rancher.container.uuid": "09d02248-7380-4034-be6c-3cf57527d207",
"io.rancher.project.name": "webapp",
"io.rancher.project_service.name": "webapp/solr-webapp",
"io.rancher.scheduler.affinity:container_label_soft": "io.rancher.service.deployment.unit=53e4984f-5de4-41a0-a793-e3d55137f236",
"io.rancher.service.deployment.unit": "53e4984f-5de4-41a0-a793-e3d55137f236",
"io.rancher.service.launch.config": "io.rancher.service.primary.launch.config",
"io.rancher.stack.name": "webapp",
"io.rancher.stack_service.name": "webapp/solr-webapp"
}
}
}
]
The hosts can ping each other, all ports are open, but the requests never reach the webapp. The result is always:
<html><body><h1>503 Service Unavailable</h1>
No server is available to handle this request.
</body></html>
Btw. I see the same racoon error as in Racon fail in Network agent 0.4.1 on RancherOS 0.33 - might that be the reason? (it’s on ubuntu 14.04.3 LTS - 3.19.30 though - not RancherOS)
How do I proceed from here?