Hi!
We have a rke1 cluster with kubernetes v1.21.5 running on 3 ubuntu 20.04 nodes for quite some time. However we decided to upgrade one node to ubuntu 22.04 and now pods running on this node are not able to communicate with the rest of the cluster. DNS resolution is not working and connections are not routed through to the other nodes.
cluster.yml
nodes:
- address: 10.0.40.15
port: "22"
role:
- controlplane
- worker
- etcd
hostname_override: node01
user: rke
docker_socket: /var/run/docker.sock
labels: {}
taints: []
- address: 10.0.40.16
port: "22"
role:
- controlplane
- worker
- etcd
hostname_override: node02
user: rke
docker_socket: /var/run/docker.sock
labels: {}
taints: []
- address: 10.0.40.17
port: "22"
role:
- controlplane
- worker
- etcd
hostname_override: node03
user: rke
docker_socket: /var/run/docker.sock
labels: {}
taints: []
services:
etcd:
image: ""
extra_args: {}
extra_binds: []
extra_env: []
win_extra_args: {}
win_extra_binds: []
win_extra_env: []
external_urls: []
ca_cert: ""
cert: ""
key: ""
path: ""
uid: 0
gid: 0
snapshot: true
retention: "72h"
creation: "6h"
backup_config:
interval_hours: 12
retention: 6
kube-api:
image: ""
extra_args: {}
extra_binds: []
extra_env: []
win_extra_args: {}
win_extra_binds: []
win_extra_env: []
service_cluster_ip_range: 10.43.0.0/16
service_node_port_range: "30000-32767"
pod_security_policy: false
always_pull_images: true
secrets_encryption_config:
enabled: true
audit_log: null
admission_configuration: null
event_rate_limit: null
kube-controller:
image: ""
extra_args: {}
extra_binds: []
extra_env: []
win_extra_args: {}
win_extra_binds: []
win_extra_env: []
cluster_cidr: 10.42.0.0/16
service_cluster_ip_range: 10.43.0.0/16
scheduler:
image: ""
extra_args: {}
extra_binds: []
extra_env: []
win_extra_args: {}
win_extra_binds: []
win_extra_env: []
kubelet:
image: ""
extra_args: {}
extra_binds: []
extra_env: []
win_extra_args: {}
win_extra_binds: []
win_extra_env: []
cluster_domain: cluster.local
infra_container_image: ""
cluster_dns_server: 10.43.0.10
fail_swap_on: false
generate_serving_certificate: false
kubeproxy:
image: ""
extra_args: {}
extra_binds: []
extra_env: []
win_extra_args: {}
win_extra_binds: []
win_extra_env: []
network:
plugin: flannel
options: {}
mtu: 0
node_selector: {}
update_strategy: null
tolerations: []
authentication:
strategy: x509
sans: []
webhook: null
addons: ""
addons_include: []
system_images:
etcd: rancher/mirrored-coreos-etcd:v3.4.16-rancher1
alpine: rancher/rke-tools:v0.1.78
nginx_proxy: rancher/rke-tools:v0.1.78
cert_downloader: rancher/rke-tools:v0.1.78
kubernetes_services_sidecar: rancher/rke-tools:v0.1.78
kubedns: rancher/mirrored-k8s-dns-kube-dns:1.17.4
dnsmasq: rancher/mirrored-k8s-dns-dnsmasq-nanny:1.17.4
kubedns_sidecar: rancher/mirrored-k8s-dns-sidecar:1.17.4
kubedns_autoscaler: rancher/mirrored-cluster-proportional-autoscaler:1.8.3
coredns: rancher/mirrored-coredns-coredns:1.8.4
coredns_autoscaler: rancher/mirrored-cluster-proportional-autoscaler:1.8.3
nodelocal: rancher/mirrored-k8s-dns-node-cache:1.18.0
kubernetes: rancher/hyperkube:v1.21.5-rancher1
flannel: rancher/mirrored-coreos-flannel:v0.14.0
flannel_cni: rancher/flannel-cni:v0.3.0-rancher6
calico_node: rancher/mirrored-calico-node:v3.19.2
calico_cni: rancher/mirrored-calico-cni:v3.19.2
calico_controllers: rancher/mirrored-calico-kube-controllers:v3.19.2
calico_ctl: rancher/mirrored-calico-ctl:v3.19.2
calico_flexvol: rancher/mirrored-calico-pod2daemon-flexvol:v3.19.2
canal_node: rancher/mirrored-calico-node:v3.19.2
canal_cni: rancher/mirrored-calico-cni:v3.19.2
canal_controllers: rancher/mirrored-calico-kube-controllers:v3.19.2
canal_flannel: rancher/mirrored-coreos-flannel:v0.14.0
canal_flexvol: rancher/mirrored-calico-pod2daemon-flexvol:v3.19.2
weave_node: weaveworks/weave-kube:2.8.1
weave_cni: weaveworks/weave-npc:2.8.1
pod_infra_container: rancher/mirrored-pause:3.4.1
ingress: rancher/nginx-ingress-controller:nginx-0.48.1-rancher1
ingress_backend: rancher/mirrored-nginx-ingress-controller-defaultbackend:1.5-rancher1
ingress_webhook: rancher/mirrored-jettech-kube-webhook-certgen:v1.5.1
metrics_server: rancher/mirrored-metrics-server:v0.5.0
windows_pod_infra_container: rancher/kubelet-pause:v0.1.6
aci_cni_deploy_container: noiro/cnideploy:5.1.1.0.1ae238a
aci_host_container: noiro/aci-containers-host:5.1.1.0.1ae238a
aci_opflex_container: noiro/opflex:5.1.1.0.1ae238a
aci_mcast_container: noiro/opflex:5.1.1.0.1ae238a
aci_ovs_container: noiro/openvswitch:5.1.1.0.1ae238a
aci_controller_container: noiro/aci-containers-controller:5.1.1.0.1ae238a
aci_gbp_server_container: noiro/gbp-server:5.1.1.0.1ae238a
aci_opflex_server_container: noiro/opflex-server:5.1.1.0.1ae238a
ssh_key_path: ~/.ssh/id_rsa
ssh_cert_path: ""
ssh_agent_auth: false
authorization:
mode: rbac
options: {}
ignore_docker_version: null
enable_cri_dockerd: null
kubernetes_version: ""
private_registries: []
ingress:
provider: ""
options: {}
node_selector: {}
extra_args: {}
dns_policy: ""
extra_envs: []
extra_volumes: []
extra_volume_mounts: []
update_strategy: null
http_port: 0
https_port: 0
network_mode: ""
tolerations: []
default_backend: null
default_http_backend_priority_class_name: ""
nginx_ingress_controller_priority_class_name: ""
cluster_name: "new-haven"
prefix_path: ""
win_prefix_path: ""
addon_job_timeout: 0
bastion_host:
address: ""
port: ""
user: ""
ssh_key: ""
ssh_key_path: ""
ssh_cert: ""
ssh_cert_path: ""
ignore_proxy_env_vars: false
monitoring:
provider: ""
options: {}
node_selector: {}
update_strategy: null
replicas: null
tolerations: []
metrics_server_priority_class_name: ""
restore:
restore: false
snapshot_name: ""
rotate_encryption_key: false
dns: null
It seems like the connections are blocked but the node has no firewall running. Maybe someone has an idea what could be wrong with this node. Thanks for your help.