RKE UCM Version - 2.6.9
RKE UCM K8s Version - 1.24.17
Downstream cluster k8s version - 1.24.17
There has been no recent upgrade on the platform. Recently observing when VM’s are being spinned, default container in Rancher is not coming up and it is throwing with below logs. Situation is like out of 10 Vm’s when spinned together two or three VM’s get into this state OR even when 1 VM is spinned then also the same situation.
[root@rancher-vm-prod ~]# docker logs 6925b0d40b61
CONTAINER ID
5925b0d4
IMAGE
ptp-docker-remote.docker/rancher/rancher-agent:v2.6.9
COMMAND
"run.sh --server htt..."
CREATED
STATUS
15 minutes ago Up 15 minutes
PORTS
[root@rancher-vm-prod ~]# docker logs 5925b0d4
INFO: Arguments: --server https://rancher-console.com —-token REDACTED -r -n m-b324d
INFO: Environment: CATTLE_ADDRESS=12.16.12.9 CATTLE_AGENT_CONNECT=true CATTLE_INTERNAL_ADDRESS= CATTLE_NODE_NAME=m-b324d CATTLE_SERVER=https://rancher-console.com CATTLE_TOKEN=REDACTED
INFO: Using resolv.conf: search rancher.com 0998.rancher.com nameserver 18.44.12.3 nameserver 18.44.12.4
INFO: https://rancher-console.com/ping is accessible
INFO: rancher-console.com resolves to 16.2.22.2
Time="2024-07-2909:23:16Z" level=info msg="Listening on /tmp/log.sock"
Time="2024-07-29T09:23:16Z" level=info msg="Rancher agent version v2.6.9 is starting"
Time="2024-07-29T09:23:16Z" level=info msg="Option controlPlane=false" ime="2024-07-29T09:23:16Z"
level=info msg="Option worker=false" ime="2024-07-29T09:23:16Z" level=info msg="Option requestedHostname=m-b324d"
Tme="2024-07-29T09:23:16Z" level=info msg="Option dockerInfo={EHMP:PJPI:HMEC:32LH:KXAL:AW4Z:LD4F: IWAW: PDPX: FG35:T5MH:L6XN 1 1 0 0 1 overlay2 [[Backing Filesystem extfs] [Supports d_type true] [Native Overlay Diff truel [userxattr falsell Il {[locall [bridge host ipvlan maclan null overlay] Il lawslogs fluentd goplogs gelf journald json-file local logentries splunk s /slog]} true true false false true true true true true true true true false 32 false 40 2024-07-2905:23:16.606711375-04:00 json-file systemd 2 0 5.14.0-362.18.1.19_3.0.1.x86_64 Rocky Linux 9.3 (Blue Onyx) 9.3 linux x86_64 https://index.docker.io/v1/ 0xc001d3c5b0 24 50204053504 I] /var/lib/docker rancher-vm-prod [provider=vmwarevspheel false 20.10.24 map [io.containerd. runc.v2: {runc I] } io.containerd.runtime.v1.linux:{runc Il } runc:{runc Il }l runc { inactive false Il 0 0 ‹nil> Il} false docke -init {a1496014c916f962104b33d1bb5bd03b0858e59 a1496014c916f9e62104b33d1bb5bd03b0858e59} {v1.1.11-0-g4bccb38 v1.1.11-0-g4bccb38} {de40adO de40ad0} [name=seccomp, profile=default name=cgroupns]［］［］｝"
ime="2024-07-29T09:23:16Z" level=info msg="Option customConfig=map[address:172.16.182.49 internalAddress: label:map[] roles:[] taints:[]]"
Time="2024-07-29T09:23:16Z" level=info msq="Option etcd=false"
Time="2024-07-29T09:23:162" level=info msg="Connecting to wss://rancher-console.com/V3/connect with token starting with gk89tjsjk5ldv8"
Time="2024-07-29T09:23:16Z" level=info msg="Connecting to proxy" url="wss://rancher-console.com/v3/connect"
Tme="2024-07-29T09:23:16Z" level=info msg="Waiting for node to register. Either cluster is not ready for registering, cluster is currently provisioning, or etcd, controlplane and worker node have to be registered"
Time="2024-07-29T09:23:18Z" level=info msg="Starting plan monitor, checking every 15 seconds"
Time="2024-07-29T09:23:33Z" level=warning msg="Error while getting agent config: invalid response 500: nodes.management.cattle.io "c-64anv/m-a9caf5d11" not found" time="2024-07-29T09:23:38Z" level=warning msg="Error while getting agent config: invalid response 500: nodes.management.cattle.io "c-64anv/m-a9caf5d11" not found"
Time="2024-07-29T09:23:43Z" level=warning msg="Error while getting agent config: invalid response 500: nodes-management.cattle.io "c-64anv/m-a9caf5d11" not found"
Time="2024-07-29T09:23:48Z" level=warning msg="Error while getting agent config: invalid response 500: nodes-management.cattle.io "c-64anv/m-a9caf5d11" not found"
Time="2024-07-2909:23:53Z" level=warning msg="Error while getting agent config: invalid response 500: nodes.management.cattle.io "C-64anv/m-a9caf5d11" not found"
Time="2024-07-29T09:23:58Z" level=warning msg="Error while getting agent config: invalid response 500: nodes.management.cattle.io "c-64anv/m-a9caf5d11" not found"
Time="2024-07-29T09:24:03Z" level=warning msg="Error while getting agent config: invalid response 500: nodes.management.cattle.io "C-64anv/m-a9caf5d11" not found"
Time="2024-07-29T09:24:08Z" level=warning msg="Error while getting agent config: invalid response 500: nodes.management.cattle.io "c-64anv/m-a9caf5d11" not found"
Time="2024-07-29T09:24:13Z" level=warning msg="Error while getting agent config: invalid response 500: nodes.management.cattle.io "c-64anv/m-a9caf5d11" not found" Time="2024-07-29T09:24:18Z" level=warning msg="Error while getting agent config: invalid response 500: nodes-management.cattle.io "c-64anv/m-a9caf5d11" not found" Time="2024-07-29T09:24:23Z" level=warning msg="Error while getting agent config: invalid response 500: nodes-management.cattle.io "c-64anv/m-a9caf5d11" not found"