Install Helm Internal error occurred: conversion webhook for cert-manager.io/v1beta1, Kind=Issuer failed: Post "https://cert-manager-webhook.cert-manager.svc:443"

cert-manager looks like fine.
any suggestion?


➤ cat config.a.yaml
tls-san:
- demo-a
- demo-a.home.monach.us
- demo-cluster.home.monach.us
- 10.10.10.51
disable: rke2-ingress-nginx
cni:
- cilium

$ scp config.a.yaml demo-a:.

$ ssh demo-a

sudo -s

# Set up environment
export VIP=10.10.10.50
export TAG=v0.3.8
export INTERFACE=ens192
export CONTAINER_RUNTIME_ENDPOINT=unix:///run/k3s/containerd/containerd.sock
export CONTAINERD_ADDRESS=/run/k3s/containerd/containerd.sock
export PATH=/var/lib/rancher/rke2/bin:$PATH
export KUBECONFIG=/etc/rancher/rke2/rke2.yaml
alias k=kubectl

# Install RKE2
mkdir -p /etc/rancher/rke2
cp config.a.yaml /etc/rancher/rke2/config.yaml
curl -sfL https://get.rke2.io | sh -
systemctl enable rke2-server
systemctl start rke2-server
# wait for rke2 to be ready
kubectl get nodes


# Pull kube-vip RBAC manifest
curl -s https://kube-vip.io/manifests/rbac.yaml > /var/lib/rancher/rke2/server/manifests/kube-vip-rbac.yaml

# pull image
crictl pull docker.io/plndr/kube-vip:$TAG

# create alias
# on k3s `ctr` is a link to `k3s` which has the namespace set by default but on rke2 we
# have to specify the namespace

alias kube-vip="ctr --namespace k8s.io run --rm --net-host docker.io/plndr/kube-vip:$TAG vip /kube-vip"

# generate manifest
kube-vip manifest daemonset \
    --arp \
    --interface $INTERFACE \
    --address $VIP \
    --controlplane \
    --leaderElection \
    --taint \
    --services \
    --inCluster | tee /var/lib/rancher/rke2/server/manifests/kube-vip.yaml

# check logs
root@demo-a:~# k get po -n kube-system | grep kube-vip
kube-vip-ds-8595m                           1/1     Running     0          48s

root@demo-a:~# k logs kube-vip-ds-8595m -n kube-system --tail 1
time="2021-03-12T12:29:38Z" level=info msg="Broadcasting ARP update for 10.68.0.80 (02:84:08:4a:dd:1c) via ens18"


# get the token from demo-a
root@demo-a:~# cat /var/lib/rancher/rke2/server/token

# add it to config.b and config.c

➤ cat config.b.yaml
token: K10400a2a885bd2afd9bbf90b3b7e3117f7b9b78393d240bc509699c04e111949d2::server:9b27c71333958cefa9c31ca3c4c3a674
server: https://demo-cluster.home.monach.us:9345
tls-san:
- demo-b
- demo-b.home.monach.us
- demo-cluster.home.monach.us
- 10.10.10.52
disable:
- rke2-ingress-nginx
cni:
- cilium

➤ cat config.c.yaml
token: K10400a2a885bd2afd9bbf90b3b7e3117f7b9b78393d240bc509699c04e111949d2::server:9b27c71333958cefa9c31ca3c4c3a674
server: https://demo-cluster.home.monach.us:9345
tls-san:
- demo-c
- demo-c.home.monach.us
- demo-cluster.home.monach.us
- 10.10.10.53
disable:
- rke2-ingress-nginx
cni:
- cilium

# copy other configs over
➤ for x in b c; scp config.$x.yaml demo-$x:config.yaml; end
config.b.yaml                                                               100%  230   385.3KB/s   00:00
config.c.yaml                                                               100%  230   263.6KB/s   00:00

# bring up other two master nodes

# demo-b
sudo -s
mkdir -p /etc/rancher/rke2
cp config.b.yaml /etc/rancher/rke2/config.yaml
curl -sfL https://get.rke2.io | sh -
systemctl enable rke2-server
systemctl start rke2-server

# demo-c
sudo -s
mkdir -p /etc/rancher/rke2
cp config.c.yaml /etc/rancher/rke2/config.yaml
curl -sfL https://get.rke2.io | sh -
systemctl enable rke2-server
systemctl start rke2-server

# demo-a
kubectl get nodes -w

# if it doesn't come up and if `crictl ps -a` shows an Exited etcd container, restart the service

# kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.5.2/cert-manager.crds.yaml

# helm install \
  cert-manager jetstack/cert-manager \
  --namespace cert-manager \
  --version v1.5.2

# helm install rancher rancher-stable/rancher --namespace cattle-system --set hostname=service.rancher.com
Install Helm Internal error occurred: conversion webhook for cert-manager.io/v1beta1, Kind=Issuer failed: Post “https://cert-manager-webhook.cert-manager.svc:443”

# # kubectl get pods --namespace cert-manager
NAME                                       READY   STATUS    RESTARTS   AGE
cert-manager-66b6d6bf59-ssvb8              1/1     Running   0          127m
cert-manager-cainjector-856d4df858-wlrxf   1/1     Running   0          127m
cert-manager-webhook-5fd7d458f7-6fjw6      1/1     Running   0          127m

# # kubectl logs -n cert-manager cert-manager-66b6d6bf59-ssvb8 
I1013 01:57:12.951606       1 start.go:75] cert-manager "msg"="starting controller"  "git-commit"="b26bd256f124d480fcb198e1464854f41b0d0d2c" "version"="v1.5.3"
W1013 01:57:12.951681       1 client_config.go:615] Neither --kubeconfig nor --master was specified.  Using the inClusterConfig.  This might not work.
I1013 01:57:12.952833       1 controller.go:268] cert-manager/controller/build-context "msg"="configured acme dns01 nameservers" "nameservers"=["10.43.0.10:53"]
I1013 01:57:12.953361       1 controller.go:85] cert-manager/controller "msg"="enabled controllers: [certificaterequests-approver certificaterequests-issuer-acme certificaterequests-issuer-ca certificaterequests-issuer-selfsigned certificaterequests-issuer-vault certificaterequests-issuer-venafi certificates-issuing certificates-key-manager certificates-metrics certificates-readiness certificates-request-manager certificates-revision-manager certificates-trigger challenges clusterissuers ingress-shim issuers orders]"
I1013 01:57:12.953588       1 controller.go:115] cert-manager/controller "msg"="starting leader election"
I1013 01:57:12.953695       1 controller.go:105] cert-manager/controller "msg"="starting metrics server"  "address"={"IP":"::","Port":9402,"Zone":""}
I1013 01:57:12.954227       1 leaderelection.go:243] attempting to acquire leader lease kube-system/cert-manager-controller...
I1013 01:57:12.981670       1 leaderelection.go:253] successfully acquired lease kube-system/cert-manager-controller
I1013 01:57:12.981877       1 controller.go:163] cert-manager/controller "msg"="not starting controller as it's disabled" "controller"="gateway-shim"
I1013 01:57:12.982141       1 controller.go:163] cert-manager/controller "msg"="not starting controller as it's disabled" "controller"="certificatesigningrequests-issuer-ca"
I1013 01:57:12.982210       1 controller.go:186] cert-manager/controller "msg"="starting controller" "controller"="certificaterequests-approver"
I1013 01:57:12.982258       1 controller.go:186] cert-manager/controller "msg"="starting controller" "controller"="certificates-readiness"
I1013 01:57:12.982286       1 controller.go:186] cert-manager/controller "msg"="starting controller" "controller"="certificaterequests-issuer-acme"
I1013 01:57:12.982320       1 controller.go:186] cert-manager/controller "msg"="starting controller" "controller"="certificaterequests-issuer-vault"
I1013 01:57:12.982304       1 controller.go:186] cert-manager/controller "msg"="starting controller" "controller"="certificates-revision-manager"
I1013 01:57:12.982337       1 controller.go:186] cert-manager/controller "msg"="starting controller" "controller"="certificates-key-manager"
I1013 01:57:12.982344       1 controller.go:186] cert-manager/controller "msg"="starting controller" "controller"="certificates-metrics"
I1013 01:57:12.982368       1 controller.go:186] cert-manager/controller "msg"="starting controller" "controller"="certificaterequests-issuer-venafi"
I1013 01:57:12.982391       1 controller.go:186] cert-manager/controller "msg"="starting controller" "controller"="issuers"
I1013 01:57:13.084817       1 controller.go:186] cert-manager/controller "msg"="starting controller" "controller"="challenges"
I1013 01:57:13.084869       1 controller.go:186] cert-manager/controller "msg"="starting controller" "controller"="orders"
I1013 01:57:13.084899       1 controller.go:186] cert-manager/controller "msg"="starting controller" "controller"="ingress-shim"
I1013 01:57:13.084901       1 controller.go:163] cert-manager/controller "msg"="not starting controller as it's disabled" "controller"="certificatesigningrequests-issuer-acme"
I1013 01:57:13.084919       1 controller.go:163] cert-manager/controller "msg"="not starting controller as it's disabled" "controller"="certificatesigningrequests-issuer-selfsigned"
I1013 01:57:13.084921       1 controller.go:186] cert-manager/controller "msg"="starting controller" "controller"="certificates-request-manager"
I1013 01:57:13.084930       1 controller.go:163] cert-manager/controller "msg"="not starting controller as it's disabled" "controller"="certificatesigningrequests-issuer-vault" 
I1013 01:57:13.085627       1 controller.go:163] cert-manager/controller "msg"="not starting controller as it's disabled" "controller"="certificatesigningrequests-issuer-venafi"
I1013 01:57:13.086002       1 controller.go:186] cert-manager/controller "msg"="starting controller" "controller"="certificates-trigger"
I1013 01:57:13.086039       1 controller.go:186] cert-manager/controller "msg"="starting controller" "controller"="certificaterequests-issuer-ca"
I1013 01:57:13.086125       1 controller.go:186] cert-manager/controller "msg"="starting controller" "controller"="certificaterequests-issuer-selfsigned"
I1013 01:57:13.086186       1 controller.go:186] cert-manager/controller "msg"="starting controller" "controller"="certificates-issuing"
I1013 01:57:13.086364       1 controller.go:186] cert-manager/controller "msg"="starting controller" "controller"="clusterissuers"
E1013 02:03:24.607195       1 controller.go:163] cert-manager/controller/ingress-shim "msg"="re-queuing item due to error processing" "error"="Internal error occurred: failed calling webhook \"webhook.cert-manager.io\": Post \"https://cert-manager-webhook.cert-manager.svc:443/mutate?timeout=10s\": context deadline exceeded" "key"="cattle-system/rancher"
E1013 02:03:39.616701       1 controller.go:163] cert-manager/controller/ingress-shim "msg"="re-queuing item due to error processing" "error"="Internal error occurred: failed calling webhook \"webhook.cert-manager.io\": Post \"https://cert-manager-webhook.cert-manager.svc:443/mutate?timeout=10s\": context deadline exceeded" "key"="cattle-system/rancher"
E1013 02:03:59.629077       1 controller.go:163] cert-manager/controller/ingress-shim "msg"="re-queuing item due to error processing" "error"="Internal error occurred: failed calling webhook \"webhook.cert-manager.io\": Post \"https://cert-manager-webhook.cert-manager.svc:443/mutate?timeout=10s\": context deadline exceeded" "key"="cattle-system/rancher"
E1013 02:04:29.642140       1 controller.go:163] cert-manager/controller/ingress-shim "msg"="re-queuing item due to error processing" "error"="Internal error occurred: failed calling webhook \"webhook.cert-manager.io\": Post \"https://cert-manager-webhook.cert-manager.svc:443/mutate?timeout=10s\": context deadline exceeded" "key"="cattle-system/rancher"
E1013 02:05:19.659027       1 controller.go:163] cert-manager/controller/ingress-shim "msg"="re-queuing item due to error processing" "error"="Internal error occurred: failed calling webhook \"webhook.cert-manager.io\": Post \"https://cert-manager-webhook.cert-manager.svc:443/mutate?timeout=10s\": context deadline exceeded" "key"="cattle-system/rancher"
E1013 02:06:49.675782       1 controller.go:163] cert-manager/controller/ingress-shim "msg"="re-queuing item due to error processing" "error"="Internal error occurred: failed calling webhook \"webhook.cert-manager.io\": Post \"https://cert-manager-webhook.cert-manager.svc:443/mutate?timeout=10s\": context deadline exceeded" "key"="cattle-system/rancher"
E1013 02:09:39.683908       1 controller.go:163] cert-manager/controller/ingress-shim "msg"="re-queuing item due to error processing" "error"="Internal error occurred: failed calling webhook \"webhook.cert-manager.io\": Post \"https://cert-manager-webhook.cert-manager.svc:443/mutate?timeout=10s\": context deadline exceeded" "key"="cattle-system/rancher"
E1013 02:14:49.696612       1 controller.go:163] cert-manager/controller/ingress-shim "msg"="re-queuing item due to error processing" "error"="Internal error occurred: failed calling webhook \"webhook.cert-manager.io\": Post \"https://cert-manager-webhook.cert-manager.svc:443/mutate?timeout=10s\": context deadline exceeded" "key"="cattle-system/rancher"

# kubectl logs -n cert-manager cert-manager-cainjector-856d4df858-wlrxf
I1013 01:57:05.496825       1 start.go:107] "starting" version="v1.5.3" revision="b26bd256f124d480fcb198e1464854f41b0d0d2c"
I1013 01:57:06.547970       1 request.go:668] Waited for 1.031471845s due to client-side throttling, not priority and fairness, request: GET:https://10.43.0.1:443/apis/certificates.k8s.io/v1beta1?timeout=32s
I1013 01:57:11.573898       1 request.go:668] Waited for 1.043908726s due to client-side throttling, not priority and fairness, request: GET:https://10.43.0.1:443/apis/authorization.k8s.io/v1?timeout=32s
I1013 01:57:15.537746       1 leaderelection.go:243] attempting to acquire leader lease kube-system/cert-manager-cainjector-leader-election...
I1013 01:57:15.553991       1 leaderelection.go:253] successfully acquired lease kube-system/cert-manager-cainjector-leader-election
I1013 01:57:15.554105       1 recorder.go:104] cert-manager/controller-runtime/manager/events "msg"="Normal"  "message"="cert-manager-cainjector-856d4df858-wlrxf_35241083-39ec-4920-872d-7ea15fbfe4dc became leader" "object"={"kind":"ConfigMap","namespace":"kube-system","name":"cert-manager-cainjector-leader-election","uid":"390704c9-5bfb-44cc-805d-bdd8a77ef749","apiVersion":"v1","resourceVersion":"383549"} "reason"="LeaderElection"
I1013 01:57:15.554138       1 recorder.go:104] cert-manager/controller-runtime/manager/events "msg"="Normal"  "message"="cert-manager-cainjector-856d4df858-wlrxf_35241083-39ec-4920-872d-7ea15fbfe4dc became leader" "object"={"kind":"Lease","namespace":"kube-system","name":"cert-manager-cainjector-leader-election","uid":"57fe6b55-6003-4ecd-b080-8dfbb47b52bc","apiVersion":"coordination.k8s.io/v1","resourceVersion":"383550"} "reason"="LeaderElection"
I1013 01:57:15.755205       1 controller.go:165] cert-manager/certificate/customresourcedefinition/controller/controller-for-certificate-customresourcedefinition "msg"="Starting EventSource"  "source"={}
I1013 01:57:15.755238       1 controller.go:165] cert-manager/certificate/customresourcedefinition/controller/controller-for-certificate-customresourcedefinition "msg"="Starting EventSource"  "source"={}
I1013 01:57:15.755251       1 controller.go:165] cert-manager/certificate/customresourcedefinition/controller/controller-for-certificate-customresourcedefinition "msg"="Starting EventSource"  "source"={}
I1013 01:57:15.755252       1 controller.go:165] cert-manager/certificate/mutatingwebhookconfiguration/controller/controller-for-certificate-mutatingwebhookconfiguration "msg"="Starting EventSource"  "source"={}
I1013 01:57:15.755260       1 controller.go:173] cert-manager/certificate/customresourcedefinition/controller/controller-for-certificate-customresourcedefinition "msg"="Starting Controller"
I1013 01:57:15.755267       1 controller.go:165] cert-manager/certificate/mutatingwebhookconfiguration/controller/controller-for-certificate-mutatingwebhookconfiguration "msg"="Starting EventSource"  "source"={}
I1013 01:57:15.755275       1 controller.go:165] cert-manager/certificate/mutatingwebhookconfiguration/controller/controller-for-certificate-mutatingwebhookconfiguration "msg"="Starting EventSource"  "source"={}
I1013 01:57:15.755282       1 controller.go:173] cert-manager/certificate/mutatingwebhookconfiguration/controller/controller-for-certificate-mutatingwebhookconfiguration "msg"="Starting Controller"
I1013 01:57:15.755330       1 controller.go:165] cert-manager/secret/mutatingwebhookconfiguration/controller/controller-for-secret-mutatingwebhookconfiguration "msg"="Starting EventSource"  "source"={}
I1013 01:57:15.755343       1 controller.go:165] cert-manager/secret/mutatingwebhookconfiguration/controller/controller-for-secret-mutatingwebhookconfiguration "msg"="Starting EventSource"  "source"={}
I1013 01:57:15.755349       1 controller.go:173] cert-manager/secret/mutatingwebhookconfiguration/controller/controller-for-secret-mutatingwebhookconfiguration "msg"="Starting Controller"
I1013 01:57:15.755403       1 controller.go:165] cert-manager/certificate/apiservice/controller/controller-for-certificate-apiservice "msg"="Starting EventSource"  "source"={}
I1013 01:57:15.755414       1 controller.go:165] cert-manager/secret/validatingwebhookconfiguration/controller/controller-for-secret-validatingwebhookconfiguration "msg"="Starting EventSource"  "source"={}
I1013 01:57:15.755418       1 controller.go:165] cert-manager/certificate/apiservice/controller/controller-for-certificate-apiservice "msg"="Starting EventSource"  "source"={}
I1013 01:57:15.755426       1 controller.go:165] cert-manager/secret/validatingwebhookconfiguration/controller/controller-for-secret-validatingwebhookconfiguration "msg"="Starting EventSource"  "source"={}
I1013 01:57:15.755434       1 controller.go:173] cert-manager/secret/validatingwebhookconfiguration/controller/controller-for-secret-validatingwebhookconfiguration "msg"="Starting Controller"
I1013 01:57:15.755473       1 controller.go:165] cert-manager/secret/apiservice/controller/controller-for-secret-apiservice "msg"="Starting EventSource"  "source"={}
I1013 01:57:15.755485       1 controller.go:165] cert-manager/secret/apiservice/controller/controller-for-secret-apiservice "msg"="Starting EventSource"  "source"={}
I1013 01:57:15.755506       1 controller.go:173] cert-manager/secret/apiservice/controller/controller-for-secret-apiservice "msg"="Starting Controller"
I1013 01:57:15.755558       1 controller.go:165] cert-manager/secret/customresourcedefinition/controller/controller-for-secret-customresourcedefinition "msg"="Starting EventSource"  "source"={}
I1013 01:57:15.755567       1 controller.go:165] cert-manager/secret/customresourcedefinition/controller/controller-for-secret-customresourcedefinition "msg"="Starting EventSource"  "source"={}
I1013 01:57:15.755574       1 controller.go:173] cert-manager/secret/customresourcedefinition/controller/controller-for-secret-customresourcedefinition "msg"="Starting Controller"
I1013 01:57:15.755426       1 controller.go:165] cert-manager/certificate/apiservice/controller/controller-for-certificate-apiservice "msg"="Starting EventSource"  "source"={}
I1013 01:57:15.755641       1 controller.go:173] cert-manager/certificate/apiservice/controller/controller-for-certificate-apiservice "msg"="Starting Controller"
I1013 01:57:15.755737       1 controller.go:165] cert-manager/certificate/validatingwebhookconfiguration/controller/controller-for-certificate-validatingwebhookconfiguration "msg"="Starting EventSource"  "source"={}
I1013 01:57:15.755748       1 controller.go:165] cert-manager/certificate/validatingwebhookconfiguration/controller/controller-for-certificate-validatingwebhookconfiguration "msg"="Starting EventSource"  "source"={}
I1013 01:57:15.755758       1 controller.go:165] cert-manager/certificate/validatingwebhookconfiguration/controller/controller-for-certificate-validatingwebhookconfiguration "msg"="Starting EventSource"  "source"={}
I1013 01:57:15.755765       1 controller.go:173] cert-manager/certificate/validatingwebhookconfiguration/controller/controller-for-certificate-validatingwebhookconfiguration "msg"="Starting Controller"
I1013 01:57:15.856510       1 controller.go:207] cert-manager/certificate/customresourcedefinition/controller/controller-for-certificate-customresourcedefinition "msg"="Starting workers"  "worker count"=1
I1013 01:57:15.856561       1 controller.go:207] cert-manager/secret/mutatingwebhookconfiguration/controller/controller-for-secret-mutatingwebhookconfiguration "msg"="Starting workers"  "worker count"=1
I1013 01:57:15.856641       1 controller.go:207] cert-manager/certificate/validatingwebhookconfiguration/controller/controller-for-certificate-validatingwebhookconfiguration "msg"="Starting workers"  "worker count"=1
I1013 01:57:15.856736       1 controller.go:207] cert-manager/secret/apiservice/controller/controller-for-secret-apiservice "msg"="Starting workers"  "worker count"=1
I1013 01:57:15.856780       1 controller.go:207] cert-manager/certificate/apiservice/controller/controller-for-certificate-apiservice "msg"="Starting workers"  "worker count"=1
I1013 01:57:15.856876       1 controller.go:207] cert-manager/secret/validatingwebhookconfiguration/controller/controller-for-secret-validatingwebhookconfiguration "msg"="Starting workers"  "worker count"=1
I1013 01:57:15.857300       1 controller.go:207] cert-manager/secret/customresourcedefinition/controller/controller-for-secret-customresourcedefinition "msg"="Starting workers"  "worker count"=1
I1013 01:57:15.856510       1 controller.go:207] cert-manager/certificate/mutatingwebhookconfiguration/controller/controller-for-certificate-mutatingwebhookconfiguration "msg"="Starting workers"  "worker count"=1
I1013 01:57:15.878681       1 controller.go:178] cert-manager/secret/mutatingwebhookconfiguration/generic-inject-reconciler "msg"="updated object" "resource_kind"="MutatingWebhookConfiguration" "resource_name"="cert-manager-webhook" "resource_namespace"="" "resource_version"="v1"
I1013 01:57:15.878783       1 controller.go:178] cert-manager/secret/validatingwebhookconfiguration/generic-inject-reconciler "msg"="updated object" "resource_kind"="ValidatingWebhookConfiguration" "resource_name"="cert-manager-webhook" "resource_namespace"="" "resource_version"="v1"
I1013 01:57:15.881374       1 controller.go:178] cert-manager/secret/mutatingwebhookconfiguration/generic-inject-reconciler "msg"="updated object" "resource_kind"="MutatingWebhookConfiguration" "resource_name"="cert-manager-webhook" "resource_namespace"="" "resource_version"="v1"
I1013 01:57:15.882900       1 controller.go:178] cert-manager/secret/validatingwebhookconfiguration/generic-inject-reconciler "msg"="updated object" "resource_kind"="ValidatingWebhookConfiguration" "resource_name"="cert-manager-webhook" "resource_namespace"="" "resource_version"="v1"
I1013 01:57:15.909703       1 controller.go:178] cert-manager/secret/customresourcedefinition/generic-inject-reconciler "msg"="updated object" "resource_kind"="CustomResourceDefinition" "resource_name"="certificates.cert-manager.io" "resource_namespace"="" "resource_version"="v1"
I1013 01:57:15.963281       1 controller.go:178] cert-manager/secret/customresourcedefinition/generic-inject-reconciler "msg"="updated object" "resource_kind"="CustomResourceDefinition" "resource_name"="orders.acme.cert-manager.io" "resource_namespace"="" "resource_version"="v1"
I1013 01:57:16.089975       1 controller.go:178] cert-manager/secret/customresourcedefinition/generic-inject-reconciler "msg"="updated object" "resource_kind"="CustomResourceDefinition" "resource_name"="challenges.acme.cert-manager.io" "resource_namespace"="" "resource_version"="v1"
I1013 01:57:16.281679       1 controller.go:178] cert-manager/secret/customresourcedefinition/generic-inject-reconciler "msg"="updated object" "resource_kind"="CustomResourceDefinition" "resource_name"="issuers.cert-manager.io" "resource_namespace"="" "resource_version"="v1"
I1013 01:57:16.483114       1 controller.go:178] cert-manager/secret/customresourcedefinition/generic-inject-reconciler "msg"="updated object" "resource_kind"="CustomResourceDefinition" "resource_name"="clusterissuers.cert-manager.io" "resource_namespace"="" "resource_version"="v1"
I1013 01:57:16.513086       1 controller.go:178] cert-manager/secret/customresourcedefinition/generic-inject-reconciler "msg"="updated object" "resource_kind"="CustomResourceDefinition" "resource_name"="certificaterequests.cert-manager.io" "resource_namespace"="" "resource_version"="v1"
I1013 01:57:16.557998       1 controller.go:178] cert-manager/secret/customresourcedefinition/generic-inject-reconciler "msg"="updated object" "resource_kind"="CustomResourceDefinition" "resource_name"="certificates.cert-manager.io" "resource_namespace"="" "resource_version"="v1"
I1013 01:57:16.578101       1 controller.go:178] cert-manager/secret/customresourcedefinition/generic-inject-reconciler "msg"="updated object" "resource_kind"="CustomResourceDefinition" "resource_name"="orders.acme.cert-manager.io" "resource_namespace"="" "resource_version"="v1"
I1013 01:57:16.719472       1 controller.go:178] cert-manager/secret/customresourcedefinition/generic-inject-reconciler "msg"="updated object" "resource_kind"="CustomResourceDefinition" "resource_name"="challenges.acme.cert-manager.io" "resource_namespace"="" "resource_version"="v1"
I1013 01:57:16.896617       1 controller.go:178] cert-manager/secret/customresourcedefinition/generic-inject-reconciler "msg"="updated object" "resource_kind"="CustomResourceDefinition" "resource_name"="issuers.cert-manager.io" "resource_namespace"="" "resource_version"="v1"
I1013 01:57:17.075392       1 controller.go:178] cert-manager/secret/customresourcedefinition/generic-inject-reconciler "msg"="updated object" "resource_kind"="CustomResourceDefinition" "resource_name"="clusterissuers.cert-manager.io" "resource_namespace"="" "resource_version"="v1"
I1013 01:57:17.096216       1 controller.go:178] cert-manager/secret/customresourcedefinition/generic-inject-reconciler "msg"="updated object" "resource_kind"="CustomResourceDefinition" "resource_name"="certificaterequests.cert-manager.io" "resource_namespace"="" "resource_version"="v1"

# kubectl logs -n cert-manager cert-manager-webhook-5fd7d458f7-6fjw6
W1013 01:57:01.362889       1 client_config.go:615] Neither --kubeconfig nor --master was specified.  Using the inClusterConfig.  This might not work.
W1013 01:57:01.363895       1 client_config.go:615] Neither --kubeconfig nor --master was specified.  Using the inClusterConfig.  This might not work.
I1013 01:57:01.364385       1 webhook.go:70] cert-manager/webhook "msg"="using dynamic certificate generating using CA stored in Secret resource"  "secret_name"="cert-manager-webhook-ca" "secret_namespace"="cert-manager"
I1013 01:57:01.364547       1 server.go:138] cert-manager/webhook "msg"="listening for insecure healthz connections"  "address"=":6080"
I1013 01:57:01.364590       1 server.go:169] cert-manager/webhook "msg"="listening for secure connections"  "address"=":10250"
I1013 01:57:01.364608       1 server.go:201] cert-manager/webhook "msg"="registered pprof handlers"
I1013 01:57:02.399099       1 dynamic_source.go:272] cert-manager/webhook "msg"="Updated serving TLS certificate"