Rancher 2.5 - Logging to Cloudwatch

Rancher 2.x
1

I am quite new to k8s and rancher. I am trying to get the cluster logs to Cloudwatch from our On Prem cluster.

I have installed the Logging App on the cluster and set up a ClusterOutput and ClusterFlow.

The output includes the creds to log into AWS.

Despite filling out all the fields, I’m not getting any data nor am I getting any errors. I feel like I am missing a step. Can someone provide some guidence as to what I am doing wrong and potentually how to fix it?

apiVersion: logging.banzaicloud.io/v1beta1
kind: ClusterOutput
metadata:
  creationTimestamp: "2021-10-12T06:21:17Z"
  generation: 4
  managedFields:
  - apiVersion: logging.banzaicloud.io/v1beta1
    fieldsType: FieldsV1
    fieldsV1:
      f:spec:
        .: {}
        f:cloudwatch:
          .: {}
          f:auto_create_stream: {}
          f:aws_key_id:
            .: {}
            f:valueFrom:
              .: {}
              f:secretKeyRef:
                .: {}
                f:key: {}
                f:name: {}
          f:aws_sec_key:
            .: {}
            f:valueFrom:
              .: {}
              f:secretKeyRef:
                .: {}
                f:key: {}
                f:name: {}
          f:log_group_name: {}
          f:log_stream_name: {}
          f:region: {}
    manager: rancher
    operation: Update
    time: "2021-10-12T06:30:45Z"
  name: pe-aws
  namespace: cattle-logging-system
  resourceVersion: "15227736"
  uid: bea8802e-0833-431f-ae08-0f236c0b449d
spec:
  cloudwatch:
    auto_create_stream: true
    aws_key_id:
      valueFrom:
        secretKeyRef:
          key: AWS_ACCESS_KEY_ID
          name: aws-access-keys
    aws_sec_key:
      valueFrom:
        secretKeyRef:
          key: AWS_SECRET_ACCESS_KEY
          name: aws-access-keys
    log_group_name: pe
    log_stream_name: pe-cluster-stream
    region: ap-southeast-2

apiVersion: logging.banzaicloud.io/v1beta1
kind: ClusterFlow
metadata:
  creationTimestamp: "2021-10-12T06:21:57Z"
  generation: 4
  managedFields:
  - apiVersion: logging.banzaicloud.io/v1beta1
    fieldsType: FieldsV1
    fieldsV1:
      f:spec:
        .: {}
        f:globalOutputRefs: {}
    manager: rancher
    operation: Update
    time: "2021-10-17T23:38:38Z"
  name: pe-aws
  namespace: cattle-logging-system
  resourceVersion: "19562606"
  uid: bd595b0a-56d0-4c20-b436-020b92ebbb7f
spec:
  globalOutputRefs:
  - pe-aws

# kubectl get clusteroutput -A
NAMESPACE               NAME             ACTIVE   PROBLEMS
cattle-logging-system   pe-aws

# kubectl get clusterflow -A -o wide
NAMESPACE               NAME             ACTIVE   PROBLEMS
cattle-logging-system   pe-aws