This auditing feature is disabled in a standard Kubernetes cluster. This blog is to show how you can enable audit logging for a TKGm cluster.
### include the following in the cluster YAML file
ENABLE_AUDIT_LOGGING: "true"
root@topgun01 [ ~ ]# k describe pod kube-apiserver-tkg-vmc-workload-c03-control-plane-lx4rb -n kube-system
Name: kube-apiserver-tkg-vmc-workload-c03-control-plane-lx4rb
Namespace: kube-system
Priority: 2000001000
Priority Class Name: system-node-critical
Node: tkg-vmc-workload-c03-control-plane-lx4rb/192.168.2.169
Start Time: Mon, 16 Aug 2021 04:30:04 +0000
Labels: component=kube-apiserver
tier=control-plane
Annotations: kubeadm.kubernetes.io/kube-apiserver.advertise-address.endpoint: 192.168.2.169:6443
kubernetes.io/config.hash: 6d516a3c6e6e32410de9f1666966d6d9
kubernetes.io/config.mirror: 6d516a3c6e6e32410de9f1666966d6d9
kubernetes.io/config.seen: 2021-08-16T04:29:57.624729486Z
kubernetes.io/config.source: file
Status: Running
IP: 192.168.2.169
IPs:
IP: 192.168.2.169
Controlled By: Node/tkg-vmc-workload-c03-control-plane-lx4rb
Containers:
kube-apiserver:
Container ID: containerd://80298008a35eff701e893784811069d998e9c29f032aefb572b6e781a15bd2e3
Image: projects.registry.vmware.com/tkg/kube-apiserver:v1.20.5_vmware.1
Image ID: sha256:5d0002e710f5d138cd45100329f18458cdf7df372aaa86f52481bd8ae9ff8583
Port: <none>
Host Port: <none>
Command:
kube-apiserver
--advertise-address=192.168.2.169
--allow-privileged=true
--audit-log-maxage=30
--audit-log-maxbackup=10
--audit-log-maxsize=100
--audit-log-path=/var/log/kubernetes/audit.log
--audit-policy-file=/etc/kubernetes/audit-policy.yaml
--authorization-mode=Node,RBAC
--client-ca-file=/etc/kubernetes/pki/ca.crt
--cloud-provider=external
--enable-admission-plugins=NodeRestriction
--enable-bootstrap-token-auth=true
--etcd-cafile=/etc/kubernetes/pki/etcd/ca.crt
--etcd-certfile=/etc/kubernetes/pki/apiserver-etcd-client.crt
--etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key
--etcd-servers=https://127.0.0.1:2379
--insecure-port=0
--kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt
--kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key
--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
--proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt
--proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key
--requestheader-allowed-names=front-proxy-client
--requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
--requestheader-extra-headers-prefix=X-Remote-Extra-
--requestheader-group-headers=X-Remote-Group
--requestheader-username-headers=X-Remote-User
--secure-port=6443
--service-account-issuer=https://kubernetes.default.svc.cluster.local
--service-account-key-file=/etc/kubernetes/pki/sa.pub
--service-account-signing-key-file=/etc/kubernetes/pki/sa.key
--service-cluster-ip-range=100.221.0.0/16
--tls-cert-file=/etc/kubernetes/pki/apiserver.crt
--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
--tls-private-key-file=/etc/kubernetes/pki/apiserver.key
State: Running
Started: Mon, 16 Aug 2021 04:29:12 +0000
Ready: True
Restart Count: 0
Requests:
cpu: 250m
Liveness: http-get <https://192.168.2.169:6443/livez> delay=10s timeout=15s period=10s #success=1 #failure=8
Readiness: http-get <https://192.168.2.169:6443/readyz> delay=0s timeout=15s period=1s #success=1 #failure=3
Startup: http-get <https://192.168.2.169:6443/livez> delay=10s timeout=15s period=10s #success=1 #failure=48
Environment: <none>
Mounts:
/etc/ca-certificates from etc-ca-certificates (ro)
/etc/kubernetes/audit-policy.yaml from audit-policy (rw)
/etc/kubernetes/pki from k8s-certs (ro)
/etc/ssl/certs from ca-certs (ro)
/usr/local/share/ca-certificates from usr-local-share-ca-certificates (ro)
/usr/share/ca-certificates from usr-share-ca-certificates (ro)
/var/log/kubernetes from audit-logs (rw)
Conditions:
Type Status
Initialized True
Ready True
ContainersReady True
PodScheduled True
Volumes:
audit-logs:
Type: HostPath (bare host directory volume)
Path: /var/log/kubernetes
HostPathType:
audit-policy:
Type: HostPath (bare host directory volume)
Path: /etc/kubernetes/audit-policy.yaml
HostPathType:
ca-certs:
Type: HostPath (bare host directory volume)
Path: /etc/ssl/certs
HostPathType: DirectoryOrCreate
etc-ca-certificates:
Type: HostPath (bare host directory volume)
Path: /etc/ca-certificates
HostPathType: DirectoryOrCreate
k8s-certs:
Type: HostPath (bare host directory volume)
Path: /etc/kubernetes/pki
HostPathType: DirectoryOrCreate
usr-local-share-ca-certificates:
Type: HostPath (bare host directory volume)
Path: /usr/local/share/ca-certificates
HostPathType: DirectoryOrCreate
usr-share-ca-certificates:
Type: HostPath (bare host directory volume)
Path: /usr/share/ca-certificates
HostPathType: DirectoryOrCreate
QoS Class: Burstable
Node-Selectors: <none>
Tolerations: :NoExecute op=Exists
Events: <none>
### audit-policy
root@topgun01 [ ~ ]# ssh [email protected]
The authenticity of host '192.168.2.169 (192.168.2.169)' can't be established.
ECDSA key fingerprint is SHA256:wRxtZRypQ9T/oCn3KNjAHRUCoW+5uICoEd+5GiMzK8o.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added '192.168.2.169' (ECDSA) to the list of known hosts.
Welcome to Ubuntu 20.04.2 LTS (GNU/Linux 5.4.0-72-generic x86_64)
* Documentation: <https://help.ubuntu.com>
* Management: <https://landscape.canonical.com>
* Support: <https://ubuntu.com/advantage>
The programs included with the Ubuntu system are free software;
the exact distribution terms for each program are described in the
individual files in /usr/share/doc/*/copyright.
Ubuntu comes with ABSOLUTELY NO WARRANTY, to the extent permitted by
applicable law.
$ cd /etc/
$ ls
adduser.conf cloud deluser.conf gshadow- kernel-img.conf lsb-release netplan polkit-1 resolv.conf subuid update-motd.d
alternatives cni depmod.d gss kubernetes ltrace.conf network popularity-contest.conf rmt subuid- vim
apparmor console-setup dhcp hdparm.conf kubernetes-version machine-id networkd-dispatcher ppp rpc sudoers vmware-tools
apparmor.d containerd dpkg host.conf ldap machine-info NetworkManager profile rsyslog.conf sudoers.d vtrgb
apt crictl.yaml e2scrub.conf hostname ld.so.cache magic networks profile.d rsyslog.d sysconfig wgetrc
audisp cron.d environment hosts ld.so.conf magic.mime newt protocols security sysctl.conf X11
audit cron.daily ethertypes hosts.allow ld.so.conf.d mailcap nsswitch.conf python3 selinux sysctl.d xattr.conf
bash.bashrc cron.hourly fonts hosts.deny legal mailcap.order opt python3.8 services systemd xdg
bash_completion cron.monthly fstab init libaudit.conf manpath.config os-release rc0.d shadow terminfo zsh_command_not_found
bindresvport.blacklist crontab fuse.conf init.d locale.alias mime.types PackageKit rc1.d shadow- timezone
binfmt.d cron.weekly gai.conf initramfs-tools locale.gen mke2fs.conf pam.conf rc2.d shells tmpfiles.d
ca-certificates dbus-1 groff inputrc localtime modprobe.d pam.d rc3.d skel ubuntu-advantage
ca-certificates.conf dconf group iproute2 logcheck modules passwd rc4.d ssh ucf.conf
ca-certificates.conf.dpkg-old debconf.conf group- issue login.defs modules-load.d passwd- rc5.d ssl udev
calendar debian_version grub.d issue.net logrotate.conf mtab perl rc6.d subgid ufw
chrony default gshadow kernel logrotate.d nanorc pm rcS.d subgid- update-manager
$ cd kubernetes
$ ls
admin.conf audit-policy.yaml controller-manager.conf kubelet.conf manifests pki scheduler.conf
$ sudo cat audit-policy.yaml
apiVersion: audit.k8s.io/v1beta1
kind: Policy
rules:
- level: None
users:
- system:serviceaccount:kube-system:kube-proxy
verbs:
- watch
resources:
- group: ""
resources:
- endpoints
- services
- services/status
- level: None
userGroups:
- system:nodes
verbs:
- get
resources:
- group: ""
resources:
- nodes
- nodes/status
- level: None
users:
- system:kube-controller-manager
- system:kube-scheduler
- system:serviceaccount:kube-system:endpoint-controller
verbs:
- get
- update
namespaces:
- kube-system
resources:
- group: ""
resources:
- endpoints
- level: None
users:
- system:apiserver
verbs:
- get
resources:
- group: ""
resources:
- namespaces
- namespaces/status
- namespaces/finalize
- level: None
users:
- system:kube-controller-manager
verbs:
- get
- list
resources:
- group: metrics.k8s.io
- level: None
nonResourceURLs:
- /healthz*
- /version
- /swagger*
- level: None
resources:
- group: ""
resources:
- events
- level: None
userGroups:
- system:serviceaccounts:vmware-system-tmc
verbs:
- get
- list
- watch
- level: None
users:
- system:serviceaccount:kube-system:generic-garbage-collector
verbs:
- get
- list
- watch
- level: Request
userGroups:
- system:nodes
verbs:
- update
- patch
resources:
- group: ""
resources:
- nodes/status
- pods/status
omitStages:
- RequestReceived
- level: Request
users:
- system:serviceaccount:kube-system:namespace-controller
verbs:
- deletecollection
omitStages:
- RequestReceived
- level: Metadata
resources:
- group: ""
resources:
- secrets
- configmaps
- group: authentication.k8s.io
resources:
- tokenreviews
omitStages:
- RequestReceived
- level: Request
verbs:
- get
- list
- watch
resources:
- group: ""
- group: admissionregistration.k8s.io
- group: apiextensions.k8s.io
- group: apiregistration.k8s.io
- group: apps
- group: authentication.k8s.io
- group: authorization.k8s.io
- group: autoscaling
- group: batch
- group: certificates.k8s.io
- group: extensions
- group: metrics.k8s.io
- group: networking.k8s.io
- group: policy
- group: rbac.authorization.k8s.io
- group: settings.k8s.io
- group: storage.k8s.io
omitStages:
- RequestReceived
- level: RequestResponse
resources:
- group: ""
- group: admissionregistration.k8s.io
- group: apiextensions.k8s.io
- group: apiregistration.k8s.io
- group: apps
- group: authentication.k8s.io
- group: authorization.k8s.io
- group: autoscaling
- group: batch
- group: certificates.k8s.io
- group: extensions
- group: metrics.k8s.io
- group: networking.k8s.io
- group: policy
- group: rbac.authorization.k8s.io
- group: settings.k8s.io
- group: storage.k8s.io
omitStages:
- RequestReceived
- level: Metadata
omitStages:
- RequestReceived
$
### log file
$ cd /var/log
$ ls
alternatives.log audit btmp cloud-init-output.log dmesg dpkg.log journal lastlog syslog vmware-network.1.log vmware-vmtoolsd-root.log
antrea auth.log chrony containers dmesg.0 faillog kern.log pods ubuntu-advantage.log vmware-network.log wtmp
apt bootstrap.log cloud-init.log dist-upgrade dmesg.1.gz installer kubernetes private unattended-upgrades vmware-vmsvc-root.log
$ cd kubernetes
$ ls
audit.log
$ cat audit.log
{"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Request","auditID":"b1040807-0861-4c63-89a2-7a45ac0dba96","stage":"ResponseComplete","requestURI":"/apis/storage.k8s.io/v1/volumeattachments?limit=500\\u0026resourceVersion=0","verb":"list","user":{"username":"system:apiserver","uid":"9c314731-8185-4128-957c-430d2f67247a","groups":["system:masters"]},"sourceIPs":["::1"],"userAgent":"kube-apiserver/v1.20.5+vmware.1 (linux/amd64) kubernetes/f455330","objectRef":{"resource":"volumeattachments","apiGroup":"storage.k8s.io","apiVersion":"v1"},"responseStatus":{"metadata":{},"code":200},"requestReceivedTimestamp":"2021-08-16T04:29:17.208270Z","stageTimestamp":"2021-08-16T04:29:17.209443Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":""}}
{"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"79fd4194-8cc1-4916-9f8a-b907232fed3c","stage":"ResponseComplete","requestURI":"/api/v1/namespaces/kube-system/configmaps?limit=500\\u0026resourceVersion=0","verb":"list","user":{"username":"system:apiserver","uid":"9c314731-8185-4128-957c-430d2f67247a","groups":["system:masters"]},"sourceIPs":["::1"],"userAgent":"kube-apiserver/v1.20.5+vmware.1 (linux/amd64) kubernetes/f455330","objectRef":{"resource":"configmaps","namespace":"kube-system","apiVersion":"v1"},"responseStatus":{"metadata":{},"code":200},"requestReceivedTimestamp":"2021-08-16T04:29:17.209995Z","stageTimestamp":"2021-08-16T04:29:17.210320Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":""}}
"log_timestamp" : "1629089515725",
"parsing_failed" : "true",
"date" : "1.62908951399903E9",
"useragent" : "Fluent-Bit",
"requesturi" : "/api/v1/namespaces/default/pods/webserver-559b886555-wqgq5",
"stagetimestamp" : "2021-08-16T04:51:53.998561Z",
"event_type" : "untyped",
"id" : "90ddf53efe3291a44b887cee644d4f65",
"text" : "{\\"date\\":1.62908951399903E9,\\"kind\\":\\"Event\\",\\"apiVersion\\":\\"audit.k8s.io/v1\\",\\"level\\":\\"Request\\",\\"auditID\\":\\"f6ee82f4-95a5-48d7-8f2b-fb722f8d3b75\\",\\"stage\\":\\"ResponseComplete\\",\\"requestURI\\":\\"/api/v1/namespaces/default/pods/webserver-559b886555-wqgq5\\",\\"verb\\":\\"get\\",\\"user_username\\":\\"system:serviceaccount:tanzu-system-logging:fluent-bit\\",\\"user_uid\\":\\"3a578eda-ff45-4881-bb4e-8ca63d6bed82\\",\\"user_groups\\":\\"[\\\\\\"system:serviceaccounts\\\\\\",\\\\\\"system:serviceaccounts:tanzu-system-logging\\\\\\",\\\\\\"system:authenticated\\\\\\"]\\",\\"sourceIPs\\":\\"[\\\\\\"192.168.2.240\\\\\\"]\\",\\"userAgent\\":\\"Fluent-Bit\\",\\"objectRef_resource\\":\\"pods\\",\\"objectRef_namespace\\":\\"default\\",\\"objectRef_name\\":\\"webserver-559b886555-wqgq5\\",\\"objectRef_apiVersion\\":\\"v1\\",\\"responseStatus_code\\":200,\\"requestReceivedTimestamp\\":\\"2021-08-16T04:51:53.995447Z\\",\\"stageTimestamp\\":\\"2021-08-16T04:51:53.998561Z\\",\\"annotations_authentication.k8s.io/legacy-token\\":\\"system:serviceaccount:tanzu-system-logging:fluent-bit\\",\\"annotations_authorization.k8s.io/decision\\":\\"allow\\",\\"annotations_authorization.k8s.io/reason\\":\\"RBAC: allowed by ClusterRoleBinding \\\\\\"fluent-bit-read\\\\\\" of ClusterRole \\\\\\"fluent-bit-read\\\\\\" to ServiceAccount \\\\\\"fluent-bit/tanzu-system-logging\\\\\\"\\",\\"tkg_cluster\\":\\"tkg-vmc-workload-c03\\",\\"tkg_instance\\":\\"dzhang-tanzu\\",\\"ingest_timestamp\\":1629089515725,\\"source\\":null,\\"timestamp\\":1629089515725,\\"log_timestamp\\":1629089515725,\\"event_type\\":\\"untyped\\"}",
"timestamp" : "2021-08-16T04:51:55.725Z",
"auditid" : "f6ee82f4-95a5-48d7-8f2b-fb722f8d3b75",
"level" : "Request",
"kind" : "Event",
"user_username" : "system:serviceaccount:tanzu-system-logging:fluent-bit",
"verb" : "get",
"apiversion" : "audit.k8s.io/v1",
"objectref_name" : "webserver-559b886555-wqgq5",
"tkg_cluster" : "tkg-vmc-workload-c03",
"responsestatus_code" : "200",
"stage" : "ResponseComplete",
"sourceips" : "[\\"192.168.2.240\\"]",
"objectref_resource" : "pods",
"requestreceivedtimestamp" : "2021-08-16T04:51:53.995447Z",
"objectref_namespace" : "default",
"tkg_instance" : "dzhang-tanzu",
"ingest_timestamp" : "1629089515725",
"solr_timestamp" : "1629089516043",
"objectref_apiversion" : "v1",
"user_uid" : "3a578eda-ff45-4881-bb4e-8ca63d6bed82",
"user_groups" : "[\\"system:serviceaccounts\\",\\"system:serviceaccounts:tanzu-system-logging\\",\\"system:authenticated\\"]"
}