list GPU allocatable resources

kubectl get node -o go-template='{{printf "%-40s %20s %20s\n" "NODE" "CAPACITY" "ALLOCATABLE"}}{{range $index,$pod := .items}}{{$capacity := index .status.capacity "nvidia.com/gpu"}}{{$alloc := index .status.allocatable "nvidia.com/gpu"}}{{printf "%-40s %20s %20s\n" .metadata.name $capacity $alloc}}{{end}}'

list all api resources

kubectl api-resources --verbs=list --namespaced -o name 

# useful for debugging
kubectl api-resources --verbs=list --namespaced -o name | xargs -n 1 -I{} /bin/sh -c "kubectl get {} -o yaml || true; kubectl get {} --all-namespaces -o yaml || true"| tee out.log

ping all pod ips at once

kubectl get po -o json | jq .items[].status.podIP -r | fping

Force delete stuck namespace

kubectl get namespace "foo-restricted" -o json \
        | tr -d "\n" | sed "s/\"finalizers\": \[[^]]\+\]/\"finalizers\": []/" \
        | kubectl replace --raw /api/v1/namespaces/foo-restricted/finalize -f -

Run pod in namespace

kubectl run -i --tty busybox --image=busybox:latest --restart Never --rm -n $NAMESPACE -- sh

Selecting pods

kubectl get pod -A
kubectl get pod --show-labels
kubectl get pod -l foo=bar,baz=ban

Deleting pods

No time to waste?

kubectl delete pod --grace-period=0 --force

Clean up

Cleanup all terminating pods

kubectl get pod|grep Terminating|awk '{print $1}'|xargs -n 1 echo kubectl delete pod --grace-period=0 --force

Cleanup all non-running pods

kubectl get pods --field-selector=status.phase!=Running -o jsonpath="{range .items[?(@.status.phase != 'Running')]}kubectl delete pod -n {.metadata.namespace}{' '}{.metadata.name}{'\n'}{end}" | sh -

kubectl get pod -A --no-headers |grep -v Running | awk '{print "kubectl delete pod -n "$1" "$2}'|sh -

Tail all logs

This script has helped me so many times quickly finding issues somewhere in k8s containers.

#!/bin/bash

function cleanup
{
    kill -9 $(jobs -p)
}
trap "cleanup" 2

function prefix
{
    while read line; do
        echo "$1$line";
    done < /dev/stdin
}

function kubectl_log
{
    typeset namespace="$1"
    typeset pod="$2"
    shift 2
    for i in $(seq 1 $#); do
        typeset container="$1"
        shift
        kubectl logs -n $namespace $pod -f $container | prefix "[ $namespace - $pod - $container ] - " &
    done
}

while read line; do
    kubectl_log $line
done < <(kubectl get pods -A -o jsonpath="{range .items[*]}{.metadata.namespace} {.metadata.name} {..containerStatuses..name}{'\n'}{end}")
wait

List all mounted host paths

#!/bin/bash

function prefix
{
    while read line; do
        echo "$1$line";
    done < /dev/stdin
}

echo "----------------------------"
echo checking direct mounts 
echo "----------------------------"
while read ns; do
    if [[ "$ns" == "kube-system" ]]; then
        continue
    fi

    kubectl get pod -n $ns -o jsonpath="{range ..volumes[?(@.hostPath.path != '')]}{.hostPath.path}{'\n'}{end}" | sort | uniq | prefix "$ns - "

done < <(kubectl get ns -o jsonpath="{range .items[*]}{.metadata.name}{'\n'}{end}")

echo "----------------------------"
echo checking persistent volumes $ns
echo "----------------------------"
kubectl get pv -o yaml -o jsonpath="{range .items[*]}{..hostPath.path}{'\n'}{end}" | sort | uniq

kubeadm

kubeadm token create --print-join-command

security

kubectl auth can-i --list --namespace=test-restricted

#

[root@nas kube]# kubectl get no Unable to connect to the server: x509: certificate is valid for 10.96.0.1, 192.168.2.2, not 10.75.99.139

[root@nas kube]# kubectl --insecure-skip-tls-verify get no ...

Leader election

root@node001:~> kubectl describe ep -n kube-system kube-controller-manager
Name:         kube-controller-manager
Namespace:    kube-system
Labels:       <none>
Annotations:  control-plane.alpha.kubernetes.io/leader:
                {"holderIdentity":"rb-haha-b_35d43390-6650-4907-99c5-57d1e6455496","leaseDurationSeconds":15,"acquireTime":"2021-01-26T13:46:55Z","renewTi...
Subsets:
Events:
  Type    Reason          Age   From                     Message
  ----    ------          ----  ----                     -------
  Normal  LeaderElection  17m   kube-controller-manager  rb-haha-b_35d43390-6650-4907-99c5-57d1e6455496 became leader

List containers inside docker registry

curl -k https://<IP>:5000/v2/_catalog | jq .

Save and Load containers

docker save myimage:latest | gzip > myimage_latest.tar.gz

docker load < myimage_latest.tar.gz