Major processes:
 - One-off setup/infrequent action for initial configuration & standard policy maintenance, e.g. enable auth method,  
create roles,policies.
 - Pod deployed with JWT token(service account token) injected.
 - API calls to Vault to authenticate from pod, Vault verifies the token and determine policies attached to the Auth token,
and return to client.
 - API calls to Vault with Auth token passed to retrieves required secrets.

export WORKDIR=$(pwd)

cat > ${WORKDIR}/vault-auth-service-account.yaml <<EOF
apiVersion: v1
kind: ServiceAccount
metadata:
  name: vault-auth
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: role-tokenreview-binding
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:auth-delegator
subjects:
  - kind: ServiceAccount
    name: vault-auth
EOF

kubectl apply --filename vault-auth-service-account.yaml -n default

cat > ${WORKDIR}/vault-auth-secret.yaml <<EOF
apiVersion: v1
kind: Secret
metadata:
  name: vault-auth-secret
  annotations:
    kubernetes.io/service-account.name: vault-auth
type: kubernetes.io/service-account-token
EOF

kubectl apply --filename vault-auth-secret.yaml

# service url is also fine, here i changed master's host, and set ingress controller's IP to vault.it-meta.space also.
export VAULT_ADDR=https://vault.it-meta.space
export CLUSTER_ROOT_TOKEN=$(cat ${WORKDIR}/cluster-keys.json | jq -r ".root_token")
vault login token=$CLUSTER_ROOT_TOKEN

root@master:/opt/k8s/config-repo/vault# export VAULT_ADDR=https://vault.it-meta.space
export CLUSTER_ROOT_TOKEN=$(cat ${WORKDIR}/cluster-keys.json | jq -r ".root_token")
vault login token=$CLUSTER_ROOT_TOKEN
Success! You are now authenticated. The token information displayed below
is already stored in the token helper. You do NOT need to run "vault login"
again. Future Vault requests will automatically use this token.

Key                  Value
---                  -----
token                hvs.oemGqKix23q5FMi6qkXiLsqN
token_accessor       BVDmPTEBLgdxLEvQs4p5bXaD
token_duration       ∞
token_renewable      false
token_policies       ["root"]
identity_policies    []
policies             ["root"]

vault secrets enable -path=secret kv
vault kv put secret/data/app/config username="jamie" password="HLWD20230326"
vault kv get secret/data/app/config

export SA_SECRET_NAME=$(kubectl get secrets --output=json \
    | jq -r '.items[].metadata | select(.name|startswith("vault-auth-")).name')
export SA_JWT_TOKEN=$(kubectl get secret $SA_SECRET_NAME \
    --output 'go-template={{ .data.token }}' | base64 --decode)
export SA_CA_CRT=$(kubectl config view --raw --minify --flatten \
    --output 'jsonpath={.clusters[].cluster.certificate-authority-data}' | base64 --decode)
export K8S_HOST=$(kubectl config view --raw --minify --flatten \
    --output 'jsonpath={.clusters[].cluster.server}')
    
vault auth enable kubernetes

vault write auth/kubernetes/config \
     token_reviewer_jwt="$SA_JWT_TOKEN" \
     kubernetes_host="$K8S_HOST" \
     kubernetes_ca_cert="$SA_CA_CRT" \
     issuer="https://kubernetes.default.svc.cluster.local"

vault write auth/kubernetes/role/app-admin \
     bound_service_account_names=vault-auth \
     bound_service_account_namespaces=default \
     token_policies=policy-app-kv-rw \
     ttl=1h

vault write auth/kubernetes/role/app-devops \
     bound_service_account_names=vault-auth \
     bound_service_account_namespaces=default \
     token_policies=policy-app-kv-ro \
     ttl=4h   

vault policy write policy-app-kv-rw - <<EOF
path "secret/data/app/*" {
    capabilities = ["read", "list", "delete", "create", "update"]
}
EOF

vault policy write policy-app-kv-ro - <<EOF
path "secret/data/app/*" {
    capabilities = ["read", "list"]
}
EOF

cat > ${WORKDIR}/test-deployment-with-vault-agent.yaml <<EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx
  labels:
    app: test-vault-agent
spec:
  selector:
    matchLabels:
      app: test-vault-agent
  replicas: 1
  template:
    metadata:
      annotations:
        vault.hashicorp.com/agent-inject: 'true'
        vault.hashicorp.com/role: 'app-devops'
        vault.hashicorp.com/tls-skip-verify: "true"
        vault.hashicorp.com/agent-pre-populate-only: 'true'
        vault.hashicorp.com/agent-inject-secret-app-config.yaml: 'secret/data/app/config'
      labels:
        app: test-vault-agent
    spec:
      serviceAccountName: vault-auth
      containers:
        - name: nginx
          image: nginx:latest
          imagePullPolicy: IfNotPresent
EOF

k apply -f test-deployment-with-vault-agent.yaml

root@master:/opt/k8s/config-repo/vault# kubectl exec $(kubectl get pod -l app=test-vault-agent -o jsonpath="{.items[0].metadata.name}") --container nginx -- cat /vault/secrets/app-config.yaml
password: HLWD20230326
username: jamie