Secrets Automation for Kubernetes

The Secrets Problem

Kubernetes Secrets are base64-encoded, not encrypted by default. Stored in etcd without additional controls, they are readable by anyone with etcd access or sufficient RBAC. The goal of secrets automation is to ensure secrets are: never stored in Git, encrypted at rest and in transit, rotated automatically, and auditable.

Secrets Solutions Comparison

HashiCorp Vault

Vault is the industry-standard secrets manager — a dedicated, auditable, policy-governed store for secrets, certificates, and dynamic credentials. In a Kubernetes platform context, Vault is the authoritative source of truth that all other secrets delivery mechanisms (ESO, CSI, Agent) pull from.

Vault HA Install on Kubernetes

helm repo add hashicorp https://helm.releases.hashicorp.com
helm repo update

helm install vault hashicorp/vault \
  --namespace vault \
  --create-namespace \
  --version 0.28.1 \
  -f vault-values.yamlCopy

# vault-values.yaml
global:
  enabled: true
  tlsDisable: false

server:
  ha:
    enabled: true
    replicas: 3
    raft:
      enabled: true   # integrated Raft storage (no external Consul needed)
      config: |
        ui = true
        listener "tcp" {
          tls_disable = 0
          address = "[::]:8200"
          cluster_address = "[::]:8201"
          tls_cert_file = "/vault/userconfig/vault-tls/tls.crt"
          tls_key_file  = "/vault/userconfig/vault-tls/tls.key"
        }
        storage "raft" {
          path    = "/vault/data"
          node_id = "HOSTNAME"
          retry_join {
            leader_api_addr = "https://vault-0.vault-internal:8200"
          }
          retry_join {
            leader_api_addr = "https://vault-1.vault-internal:8200"
          }
          retry_join {
            leader_api_addr = "https://vault-2.vault-internal:8200"
          }
        }
        service_registration "kubernetes" {}
  resources:
    requests:
      cpu: 250m
      memory: 256Mi
    limits:
      memory: 512Mi
  affinity: |
    podAntiAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
      - labelSelector:
          matchLabels:
            app.kubernetes.io/name: vault
            component: server
        topologyKey: topology.kubernetes.io/zone
  auditStorage:
    enabled: true
    size: 10Gi

ui:
  enabled: true

injector:
  enabled: true   # Vault Agent Injector for pod annotation-based deliveryCopy

Vault Init and Unseal

# Initialize Vault (first time only)
kubectl exec -n vault vault-0 -- vault operator init \
  -key-shares=5 \
  -key-threshold=3 \
  -format=json > vault-init.json

# Store unseal keys in AWS SSM or Secrets Manager — never in Git
aws secretsmanager create-secret \
  --name vault/unseal-keys \
  --secret-string "$(cat vault-init.json)"

# Unseal each replica (requires 3 of 5 keys)
for i in 0 1 2; do
  kubectl exec -n vault vault-$i -- vault operator unseal \
    $(cat vault-init.json | jq -r '.unseal_keys_b64[0]')
  kubectl exec -n vault vault-$i -- vault operator unseal \
    $(cat vault-init.json | jq -r '.unseal_keys_b64[1]')
  kubectl exec -n vault vault-$i -- vault operator unseal \
    $(cat vault-init.json | jq -r '.unseal_keys_b64[2]')
done

# Auto-unseal in production: use AWS KMS or GCP Cloud KMS
# vault-values.yaml:
# server.extraEnvironmentVars:
#   VAULT_SEAL_TYPE: awskms
#   VAULT_AWSKMS_SEAL_KEY_ID: arn:aws:kms:us-east-1:123456789:key/mrk-abcCopy

Vault Policies

# Enable KV secrets engine v2
vault secrets enable -path=secret kv-v2

# Create policy for payments team
cat <<'EOF' | vault policy write payments-app -
path "secret/data/payments/*" {
  capabilities = ["read"]
}
path "secret/metadata/payments/*" {
  capabilities = ["list"]
}
# Allow reading database creds (dynamic secrets)
path "database/creds/payments-readwrite" {
  capabilities = ["read"]
}
EOF

# Write a secret
vault kv put secret/payments/db-password \
  password=S3cur3P@ss! \
  username=payments_app

# Read a secret
vault kv get -format=json secret/payments/db-password | jq .data.dataCopy

Vault + Kubernetes Integration

The Vault Kubernetes auth method allows pods to authenticate to Vault using their ServiceAccount JWT token. No static credentials needed — the pod's identity proves itself via the K8s TokenReview API.

Enable Kubernetes Auth Method

# Enable the auth method
vault auth enable kubernetes

# Configure it with the cluster's API server
vault write auth/kubernetes/config \
  kubernetes_host="https://kubernetes.default.svc.cluster.local:443" \
  kubernetes_ca_cert=@/var/run/secrets/kubernetes.io/serviceaccount/ca.crt \
  token_reviewer_jwt=@/var/run/secrets/kubernetes.io/serviceaccount/token \
  issuer="https://kubernetes.default.svc.cluster.local"

# Create a role binding ServiceAccount → Vault policy
vault write auth/kubernetes/role/payments-app \
  bound_service_account_names=payments-api \
  bound_service_account_namespaces=payments-api-production \
  policies=payments-app \
  ttl=1h   # token TTL; Vault Agent renews automaticallyCopy

Vault Agent Injector (Sidecar)

apiVersion: apps/v1
kind: Deployment
metadata:
  name: payments-api
  namespace: payments-api-production
spec:
  template:
    metadata:
      annotations:
        # Vault Agent Injector annotations
        vault.hashicorp.com/agent-inject: "true"
        vault.hashicorp.com/role: "payments-app"
        vault.hashicorp.com/tls-skip-verify: "false"
        vault.hashicorp.com/ca-cert: "/vault/tls/ca.crt"
        # Inject secret as file at /vault/secrets/db-password
        vault.hashicorp.com/agent-inject-secret-db-password: "secret/data/payments/db-password"
        vault.hashicorp.com/agent-inject-template-db-password: |
          {{- with secret "secret/data/payments/db-password" -}}
          export DB_PASSWORD="{{ .Data.data.password }}"
          export DB_USERNAME="{{ .Data.data.username }}"
          {{- end }}
        # Pre-populate (init container) + keep alive (sidecar for renewal)
        vault.hashicorp.com/agent-pre-populate-only: "false"
    spec:
      serviceAccountName: payments-api
      containers:
      - name: payments-api
        command: ["/bin/sh","-c","source /vault/secrets/db-password && ./payments-api"]Copy

Vault Agent Config (Direct, without Injector)

# ConfigMap for Vault Agent running as init + sidecar
apiVersion: v1
kind: ConfigMap
metadata:
  name: vault-agent-config
  namespace: payments-api-production
data:
  config.hcl: |
    vault {
      address = "https://vault.vault.svc.cluster.local:8200"
      ca_cert = "/vault/tls/ca.crt"
    }
    auto_auth {
      method "kubernetes" {
        mount_path = "auth/kubernetes"
        config = {
          role = "payments-app"
        }
      }
    }
    template {
      source      = "/vault/templates/db.ctmpl"
      destination = "/vault/secrets/db.env"
      command     = "kill -HUP $(pidof payments-api)"  # reload on rotation
    }
    template_config {
      static_secret_render_interval = "5m"
      exit_on_retry_failure         = true
    }Copy

Dynamic Secrets

Dynamic secrets are credentials that Vault generates on-demand, tied to a lease TTL. When the lease expires, the credentials are revoked. No long-lived database passwords — each pod gets unique, time-limited credentials.

Vault Database Secrets Engine

# Enable the database secrets engine
vault secrets enable database

# Configure PostgreSQL connection
vault write database/config/payments-postgres \
  plugin_name=postgresql-database-plugin \
  allowed_roles="payments-readwrite,payments-readonly" \
  connection_url="postgresql://{{username}}:{{password}}@payments-db.internal:5432/payments?sslmode=require" \
  username="vault_admin" \
  password="vault_admin_password" \
  password_authentication="scram-sha-256"

# Rotate the root credentials (Vault now owns them — humans cannot log in directly)
vault write -force database/rotate-root/payments-postgres

# Create role for app credentials (TTL 1h, max 4h)
vault write database/roles/payments-readwrite \
  db_name=payments-postgres \
  creation_statements="CREATE ROLE \"{{name}}\" WITH LOGIN PASSWORD '{{password}}' VALID UNTIL '{{expiration}}';
    GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO \"{{name}}\";
    GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO \"{{name}}\";" \
  revocation_statements="REVOKE ALL ON ALL TABLES IN SCHEMA public FROM \"{{name}}\";
    DROP ROLE IF EXISTS \"{{name}}\";" \
  default_ttl="1h" \
  max_ttl="4h"

# Test: generate credentials on demand
vault read database/creds/payments-readwrite
# Key                Value
# ---                -----
# lease_id           database/creds/payments-readwrite/abc123
# lease_duration     1h
# username           v-kubrnts-payments-abc123
# password           A1b2C3d4-randomly-generatedCopy

Dynamic AWS IAM Credentials

# Vault AWS secrets engine: generate temporary AWS access keys
vault secrets enable aws

vault write aws/config/root \
  region=us-east-1 \
  iam_endpoint=https://iam.amazonaws.com

# Role: generate IAM credentials for S3 access
vault write aws/roles/payments-s3-role \
  credential_type=iam_user \
  policy_document='{
    "Version":"2012-10-17",
    "Statement":[{
      "Effect":"Allow",
      "Action":["s3:GetObject","s3:PutObject","s3:ListBucket"],
      "Resource":["arn:aws:s3:::payments-documents/*","arn:aws:s3:::payments-documents"]
    }]
  }' \
  default_ttl=1h \
  max_ttl=4h

# Generate credentials
vault read aws/creds/payments-s3-roleCopy

External Secrets Operator

ESO watches ExternalSecret resources and syncs secrets from external stores (Vault, AWS Secrets Manager, GCP Secret Manager, Azure Key Vault) into Kubernetes Secrets. It supports automatic refresh, secret templating, and multi-store federation.

Install ESO

helm repo add external-secrets https://charts.external-secrets.io
helm repo update

helm install external-secrets external-secrets/external-secrets \
  --namespace external-secrets \
  --create-namespace \
  --version 0.10.3 \
  --set installCRDs=true \
  --set replicaCount=2 \
  --set metrics.service.enabled=true \
  --set webhook.replicaCount=2Copy

SecretStore: Vault Backend

# ClusterSecretStore — available to all namespaces
apiVersion: external-secrets.io/v1beta1
kind: ClusterSecretStore
metadata:
  name: vault-backend
spec:
  provider:
    vault:
      server: "https://vault.vault.svc.cluster.local:8200"
      path: "secret"
      version: "v2"
      caBundle: "LS0tLS1CRUdJT..."   # base64-encoded Vault CA cert
      auth:
        kubernetes:
          mountPath: "kubernetes"
          role: "external-secrets"
          serviceAccountRef:
            name: external-secrets
            namespace: external-secretsCopy

SecretStore: AWS Secrets Manager

apiVersion: external-secrets.io/v1beta1
kind: ClusterSecretStore
metadata:
  name: aws-secrets-manager
spec:
  provider:
    aws:
      service: SecretsManager
      region: us-east-1
      auth:
        jwt:
          serviceAccountRef:
            name: external-secrets
            namespace: external-secrets
            # ServiceAccount must be annotated with IRSA role ARNCopy

ExternalSecret: Sync from Vault

apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
  name: payments-db-secret
  namespace: payments-api-production
spec:
  refreshInterval: 1h    # re-sync every hour (picks up rotated secrets)
  secretStoreRef:
    name: vault-backend
    kind: ClusterSecretStore
  target:
    name: payments-db-conn   # name of the K8s Secret to create
    creationPolicy: Owner    # ESO owns the secret; deletes it if ExternalSecret is deleted
    template:
      type: Opaque
      data:
        DATABASE_URL: "postgresql://{{ .username }}:{{ .password }}@{{ .endpoint }}:{{ .port }}/{{ .dbname }}?sslmode=require"
  data:
  - secretKey: username
    remoteRef:
      key: secret/data/payments/db-credentials
      property: username
  - secretKey: password
    remoteRef:
      key: secret/data/payments/db-credentials
      property: password
  - secretKey: endpoint
    remoteRef:
      key: secret/data/payments/db-credentials
      property: endpoint
  - secretKey: port
    remoteRef:
      key: secret/data/payments/db-credentials
      property: port
  - secretKey: dbname
    remoteRef:
      key: secret/data/payments/db-credentials
      property: dbnameCopy

ExternalSecret: Sync from AWS Secrets Manager

apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
  name: stripe-api-key
  namespace: payments-api-production
spec:
  refreshInterval: 24h
  secretStoreRef:
    name: aws-secrets-manager
    kind: ClusterSecretStore
  target:
    name: stripe-credentials
    creationPolicy: Owner
  data:
  - secretKey: api-key
    remoteRef:
      key: prod/payments/stripe    # ARN or name of the AWS secret
      property: api_key
  - secretKey: webhook-secret
    remoteRef:
      key: prod/payments/stripe
      property: webhook_secretCopy

ExternalSecret Status Check

# Check sync status
kubectl get externalsecrets -n payments-api-production
# NAME                  STORE           REFRESH INTERVAL   STATUS   READY
# payments-db-secret    vault-backend   1h                 Valid    True

# Force immediate refresh
kubectl annotate externalsecret payments-db-secret \
  -n payments-api-production \
  force-sync=$(date +%s) --overwrite

# View ESO operator logs for sync errors
kubectl logs -n external-secrets -l app.kubernetes.io/name=external-secrets -fCopy

Sealed Secrets

Sealed Secrets encrypts K8s Secrets using the cluster's public key, producing a SealedSecret resource that is safe to commit to Git. The controller in the cluster decrypts it using the private key and creates the actual Secret.

Install Sealed Secrets

helm repo add sealed-secrets https://bitnami-labs.github.io/sealed-secrets
helm install sealed-secrets sealed-secrets/sealed-secrets \
  --namespace kube-system \
  --version 2.16.1

# Install kubeseal CLI
brew install kubeseal   # or download from GitHub releasesCopy

Seal a Secret

# Fetch the cluster's public certificate
kubeseal --fetch-cert \
  --controller-namespace=kube-system \
  --controller-name=sealed-secrets > pub-cert.pem

# Create a regular Secret manifest and seal it
kubectl create secret generic stripe-key \
  --namespace=payments-api-production \
  --from-literal=api-key=sk_live_abc123 \
  --dry-run=client -o yaml | \
  kubeseal --cert pub-cert.pem --format yaml > stripe-key-sealed.yaml

# The sealed file is safe to commit to Git
git add stripe-key-sealed.yaml
git commit -m "feat: add Stripe API key sealed secret"Copy

SealedSecret Manifest

# stripe-key-sealed.yaml — safe to store in Git
apiVersion: bitnami.com/v1alpha1
kind: SealedSecret
metadata:
  name: stripe-key
  namespace: payments-api-production
spec:
  encryptedData:
    api-key: AgBy3... # RSA-OAEP encrypted with cluster public key
  template:
    type: Opaque
    metadata:
      name: stripe-key
      namespace: payments-api-productionCopy

SOPS + age for GitOps

SOPS (Secrets OPerationS) encrypts YAML/JSON files in-place, leaving keys readable but values encrypted. Combined with age (or AWS KMS / GCP KMS), it enables multi-cluster GitOps where secrets can be decrypted on any cluster that has the key.

Setup SOPS with age

# Generate age key pair
age-keygen -o age.key
# Public key: age1ql3z7hjy54pw3hyww5ayyfg7zqgvc7w3j2elw8zmrj2kg5sfn9aqmcac8p

# Store private key as K8s Secret (for Flux decryption)
kubectl create secret generic sops-age-key \
  --namespace=flux-system \
  --from-file=age.agekey=age.key

# Store public key — safe to commit
cat age.key | grep "^# public key:" | awk '{print $4}'Copy

.sops.yaml Configuration

# .sops.yaml in repo root — controls which files SOPS encrypts and with which keys
creation_rules:
  # Encrypt all secrets.yaml files with age
  - path_regex: .*/secrets\.yaml$
    age: age1ql3z7hjy54pw3hyww5ayyfg7zqgvc7w3j2elw8zmrj2kg5sfn9aqmcac8p

  # Encrypt production secrets with AWS KMS (more secure, audit trail)
  - path_regex: clusters/production/.*secrets.*\.yaml$
    kms: arn:aws:kms:us-east-1:123456789012:key/mrk-abc123def456

  # Encrypt staging with age key
  - path_regex: clusters/staging/.*secrets.*\.yaml$
    age: age1ql3z7hjy54pw3hyww5ayyfg7zqgvc7w3j2elw8zmrj2kg5sfn9aqmcac8pCopy

Encrypt and Decrypt

# Encrypt a K8s Secret YAML in-place
sops --encrypt --in-place secrets.yaml

# Decrypt (for editing)
sops --decrypt secrets.yaml

# Edit encrypted file directly (decrypts, opens $EDITOR, re-encrypts on save)
sops secrets.yaml

# Verify SOPS-encrypted file structure
# Keys are visible; values are encrypted:
# apiVersion: v1
# kind: Secret
# data:
#   api-key: ENC[AES256_GCM,data:xyz...,iv:...,tag:...,type:str]Copy

Flux SOPS Decryption

# Kustomization with SOPS decryption enabled
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
  name: payments-secrets
  namespace: flux-system
spec:
  interval: 10m
  path: ./clusters/production/payments/secrets
  sourceRef:
    kind: GitRepository
    name: platform-gitops
  decryption:
    provider: sops
    secretRef:
      name: sops-age-key   # contains age.agekey
  prune: trueCopy

CSI Secret Store Driver

The Secrets Store CSI Driver mounts external secrets as files into pods — without creating a Kubernetes Secret in etcd. Secrets exist only in the pod's tmpfs mount. Combined with auto-rotation, the file is updated in-place when the upstream secret rotates.

Install CSI Driver + AWS Provider

helm repo add secrets-store-csi-driver https://kubernetes-sigs.github.io/secrets-store-csi-driver/charts
helm install csi-secrets-store secrets-store-csi-driver/secrets-store-csi-driver \
  --namespace kube-system \
  --set syncSecret.enabled=true \      # optionally sync to K8s Secret too
  --set enableSecretRotation=true \
  --set rotationPollInterval=2m

# AWS provider for Secrets Manager + Parameter Store
helm repo add aws-secrets-manager https://aws.github.io/secrets-store-csi-driver-provider-aws
helm install secrets-provider-aws aws-secrets-manager/secrets-store-csi-driver-provider-aws \
  --namespace kube-systemCopy

SecretProviderClass

apiVersion: secrets-store.csi.x-k8s.io/v1
kind: SecretProviderClass
metadata:
  name: payments-aws-secrets
  namespace: payments-api-production
spec:
  provider: aws
  parameters:
    objects: |
      - objectName: "prod/payments/stripe"
        objectType: "secretsmanager"
        objectAlias: "stripe-credentials.json"
      - objectName: "/payments/db-password"
        objectType: "ssmparameter"
        objectAlias: "db-password"
  # Optionally sync to K8s Secret as well
  secretObjects:
  - secretName: payments-stripe-secret
    type: Opaque
    data:
    - objectName: "stripe-credentials.json"
      key: credentialsCopy

Pod Mounting CSI Secret

spec:
  serviceAccountName: payments-api   # must have IRSA role for Secrets Manager
  volumes:
  - name: secrets-store
    csi:
      driver: secrets-store.csi.k8s.io
      readOnly: true
      volumeAttributes:
        secretProviderClass: payments-aws-secrets
  containers:
  - name: payments-api
    volumeMounts:
    - name: secrets-store
      mountPath: /mnt/secrets
      readOnly: true
    # /mnt/secrets/stripe-credentials.json — refreshed every 2m
    # /mnt/secrets/db-passwordCopy

cert-manager

cert-manager automates TLS certificate provisioning and renewal — from Let's Encrypt public certificates to internal CA-signed certificates. It eliminates manual certificate management and the "certificate expired" class of outages.

Install cert-manager

helm repo add jetstack https://charts.jetstack.io
helm repo update

helm install cert-manager jetstack/cert-manager \
  --namespace cert-manager \
  --create-namespace \
  --version v1.16.2 \
  --set crds.enabled=true \
  --set replicaCount=2 \
  --set webhook.replicaCount=2 \
  --set cainjector.replicaCount=2 \
  --set prometheus.enabled=trueCopy

ClusterIssuer: Let's Encrypt (ACME DNS-01)

apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: letsencrypt-production
spec:
  acme:
    server: https://acme-v02.api.letsencrypt.org/directory
    email: platform@company.com
    privateKeySecretRef:
      name: letsencrypt-account-key
    solvers:
    - dns01:
        route53:
          region: us-east-1
          role: arn:aws:iam::123456789012:role/cert-manager-route53
          # IRSA: cert-manager ServiceAccount annotated with this role
      selector:
        dnsZones:
        - "company.com"
        - "*.company.com"
---
# Staging issuer for testing
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: letsencrypt-staging
spec:
  acme:
    server: https://acme-staging-v02.api.letsencrypt.org/directory
    email: platform@company.com
    privateKeySecretRef:
      name: letsencrypt-staging-key
    solvers:
    - dns01:
        route53:
          region: us-east-1
          role: arn:aws:iam::123456789012:role/cert-manager-route53Copy

ClusterIssuer: Internal CA

# Step 1: Create root CA as a K8s Secret (or via Vault PKI engine)
kubectl create secret tls internal-ca-key-pair \
  --namespace cert-manager \
  --cert=ca.crt \
  --key=ca.key

# Step 2: CA Issuer backed by the root CA secret
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: internal-ca
spec:
  ca:
    secretName: internal-ca-key-pairCopy

Certificate Resource

apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: payments-api-tls
  namespace: payments-api-production
spec:
  secretName: payments-api-tls-cert   # K8s Secret where cert+key are stored
  issuerRef:
    name: letsencrypt-production
    kind: ClusterIssuer
  commonName: payments-api.company.com
  dnsNames:
  - payments-api.company.com
  - payments.company.com
  duration: 2160h      # 90 days (Let's Encrypt max)
  renewBefore: 360h    # renew 15 days before expiry
  usages:
  - digital signature
  - key encipherment
  - server authCopy

Ingress TLS via cert-manager Annotation

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: payments-api
  namespace: payments-api-production
  annotations:
    cert-manager.io/cluster-issuer: letsencrypt-production
    nginx.ingress.kubernetes.io/ssl-redirect: "true"
spec:
  tls:
  - hosts:
    - payments-api.company.com
    secretName: payments-api-tls-cert   # cert-manager creates this
  rules:
  - host: payments-api.company.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: payments-api
            port:
              number: 80Copy

cert-manager Certificate Status

# Check certificate status
kubectl get certificates -n payments-api-production
# NAME               READY   SECRET                   AGE
# payments-api-tls   True    payments-api-tls-cert    30d

# Check CertificateRequest details
kubectl describe certificaterequest -n payments-api-production

# Check for expiring certificates (within 30 days)
kubectl get certificates -A -o json | jq '
  .items[] | select(
    (.status.notAfter | fromdateiso8601) < (now + 30*86400)
  ) | {
    namespace: .metadata.namespace,
    name: .metadata.name,
    expiry: .status.notAfter
  }'Copy

IRSA & Workload Identity

IRSA (IAM Roles for Service Accounts) on AWS and Workload Identity on GCP eliminate the need for static cloud credentials entirely. Pods authenticate to cloud APIs via short-lived OIDC tokens — no AWS_ACCESS_KEY_ID ever touches a Secret or environment variable.

AWS IRSA Setup

# 1. Create OIDC provider for the cluster
eksctl utils associate-iam-oidc-provider \
  --cluster prod-us-east-1 \
  --region us-east-1 \
  --approve

# Get OIDC issuer URL
OIDC_URL=$(aws eks describe-cluster --name prod-us-east-1 \
  --query "cluster.identity.oidc.issuer" --output text)

# 2. Create IAM role with trust policy
aws iam create-role \
  --role-name payments-api-s3-role \
  --assume-role-policy-document '{
    "Version": "2012-10-17",
    "Statement": [{
      "Effect": "Allow",
      "Principal": {"Federated": "arn:aws:iam::123456789012:oidc-provider/'"${OIDC_URL#https://}"'"},
      "Action": "sts:AssumeRoleWithWebIdentity",
      "Condition": {
        "StringEquals": {
          "'"${OIDC_URL#https://}"':sub": "system:serviceaccount:payments-api-production:payments-api",
          "'"${OIDC_URL#https://}"':aud": "sts.amazonaws.com"
        }
      }
    }]
  }'

# 3. Attach the desired policy
aws iam attach-role-policy \
  --role-name payments-api-s3-role \
  --policy-arn arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccessCopy

Annotate ServiceAccount for IRSA

apiVersion: v1
kind: ServiceAccount
metadata:
  name: payments-api
  namespace: payments-api-production
  annotations:
    eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/payments-api-s3-role
    eks.amazonaws.com/token-expiration: "86400"   # 24h token (default 86400)Copy

GCP Workload Identity

# Bind K8s ServiceAccount to GCP Service Account
gcloud iam service-accounts add-iam-policy-binding \
  payments-api@my-project.iam.gserviceaccount.com \
  --role roles/iam.workloadIdentityUser \
  --member "serviceAccount:my-project.svc.id.goog[payments-api-production/payments-api]"

kubectl annotate serviceaccount payments-api \
  --namespace payments-api-production \
  iam.gke.io/gcp-service-account=payments-api@my-project.iam.gserviceaccount.comCopy

Secret Rotation Patterns

Rotation Strategy by Secret Type

ESO Force Refresh on Rotation

# When a secret is rotated in the external store, force ESO to sync immediately
kubectl annotate externalsecret -n payments-api-production \
  payments-db-secret \
  force-sync=$(date +%s) --overwrite

# Or wait for the refreshInterval to elapse (default: ESO checks every hour)

# Trigger pod restart to pick up new secret (if not using dynamic reloading)
kubectl rollout restart deployment/payments-api -n payments-api-productionCopy

PrometheusRule: Secret Health Alerts

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: secrets-health-alerts
  namespace: monitoring
spec:
  groups:
  - name: secrets.health
    rules:

    # cert-manager: certificate expiring within 14 days
    - alert: CertificateExpiringSoon
      expr: |
        certmanager_certificate_expiration_timestamp_seconds - time() < 14 * 86400
      for: 1h
      labels:
        severity: warning
      annotations:
        summary: "Certificate {{ $labels.namespace }}/{{ $labels.name }} expires in {{ $value | humanizeDuration }}"
        description: "cert-manager should auto-renew. Check: kubectl describe certificate {{ $labels.name }} -n {{ $labels.namespace }}"

    # cert-manager: certificate not ready
    - alert: CertificateNotReady
      expr: |
        certmanager_certificate_ready_status{condition="False"} == 1
      for: 15m
      labels:
        severity: critical
      annotations:
        summary: "Certificate {{ $labels.namespace }}/{{ $labels.name }} is not ready"

    # ESO sync failure
    - alert: ExternalSecretSyncFailed
      expr: |
        externalsecret_status_condition{condition="Ready",status="False"} > 0
      for: 10m
      labels:
        severity: critical
      annotations:
        summary: "ExternalSecret sync failed: {{ $labels.namespace }}/{{ $labels.name }}"
        description: "Check ESO logs and upstream secret store connectivity."

    # Vault: sealed or unavailable
    - alert: VaultSealed
      expr: |
        vault_core_unsealed == 0
      for: 2m
      labels:
        severity: critical
      annotations:
        summary: "Vault instance is sealed"
        description: "Vault at {{ $labels.instance }} is sealed. Unseal immediately or verify auto-unseal KMS."Copy

Best Practices