EKS · ECR · CircleCI Pro Playbook

$ docker pull ruby:3.3-slim
$ docker history ruby:3.3-slim --format "{{.CreatedBy}}\t{{.Size}}" | head -4
$ docker image ls ruby:3.3-slim

# run it — the writable film:
$ docker run -it ruby:3.3-slim bash
root@c1f2:/# echo "scribble" > /tmp/x     # exists only in THIS container
root@c1f2:/# exit
$ docker run ruby:3.3-slim cat /tmp/x     # fresh container, fresh film

FROM ruby:3.3-slim

WORKDIR /app

# 1. dependency manifests FIRST — this layer rarely changes:
COPY Gemfile Gemfile.lock ./
RUN bundle install --jobs 4

# 2. app code LAST — changes every commit, busts only this layer:
COPY . .

# 3. never root:
RUN useradd -m app && chown -R app:app /app
USER app

EXPOSE 3000
CMD ["bin/rails", "server", "-b", "0.0.0.0"]

# ---------- stage 1: the workshop ----------
FROM ruby:3.3 AS build
WORKDIR /app
RUN apt-get update && apt-get install -y build-essential libpq-dev nodejs
COPY Gemfile Gemfile.lock ./
RUN bundle config set without "development test" && bundle install
COPY . .
RUN SECRET_KEY_BASE=dummy bin/rails assets:precompile

# ---------- stage 2: the showroom ----------
FROM ruby:3.3-slim
WORKDIR /app
RUN apt-get update && apt-get install -y libpq5 && rm -rf /var/lib/apt/lists/*
COPY --from=build /usr/local/bundle /usr/local/bundle
COPY --from=build /app /app
USER 1000
CMD ["bin/rails", "server", "-b", "0.0.0.0"]

.git
.env*
log/*
tmp/*
node_modules
storage/*
coverage
*.md
.dockerignore
Dockerfile

# CI builds ONE image, labels it three ways (labels are free — same digest):
$ docker build -t shop:${CIRCLE_SHA1} .
$ docker tag shop:${CIRCLE_SHA1}  $ECR/shop:${CIRCLE_SHA1}      # forever tag
$ docker tag shop:${CIRCLE_SHA1}  $ECR/shop:main                # branch pointer
$ docker tag shop:${CIRCLE_SHA1}  $ECR/shop:v1.42.0             # release tag

# deploys reference the SHA — always:
$ kubectl set image deploy/shop app=$ECR/shop:8f3ab12
# rollback = pointing at the previous SHA. No rebuild.

# what's REALLY running (tags can lie, digests can't):
$ kubectl get pod shop-abc -o jsonpath='{.status.containerStatuses[0].imageID}'

# one-time: create the repo
$ aws ecr create-repository --repository-name shop \
    --image-scanning-configuration scanOnPush=true \
    --image-tag-mutability IMMUTABLE

# the auth dance — badge → 12h docker credential:
$ aws ecr get-login-password --region ap-south-1 \
  | docker login --username AWS --password-stdin \
    203856262582.dkr.ecr.ap-south-1.amazonaws.com

# push — the registry hostname IS part of the tag:
$ docker tag shop:8f3ab12 203856262582.dkr.ecr.ap-south-1.amazonaws.com/shop:8f3ab12
$ docker push 203856262582.dkr.ecr.ap-south-1.amazonaws.com/shop:8f3ab12

$ aws ecr describe-images --repository-name shop \
    --query "imageDetails[].{tags:imageTags,size:imageSizeInBytes}" --output table

{
  "rules": [
    {
      "rulePriority": 1,
      "description": "expire untagged images after 14 days",
      "selection": {
        "tagStatus": "untagged",
        "countType": "sinceImagePushed",
        "countUnit": "days", "countNumber": 14
      },
      "action": { "type": "expire" }
    },
    {
      "rulePriority": 2,
      "description": "keep last 50 SHA builds",
      "selection": {
        "tagStatus": "tagged", "tagPrefixList": ["sha-"],
        "countType": "imageCountMoreThan", "countNumber": 50
      },
      "action": { "type": "expire" }
    }
  ]
}

# basic: scan on push (set at repo creation — topic 6)
# enhanced (Amazon Inspector): continuous rescanning as new CVEs publish
$ aws ecr put-registry-scanning-configuration \
    --scan-type ENHANCED \
    --rules '[{"scanFrequency":"CONTINUOUS_SCAN","repositoryFilters":[{"filter":"*","filterType":"WILDCARD"}]}]'

$ aws ecr describe-image-scan-findings \
    --repository-name shop --image-id imageTag=8f3ab12 \
    --query "imageScanFindings.findingSeverityCounts"

# gate the pipeline on it (topic 60):
$ SEV=$(aws ecr describe-image-scan-findings ... \
        --query "imageScanFindings.findingSeverityCounts.CRITICAL" --output text)
$ [ "$SEV" = "None" ] || { echo "CRITICAL CVEs — blocking deploy"; exit 1; }

# 1. immutable tags — reject overwrites:
$ aws ecr put-image-tag-mutability --repository-name shop \
    --image-tag-mutability IMMUTABLE
$ docker push $ECR/shop:v1.42.0     # second push of same tag:
# → tag invalid: The image tag 'v1.42.0' already exists ✋

# 2. cross-region replication (registry-level, one-time):
$ aws ecr put-replication-configuration --replication-configuration \
  '{"rules":[{"destinations":[{"region":"ap-southeast-1","registryId":"203856262582"}]}]}'

# 3. pull-through cache — stop depending on Docker Hub availability:
$ aws ecr create-pull-through-cache-rule \
    --ecr-repository-prefix dockerhub \
    --upstream-registry-url registry-1.docker.io
# then: FROM 2038...amazonaws.com/dockerhub/library/ruby:3.3-slim

# run the EXACT prod image, mapping ports and env:
$ docker run --rm -p 3000:3000 \
    -e DATABASE_URL=postgres://localhost:5432/dev \
    -e RAILS_ENV=production \
    $ECR/shop:8f3ab12

# poke inside a RUNNING container:
$ docker exec -it shop-local bash
root@a1:/app# env | grep RAILS

# bypass the entrypoint entirely — "just give me a shell in the image":
$ docker run --rm -it --entrypoint bash $ECR/shop:8f3ab12

# the forensic trio:
$ docker logs -f shop-local
$ docker inspect shop-local --format '{{.State.ExitCode}} {{.State.OOMKilled}}'
$ docker stats --no-stream

$ aws eks describe-cluster --name prod --query \
    "cluster.{ver:version,endpoint:endpoint,status:status}"
$ kubectl get nodes -o wide          # YOUR side: EC2 instances (or Fargate)
$ kubectl get pods -n kube-system    # the shared frontier:
NAME                        # aws-node (CNI), coredns, kube-proxy, csi drivers
                            # — run on YOUR nodes, shipped as EKS add-ons (topic 43)

# control plane logs land in CloudWatch when enabled (topic 45):
$ aws eks update-cluster-config --name prod \
    --logging '{"clusterLogging":[{"types":["api","audit","authenticator"],"enabled":true}]}'

apiVersion: eksctl.io/v1alpha5
kind: ClusterConfig
metadata:
  name: prod
  region: ap-south-1
  version: "1.32"
iam:
  withOIDC: true                  # enables IRSA (topic 38) — always yes
managedNodeGroups:
  - name: general
    instanceTypes: [m6i.large, m6a.large]
    minSize: 2
    maxSize: 10
    desiredCapacity: 3
    privateNetworking: true       # nodes in private subnets
addons:
  - { name: vpc-cni }
  - { name: coredns }
  - { name: kube-proxy }
  - { name: aws-ebs-csi-driver }

$ aws eks update-kubeconfig --name prod --region ap-south-1
Updated context arn:aws:eks:ap-south-1:2038…:cluster/prod in ~/.kube/config

# the keyring:
$ kubectl config get-contexts
CURRENT   NAME                          CLUSTER
*         …cluster/prod                 prod
          …cluster/staging              staging
$ kubectl config use-context …cluster/staging     # swap cards (or: kubectx)

# what the kubeconfig actually does per request:
#   exec: aws eks get-token --cluster-name prod
#   → short-lived token, signed by YOUR IAM identity
$ aws sts get-caller-identity      # ← "who am I" — the first auth-debug command

# grant a role cluster-admin (the break-glass/platform team entry):
$ aws eks create-access-entry --cluster-name prod \
    --principal-arn arn:aws:iam::2038…:role/platform-admin
$ aws eks associate-access-policy --cluster-name prod \
    --principal-arn arn:aws:iam::2038…:role/platform-admin \
    --policy-arn arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy \
    --access-scope type=cluster

# grant CI a NARROW entry — mapped to k8s RBAC (topic 27's deployer Role):
$ aws eks create-access-entry --cluster-name prod \
    --principal-arn arn:aws:iam::2038…:role/circleci-deployer \
    --kubernetes-groups deployers        # ← RoleBindings bind this group

$ aws eks list-access-entries --cluster-name prod

# managed node group (in cluster.yaml — topic 34):
managedNodeGroups:
  - name: workers-spot
    instanceTypes: [m6i.large, m6a.large, m5.large]   # diversity = spot safety
    spot: true                                        # ~70% off (topic 46)
    labels: { workload: batch }
    taints: [{ key: spot, value: "true", effect: NoSchedule }]   # topic 28!

# Fargate profile — pods matching land on Fargate automatically:
fargateProfiles:
  - name: cron-stuff
    selectors:
      - namespace: cron          # every pod in ns "cron" → its own micro-VM

# create role + trust + ServiceAccount annotation in one shot:
$ eksctl create iamserviceaccount --cluster prod \
    --namespace default --name shop \
    --attach-policy-arn arn:aws:iam::2038…:policy/shop-s3-readwrite \
    --approve

# newer alternative — EKS Pod Identity (no OIDC ceremony):
$ aws eks create-pod-identity-association --cluster-name prod \
    --namespace default --service-account shop \
    --role-arn arn:aws:iam::2038…:role/shop-s3

spec:
  serviceAccountName: shop      # ← that's it. SDKs discover the rest.
  containers:
    - name: app
      image: $ECR/shop:8f3ab12  # aws-sdk / aws cli inside: just works

$ kubectl get pods -o wide | head -3
shop-7f9b-2xkcd   Running   10.0.42.17    ip-10-0-40-113…
shop-7f9b-8mzpq   Running   10.0.42.31    ip-10-0-40-113…
                            ↑ real subnet IPs, same range as the node

# the ENI math — why nodes have pod limits:
# m6i.large: 3 ENIs × 10 IPs ≈ 29 pods max, regardless of CPU/mem!
$ kubectl describe node ip-10-0-40-113… | grep -A1 "Allocatable" | grep pods

# because pods are VPC citizens, this works:
#   RDS security group: allow 5432 FROM sg-cluster-nodes
# and prefix delegation raises the density ceiling:
$ kubectl set env ds/aws-node -n kube-system ENABLE_PREFIX_DELEGATION=true

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: shop
  annotations:
    alb.ingress.kubernetes.io/scheme: internet-facing
    alb.ingress.kubernetes.io/target-type: ip          # ALB → pod IPs directly (topic 39!)
    alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443},{"HTTP":80}]'
    alb.ingress.kubernetes.io/ssl-redirect: "443"
    alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:…:certificate/9a1…
    alb.ingress.kubernetes.io/healthcheck-path: /up
spec:
  ingressClassName: alb
  rules:
    - host: shop.example.com
      http:
        paths:
          - path: /
            pathType: Prefix
            backend: { service: { name: shop, port: { number: 80 } } }

apiVersion: karpenter.sh/v1
kind: NodePool
metadata: { name: general }
spec:
  template:
    spec:
      requirements:
        - { key: karpenter.sh/capacity-type, operator: In, values: [spot, on-demand] }
        - { key: kubernetes.io/arch, operator: In, values: [amd64, arm64] }
        - { key: karpenter.k8s.aws/instance-category, operator: In, values: [m, c, r] }
      nodeClassRef: { name: default }       # AMI/subnets/SGs live here
  disruption:
    consolidationPolicy: WhenEmptyOrUnderutilized
    consolidateAfter: 60s
  limits: { cpu: "500" }                    # cluster-wide brake

# StorageClass — the gp3 default you actually want:
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata: { name: gp3 }
provisioner: ebs.csi.aws.com
parameters: { type: gp3, encrypted: "true" }
volumeBindingMode: WaitForFirstConsumer   # ← bind AFTER scheduling: right AZ!
allowVolumeExpansion: true
---
# PVC — a Deployment-attached disk (single replica!) or via
# volumeClaimTemplates in a StatefulSet (topic 25):
apiVersion: v1
kind: PersistentVolumeClaim
metadata: { name: reports-cache }
spec:
  accessModes: [ReadWriteOnce]            # EBS = one node at a time
  storageClassName: gp3
  resources: { requests: { storage: 50Gi } }
# EFS: accessModes: [ReadWriteMany], provisioner: efs.csi.aws.com

$ aws eks list-addons --cluster-name prod
{ "addons": ["vpc-cni", "coredns", "kube-proxy", "aws-ebs-csi-driver"] }

# what versions exist / what's compatible with my k8s version?
$ aws eks describe-addon-versions --addon-name coredns \
    --kubernetes-version 1.32 \
    --query "addons[0].addonVersions[0].addonVersion"

# upgrade one — with explicit conflict strategy:
$ aws eks update-addon --cluster-name prod --addon-name coredns \
    --addon-version v1.11.3-eksbuild.2 \
    --resolve-conflicts PRESERVE          # keep my custom replica count etc.

$ kubectl get pods -n kube-system         # watch the rolling replacement

# 0. preflight — what breaks? EKS runs upgrade insights for you:
$ aws eks list-insights --cluster-name prod \
    --filter kubernetesVersions=1.33 --query "insights[].{n:name,s:insightStatus.status}"
# (kubent/pluto scan manifests for deprecated APIs too)

# 1. control plane (one minor at a time — 1.32 → 1.33):
$ aws eks update-cluster-version --name prod --kubernetes-version 1.33

# 2. add-ons to versions compatible with 1.33 (topic 43)

# 3. nodes — managed groups roll respecting PDBs:
$ eksctl upgrade nodegroup --cluster prod --name general --kubernetes-version 1.33
#    (Karpenter nodes: drift replacement handles it — topic 41)

# 4. verify:
$ kubectl get nodes    # all v1.33.x
$ kubectl get pods -A | grep -v Running | grep -v Completed

# logs + metrics in one add-on (Container Insights):
$ aws eks create-addon --cluster-name prod \
    --addon-name amazon-cloudwatch-observability
# → Fluent Bit DaemonSet (stdout/stderr → CloudWatch Logs)
# → CloudWatch agent (node/pod metrics → Container Insights)

# the app side of the contract: log to STDOUT as JSON, one event per line
# (Rails: config.logger = stdout, lograge/json — no log FILES in containers)

# then questions become queries (Logs Insights):
fields @timestamp, kubernetes.pod_name, log
| filter kubernetes.labels.app = "shop" and log like /ERROR/
| sort @timestamp desc | limit 50

# the metrics that page you (alarms on):
# pod restarts spiking · deployment replicas < desired · node NotReady
# CPU throttling (topic 19!) · 5xx on the ALB (topic 40) · PVC near-full

# 1. requests vs reality — the #1 leak:
$ kubectl top pods -A --sort-by=memory | head          # actual
$ kubectl get pods -A -o custom-columns=\
NAME:.metadata.name,REQ_MEM:.spec.containers[0].resources.requests.memory | head
# requested 2Gi, using 300Mi × 40 replicas = ~70Gi of phantom desks

# 2. node utilization — are floors lit for nobody?
$ kubectl top nodes
# consistently <40%? Karpenter consolidation (topic 41) or smaller instances

# 3. capacity mix — what % rides spot?
$ kubectl get nodes -L karpenter.sh/capacity-type --no-headers | \
    awk '{print $NF}' | sort | uniq -c

# 4. visibility — per-namespace/label cost attribution:
#    Kubecost / CloudWatch Container Insights / split-cost-allocation in CUR