Distributed Systems Pro Playbook

def get_user(id):
    key = f"user:{id}"
    if v := redis.get(key):
        return v                       # hit
    # miss: use a short lock so ONE caller rebuilds, others wait (no herd)
    with redis.lock(f"lock:{key}", timeout=2):
        if v := redis.get(key):        # double-check inside lock
            return v
        v = db.get_user(id)
        redis.set(key, v, ex=60 + random.randint(0, 10))  # jittered TTL
        return v
# jitter prevents thousands of keys expiring the same second (stampede)

# service discovery via DNS (Kubernetes gives every Service a name)
$ nslookup orders.default.svc.cluster.local   # → current healthy pod IPs

# the Service load-balances across ready pods automatically:
#   readinessProbe gates membership → unhealthy pods get no traffic
$ kubectl get endpoints orders           # the live set behind the name
# L7 balancers add: least-connections, sticky sessions, retries, TLS

import time
class TokenBucket:
    def __init__(self, rate, capacity):
        self.rate, self.cap = rate, capacity
        self.tokens, self.ts = capacity, time.monotonic()
    def allow(self):
        now = time.monotonic()
        self.tokens = min(self.cap, self.tokens + (now - self.ts) * self.rate)
        self.ts = now
        if self.tokens >= 1:
            self.tokens -= 1; return True     # allow
        return False                          # reject → return 429
# backpressure: use a BOUNDED queue; when full, block or shed, never grow forever

breaker = CircuitBreaker(fail_max=5, reset_timeout=30)

def call_downstream(req):
    if breaker.is_open():
        raise ServiceUnavailable("circuit open — failing fast")
    for attempt in range(3):
        try:
            return http.post(url, req, timeout=2.0)   # ALWAYS a timeout
        except (Timeout, ConnErr):
            breaker.record_failure()
            time.sleep((2 ** attempt) + random.random())  # backoff + jitter
    raise
# retry ONLY idempotent calls (topic 8); jitter avoids synchronized retries

# ✗ shared pool: one slow dep starves everything
shared = ThreadPool(50)

# ✓ bulkheads: isolate so a flood is contained
pools = {
    "payments": ThreadPool(20),    # payments can use at most 20
    "search":   ThreadPool(20),    # its own budget
    "profile":  ThreadPool(10),
}
def call(dep, fn):
    return pools[dep].submit(fn)   # payments saturating its 20 can't
                                   # starve search or profile
# combine with timeouts + breakers (topic 18); ask "if this dies, what else?"

# edge: start a trace; every downstream call carries the context
with tracer.start_span("checkout") as span:
    span.set_attribute("user.id", user.id)
    headers = {}
    inject(headers)                      # W3C traceparent header
    payment.post(url, req, headers=headers)   # child span, same trace_id
# each service: extract(headers) → continue the SAME trace
# logs include trace_id → grep one id to see the whole request

# CONTRACT: consumer pins the shape it needs; CI fails on drift
def test_order_placed_contract():
    event = producer.sample("OrderPlaced")
    assert set(event) >= {"id", "total", "customer_id"}   # required fields
    assert isinstance(event["total"], (int, float))

# CHAOS: assert the system SURVIVES failure
def test_survives_payment_slowness():
    with fault(payment_service, mode="latency", ms=800):  # inject failure
        r = checkout(order)
        assert r.status in ("ok", "queued")   # breaker/timeout held, no hang

def handle(req):
    trace = start_trace(req)                        # tracing (20)
    if req.is_read:
        if v := cache.get(req.key): return v        # cache-aside (15)
        v = replica.read(req.key, consistency="QUORUM")  # replica + quorum (10,13)
        cache.set(req.key, v, ttl=jitter(60)); return v
    # write path — idempotent, sharded, bounded, isolated:
    if seen(req.idem_key): return stored(req.idem_key)   # idempotency (8)
    shard = shard_for(req.key)                      # sharding (11)
    with bulkhead("db"), breaker("db"):             # isolation + fail-fast (18,19)
        r = shard.leader.write(req, timeout=2.0, consistency="QUORUM")  # (13)
    cache.delete(req.key); remember(req.idem_key, r)
    return r
# overload → rate-limit & shed (17); partition → CP/AP per feature (3)