Linux Pro Playbook

$ cat /proc/cpuinfo | grep "model name" | head -1   # hardware, as a file
$ cat /proc/1/cmdline; echo                          # what is PID 1?
$ cat /sys/class/net/eth0/address                    # NIC's MAC, a file
$ echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward    # WRITE kernel config
$ ls -l /dev/null /dev/sda /dev/random               # devices are files too

/etc      # system config — nginx.conf, ssh/, cron.d/  (text, in git-able form)
/var      # grows: /var/log, /var/lib/postgresql, /var/spool
/usr      # installed software: /usr/bin, /usr/lib, /usr/local/bin (yours!)
/tmp      # scratch — often wiped on reboot, world-writable
/home     # users; /root is root's home (NOT under /home)
/opt      # vendor bundles: /opt/datadog, /opt/your-app
/srv      # served content (rarer)
/proc /sys /dev   # kernel & devices (topic 1)

$ df -h / /var        # are they separate partitions? (matters — topic 8)
$ command -v ruby     # which binary actually runs?

$ ls *.log                    # shell expands * → app.log db.log BEFORE ls runs
app.log  db.log
$ echo "$HOME/bin"            # double quotes: $HOME expands
/home/balu/bin
$ echo '$HOME/bin'            # single quotes: literal
$HOME/bin
$ files=*.log; echo $files    # unquoted: glob expands at USE time
app.log db.log
$ echo "$files"               # quoted: the literal pattern, unexpanded
*.log

$ ls -l deploy.sh
-rwxr-xr-x 1 balu devs 214 Jul 11 deploy.sh
 -  rwx r-x r-x    →  owner=7  group=5  other=5  →  755
 └ type (- file, d dir, l link)

# octal: r=4 w=2 x=1, summed per keyring
$ chmod 755 deploy.sh    # rwx r-x r-x   scripts, directories
$ chmod 644 config.yml   # rw- r-- r--   data files, web content
$ chmod 600 id_rsa       # rw- --- ---   SSH keys MUST be this
$ chmod +x deploy.sh     # symbolic: just add execute

$ id
uid=1000(balu) gid=1000(balu) groups=1000(balu),27(sudo),999(docker)
$ sudo systemctl restart nginx      # borrow root for ONE command (logged)
$ chown www-data:www-data /var/www/app   # give files to the service's badge
$ sudo -u postgres psql             # run AS another user
$ sudo usermod -aG docker balu      # -aG = APPEND (forgetting -a locks you out!)

$ echo data > original.txt
$ ln original.txt hard.txt        # hard link — same inode
$ ln -s original.txt soft.txt     # symlink — points at the NAME
$ ls -li                          # -i shows inode numbers
1839 -rw-r--r-- 2 balu balu  5 original.txt   # link count 2!
1839 -rw-r--r-- 2 balu balu  5 hard.txt       # SAME inode 1839
1902 lrwxrwxrwx 1 balu balu 12 soft.txt -> original.txt

$ rm original.txt
$ cat hard.txt      # still works — inode alive (hard link holds it)
data
$ cat soft.txt      # BROKEN — the name it pointed at is gone
cat: soft.txt: No such file or directory

$ which python3           # first match on PATH
/usr/bin/python3
$ type ll                 # is it a program, alias, function, builtin?
ll is aliased to `ls -alF'
$ type -a python3         # ALL matches, in PATH order — shadowing revealed
python3 is /home/balu/.venv/bin/python3
python3 is /usr/bin/python3
$ command -v node         # script-safe "does this exist / where"
/usr/bin/node
$ locate nginx.conf       # instant DB lookup (run updatedb first)
/etc/nginx/nginx.conf

$ df -h                    # which filesystem is full? (the gauge)
Filesystem  Size  Used Avail Use% Mounted on
/dev/root    50G   50G     0 100% /
$ du -sh /var/* | sort -h | tail -5   # what's heavy under the culprit?
1.2G  /var/cache
2.1G  /var/lib
14G   /var/log            # ← found it
$ du -sh /var/log/* | sort -h | tail -3
13G   /var/log/app.log    # one runaway log

# the trap: deleted the file, df STILL shows full?
$ sudo lsof +L1           # files with link-count 0 but still OPEN
COMMAND  PID  USER  FD   ...  SIZE     NLINK  NAME
ruby    8823  app   7w   ...  13000M   0      /var/log/app.log (deleted)
$ sudo systemctl restart app     # make it let go → space returns

$ ps aux | sort -rk3 | head -3      # top CPU (col 3); -rk4 for memory
USER  PID  %CPU %MEM  ...  COMMAND
app   8823 98.2  4.1  ...  ruby sidekiq
$ ps -ef --forest | grep -A3 nginx  # the process TREE (parent/child)
$ pgrep -a nginx                    # PIDs by name, with command line
$ ls -l /proc/8823/                 # everything about PID 8823, as files
$ cat /proc/8823/cmdline | tr '\0' ' '   # exact command it launched with
$ ls -l /proc/8823/cwd /proc/8823/exe    # its working dir + binary path
$ cat /proc/8823/limits             # its ulimits (topic 15)

$ kill 8823           # sends SIGTERM (15) — polite, lets it clean up
$ kill -9 8823        # SIGKILL — trapdoor, no cleanup (last resort)
$ kill -HUP 712       # SIGHUP — many daemons RELOAD config on this (nginx!)
$ pkill -f sidekiq    # by command-line match
$ echo $?             # exit code of the LAST command (0 = success)

# reading exit codes — the vocabulary:
0    success
1-125  the program's own error codes (check its docs)
126   found but not executable   127  command not found (typo/PATH)
130   Ctrl-C (SIGINT, 128+2)     137  SIGKILL/OOM (128+9)   143  SIGTERM (128+15)

# shell job control (single session):
$ long_task &            # run in background (& = detach from foreground)
$ jobs                   # list background jobs
$ fg %1                  # bring job 1 back to foreground
# Ctrl-Z suspends the foreground job; bg resumes it in background

# tmux — the real answer for remote long-running work:
$ tmux new -s deploy     # create a named session
  # ... start your migration ...
  # Ctrl-b then d         → DETACH (leaves everything running)
$ ssh server             # reconnect later, even from another machine
$ tmux attach -t deploy  # walk back into the office, work still there
$ tmux ls                # list sessions

# quick one-offs without tmux:
$ nohup ./task.sh &      # immune to hangup; output → nohup.out

# the daily verbs:
$ sudo systemctl status nginx      # running? enabled? recent log lines?
$ sudo systemctl restart nginx     # stop then start
$ sudo systemctl reload nginx      # re-read config, no downtime (SIGHUP-ish)
$ sudo systemctl enable --now app  # start now AND on every boot
$ systemctl is-active app; systemctl is-enabled app

# the unified logbook:
$ journalctl -u nginx -f           # follow ONE service's logs live
$ journalctl -u app --since "10 min ago"
$ journalctl -u app -p err         # only errors and worse
$ journalctl -b                    # everything since THIS boot
$ journalctl --disk-usage          # how big is the journal?

$ uptime
 14:22:07 up 40 days,  load average: 3.91, 2.10, 1.05
                                      └1min └5min └15min  (trend: rising fast!)
$ nproc          # how many cores? (the number load is relative to)
4                # load 3.91 on 4 cores = ~fully used, not yet overloaded

$ top            # live; press: P=sort CPU, M=sort mem, 1=per-core, q=quit
top - 14:22  load average: 3.91, 2.10, 1.05
%Cpu(s): 71 us, 12 sy,  0 ni,  5 id,  9 wa   ← wa = I/O WAIT (disk/net stall!)
  PID USER  %CPU %MEM  COMMAND
 8823 app   190  4.1   ruby          ← >100% = using multiple cores

$ free -h
              total   used   free   shared  buff/cache   available
Mem:           16Gi  6.2Gi  412Mi     88Mi       9.4Gi      9.1Gi
                                  ↑ tiny!         ↑ cache    ↑ REAL headroom
# read "available", NOT "free": 9.1Gi is what apps can actually still get.
# buff/cache (9.4Gi) is reclaimable — the kernel gives it up on demand.

# did the OOM killer strike?
$ dmesg -T | grep -i "killed process"
[Sat Jul 11 03:14] Out of memory: Killed process 8823 (ruby) total-vm:...
$ journalctl -k | grep -i oom          # same, via the journal

$ ulimit -n              # this shell's soft limit on open files
1024                     # default — laughably low for a busy server
$ ulimit -Hn             # the hard ceiling you can raise up to
1048576

# how many is a process actually holding?
$ ls /proc/8823/fd | wc -l
1019                     # nearly at 1024 → about to hit the wall

# raise it — for a systemd service (the right place):
# [Service]
# LimitNOFILE=65535
$ sudo systemctl daemon-reload && sudo systemctl restart app
# or system-wide in /etc/security/limits.conf

# why can't it find its config? watch the file opens:
$ strace -f -e trace=openat ./app 2>&1 | grep -i config
openat(AT_FDCWD, "/etc/app/config.yml", O_RDONLY) = -1 ENOENT (No such file)
                                                    └─ it's looking HERE, and failing

# attach to an already-running, hung process:
$ sudo strace -p 8823
read(7, ...        ← stuck here: blocked reading fd 7. What's fd 7?
$ ls -l /proc/8823/fd/7
lrwx... 7 -> socket:[48291]     ← waiting on a network socket (a stuck DB call?)

$ strace -c ./app      # -c = summary: which syscalls dominate (counts + time)
$ strace -f -e trace=network ./app   # only network syscalls

# the three streams: 0=stdin  1=stdout  2=stderr
$ command > out.txt          # stdout → file (overwrite)
$ command >> out.txt          # stdout → file (append)
$ command 2> err.txt          # stderr → file
$ command > all.txt 2>&1      # BOTH → file (2>&1 = "send 2 where 1 goes")
$ command 2>/dev/null         # discard errors
$ command | tee log.txt       # to screen AND file (a T-junction)

# real pipelines you'll actually write:
$ cat access.log | grep 500 | wc -l              # count 500 errors
$ ps aux | grep -v grep | grep ruby              # ruby procs (minus grep itself)
$ history | awk '{print $2}' | sort | uniq -c | sort -rn | head   # top commands

$ grep ERROR app.log             # lines containing ERROR
$ grep -i error app.log          # -i = case-insensitive
$ grep -r "TODO" src/            # -r = recursive through a directory
$ grep -n ERROR app.log          # -n = show line numbers
$ grep -c ERROR app.log          # -c = COUNT matches (not the lines)
$ grep -v healthcheck app.log    # -v = INVERT: lines WITHOUT the pattern
$ grep -C3 "OutOfMemory" app.log # -C3 = 3 lines of context around each hit
$ grep -E "5[0-9]{2}" access.log # -E = extended regex (5xx status codes)
$ grep -o "user_id=[0-9]*" app.log  # -o = print only the matched part

# the combo you'll use constantly — errors with surrounding context, live:
$ tail -f app.log | grep --line-buffered -iC2 error

# $1,$2... = columns; $0 = whole line; NF = column count; NR = row number
$ awk '{print $1}' access.log            # first column (client IPs)
$ awk '{print $1, $NF}' file             # first and LAST column
$ awk -F: '{print $1}' /etc/passwd        # -F: = split on colons, not spaces

# filter + extract in one step:
$ awk '$9 == 500 {print $7}' access.log   # paths that returned 500
$ awk '$3 > 1000' data.txt                 # rows where column 3 exceeds 1000

# math over a column — no scratch variables, no loop:
$ awk '{sum += $5} END {print sum}' sales.txt          # total column 5
$ awk '{sum+=$1; n++} END {print sum/n}' latencies.txt # average

# tally with associative arrays (grouping!):
$ awk '{count[$1]++} END {for (ip in count) print count[ip], ip}' access.log \
    | sort -rn | head       # top client IPs by request count

# substitute: s/find/replace/  (add g for ALL matches on a line)
$ sed 's/localhost/db.prod/' config.txt        # first match per line → stdout
$ sed 's/localhost/db.prod/g' config.txt       # every match
$ echo "hello world" | sed 's/o/0/g'
hell0 w0rld

$ sed -n '10,20p' file           # -n + p = print ONLY lines 10-20
$ sed '/^#/d' config             # delete comment lines (start with #)
$ sed '/^$/d' file               # delete blank lines
$ sed 's/ *$//' file             # strip trailing whitespace

# edit files IN PLACE — the powerful, dangerous one:
$ sed -i 's/debug/info/g' app.conf          # modifies the file directly
$ sed -i.bak 's/debug/info/g' app.conf      # SAME, but keeps app.conf.bak

# find files, delete them (rm wants ARGS, not stdin):
$ find . -name "*.tmp" | xargs rm
# but filenames with spaces break that! the SAFE version:
$ find . -name "*.tmp" -print0 | xargs -0 rm        # -print0/-0 = null-separated

$ echo "1 2 3" | xargs -n1 echo        # -n1 = one arg per invocation
echo 1 ; echo 2 ; echo 3

# {} placeholder — put the arg WHERE you need it:
$ ls *.jpg | xargs -I{} cp {} /backup/{}
$ cat pids.txt | xargs -I{} kill {}

# parallelize — run 4 at a time (huge speedup for slow per-item work):
$ cat urls.txt | xargs -P4 -I{} curl -sO {}        # -P4 = 4 in parallel

# find PATH  criteria...  [action]
$ find . -name "*.log"                    # by name (quote it — topic 3!)
$ find /var -type f -size +100M           # files over 100MB
$ find . -mtime -1                        # modified in the last 1 day
$ find . -mmin -60                        # ...last 60 minutes
$ find /home -name "*.env" -type f        # secrets audit
$ find . -type d -empty                   # empty directories
$ find . -user www-data                   # by owner (topic 5)

# combine criteria (implicit AND) and ACT on results:
$ find /var/log -name "*.log" -mtime +7 -size +50M -delete
$ find . -name "*.jpg" -exec convert {} {}.webp \;   # run a cmd per file
$ find . -name "*.tmp" -print0 | xargs -0 rm         # or hand off to xargs (topic 21)

# the canonical "top N" pipeline:
$ awk '{print $1}' access.log | sort | uniq -c | sort -rn | head
#       └ extract IP        └sort  └count  └rank desc └top 10

$ sort file | uniq              # dedupe (needs sort first!)
$ sort -u file                  # dedupe in one step
$ sort file | uniq -c           # count occurrences of each line
$ sort file | uniq -d           # only DUPLICATED lines
$ sort file | uniq -u           # only UNIQUE (appearing once) lines

# sort flags that matter:
$ sort -n     # numeric (10 after 9, not before)
$ sort -rn    # numeric, reversed (biggest first)
$ sort -h     # human sizes (2K < 1M < 3G) — great with du (topic 8)
$ sort -k3 -t,  # sort by 3rd column, comma-separated

#!/usr/bin/env bash
set -euo pipefail          # the three-word seatbelt:
#   -e          exit immediately if any command fails
#   -u          error on undefined variables (catches typos!)
#   -o pipefail a failure ANYWHERE in a pipe fails the whole pipe
IFS=$'\n\t'                # safer word-splitting (spaces in filenames)

# quote everything (topic 3), default carefully:
readonly DIR="${1:?usage: deploy.sh <dir>}"    # :? = fail if $1 unset
cd "$DIR"                                       # quoted!

# the disaster this prevents:
rm -rf "$BUILD/"        # if BUILD is unset → rm -rf "/"  ...
# with set -u, unset $BUILD errors out BEFORE the rm runs.

trap 'echo "failed at line $LINENO"' ERR       # report where it died
$ bash -n script.sh     # syntax-check without running
$ shellcheck script.sh  # LINT it — catches quoting/logic bugs

# the five fields:  minute hour day-of-month month day-of-week  command
$ crontab -e         # edit YOUR crontab
# ┌ min  ┌ hour ┌ dom ┌ mon ┌ dow
  0      2      *     *     *    /opt/backup.sh        # 02:00 every day
  */5    *      *     *     *    /opt/healthcheck.sh   # every 5 minutes
  0      9      *     *     1    /opt/weekly.sh        # 09:00 every Monday

$ crontab -l         # list current jobs
# system-wide: /etc/cron.d/*, /etc/cron.daily/, etc.

# systemd timer equivalent (the modern way) — two files:
# backup.service (what to run) + backup.timer (when):
#   [Timer]
#   OnCalendar=*-*-* 02:00:00
#   Persistent=true          ← runs on next boot if the machine was OFF at 02:00
$ systemctl list-timers      # all timers + next/last run — cron can't show this

# apt (Debian/Ubuntu):
$ sudo apt update                # refresh the catalog (NOT install anything)
$ sudo apt upgrade               # install available updates
$ sudo apt install nginx         # install + dependencies
$ sudo apt remove nginx          # remove (purge also deletes config)
$ apt search postgres            # find packages
$ apt list --installed | grep ssl
$ dpkg -S $(which nginx)         # which PACKAGE owns this file?

# dnf/yum (RHEL/Rocky/Fedora) — same ideas, different verbs:
$ sudo dnf install nginx
$ sudo dnf upgrade
$ dnf provides */nginx           # which package provides this file/command?

# find what provides a MISSING command:
$ command-not-found htop   →   apt install htop     (many distros suggest it)

# tar — remember "czf = Create Zip File", "xzf = eXtract":
$ tar czf backup.tar.gz /var/www     # create gzipped archive
$ tar xzf backup.tar.gz              # extract it
$ tar tzf backup.tar.gz              # LIST contents without extracting
$ tar czf - /data | ssh host 'cat > /backup/data.tgz'   # stream over ssh

# rsync — the -a (archive) flag preserves perms/times/links:
$ rsync -av /src/ /dst/              # local sync (trailing / matters! see gotcha)
$ rsync -avz /app/ user@host:/app/   # -z = compress over network
$ rsync -av --delete /src/ /dst/     # mirror: delete extras in dst
$ rsync -av --dry-run --delete /src/ /dst/   # PREVIEW what --delete would do
$ rsync -av --progress big.iso host:/tmp/    # resumable-ish, shows progress

$ vim /etc/nginx/nginx.conf

# you start in NORMAL mode. To type text:
i        # enter INSERT mode (now type normally)
Esc      # back to NORMAL mode (do this when confused — always safe)

# save & quit (from Normal mode — press Esc first):
:w       # write (save)
:wq      # write and quit    (or ZZ)
:q       # quit (if no changes)
:q!      # quit WITHOUT saving — the "get me out of here" escape

# navigation (Normal mode) & the few that pay off:
/error   # search for "error" (n = next match)
dd       # delete current line       u = undo       Ctrl-r = redo
gg / G   # jump to top / bottom
:120     # jump to line 120

$ echo $PATH | tr ':' '\n'        # the search list, one dir per line
/home/balu/.venv/bin              ← checked FIRST (shadows later matches)
/usr/local/bin
/usr/bin
/bin
$ type -a python3                  # every match in order (topic 7)

# add a dir — put it FIRST to win, LAST to be a fallback:
$ export PATH="$HOME/bin:$PATH"    # prepend (yours wins)
# make it permanent — add that line to your shell's dotfile:
#   ~/.bashrc     (bash, interactive)     ~/.zshrc  (zsh)
#   ~/.profile    (login shells, DE)      /etc/environment (system-wide)

$ source ~/.bashrc                # reload dotfile without logging out
$ which -a ruby                   # is there a shadowing second ruby?

# reading & following:
$ tail -f /var/log/nginx/error.log         # follow live (incident mode)
$ tail -n 200 /var/log/syslog              # last 200 lines
$ less +F /var/log/app.log                 # follow, Ctrl-C to scroll back
$ grep -i error /var/log/app.log | tail    # topic 18 + tail

# modern systems: journald (topic 12) holds much of it:
$ journalctl -u nginx --since "1 hour ago" -p warning
$ journalctl -f                            # follow ALL system logs live

# taming size — logrotate config (/etc/logrotate.d/app):
#   /var/log/app/*.log {
#     daily
#     rotate 14           ← keep 14 days
#     compress            ← gzip old ones
#     copytruncate        ← don't break the app's open file handle (topic 6!)
#   }
$ sudo logrotate -f /etc/logrotate.d/app   # force a rotation to test

$ df -h                    # what's mounted where, and how full (topic 8)
$ mount | column -t        # every mount, with type & options
$ lsblk                    # block devices as a tree — disks & partitions
NAME   SIZE TYPE MOUNTPOINT
nvme0n1 50G disk
└─nvme0n1p1 50G part /
nvme1n1 100G disk          ← a new disk, not yet mounted

# mount it (manually — gone on reboot):
$ sudo mkdir /data
$ sudo mount /dev/nvme1n1 /data

# make it permanent — /etc/fstab (mounted every boot):
# UUID=abc-123  /data  ext4  defaults,noatime  0  2
$ blkid /dev/nvme1n1       # get the UUID (stable across reboots, unlike names)
$ sudo mount -a            # test fstab NOW without rebooting (catch errors!)
$ umount /data             # detach (fails if something's using it → lsof)

# CPU — busy? queued? (topic 13)
$ uptime; mpstat 1 3        # load vs nproc; %usr %sys %iowait per core

# MEMORY — out? swapping? (topic 14)
$ free -h; vmstat 1 5       # 'si/so' columns nonzero = SWAPPING = pain

# DISK I/O — the silent killer
$ iostat -xz 1 3            # %util near 100 + high await = disk saturated
$ iotop -o                  # WHICH process is doing the I/O

# NETWORK
$ ss -s                     # socket summary
$ sar -n DEV 1 3            # per-interface throughput (or nload/iftop)

# the all-in-one modern dashboards:
$ htop      # CPU/mem/procs, interactive     btop  # prettier, same idea
$ dstat -tcmnd    # cpu+mem+net+disk in one scrolling table

$ sysctl vm.swappiness              # read one knob
vm.swappiness = 60
$ sudo sysctl -w vm.swappiness=10   # set it NOW (temporary — lost on reboot)
$ sysctl -a | grep somaxconn        # search all knobs

# the knobs you'll actually touch:
net.core.somaxconn=1024             # bigger connection accept queue (busy servers)
net.ipv4.ip_forward=1               # route packets between interfaces (containers/routers)
fs.inotify.max_user_watches=524288  # file-watchers (webpack/IDE "ENOSPC" fix)
vm.max_map_count=262144             # Elasticsearch and friends require this

# make it PERMANENT — /etc/sysctl.d/99-tuning.conf:
#   net.core.somaxconn = 1024
$ sudo sysctl -p /etc/sysctl.d/99-tuning.conf   # apply the file now

# a container is just a process — find it on the HOST:
$ docker run -d --name web --memory 512m --cpus 2 nginx
$ docker inspect -f '{{.State.Pid}}' web
9102
$ ps -o pid,cmd 9102          # it's a normal nginx process on the host!

# its NAMESPACES (what it can see) — under /proc:
$ sudo ls -l /proc/9102/ns/
lrwxrwxrwx net -> net:[4026532...]   ← its own network stack
lrwxrwxrwx pid -> pid:[4026532...]   ← its own PID numbering (it thinks it's PID 1)
lrwxrwxrwx mnt -> mnt:[4026532...]   ← its own filesystem view

# its CGROUP limits (what it can use):
$ cat /sys/fs/cgroup/.../memory.max     # 536870912 = the 512m cap
$ systemd-cgtop                          # live cgroup resource usage