Linux Internals for Defenders

# View the process tree rooted at PID 1
pstree -p 1 | head -30

# Alternative: full process listing with parent PIDs
ps auxf

# Investigate a suspicious process (PID 4821)
cat /proc/4821/cmdline | tr '\0' ' '
ls -la /proc/4821/exe
ls -la /proc/4821/cwd
ls -la /proc/4821/fd/
cat /proc/4821/environ | tr '\0' '\n'

# Find all SUID binaries on the system
find / -perm -4000 -type f 2>/dev/null

# Find all SGID binaries
find / -perm -2000 -type f 2>/dev/null

# Find world-writable files (excluding /proc and /sys)
find / -perm -o+w -type f ! -path "/proc/*" ! -path "/sys/*" 2>/dev/null

# /etc/passwd — readable by all users
# Format: username:x:UID:GID:comment:home:shell
root:x:0:0:root:/root:/bin/bash
www-data:x:33:33:www-data:/var/www:/usr/sbin/nologin
analyst:x:1001:1001:SOC Analyst:/home/analyst:/bin/bash

# /etc/shadow — readable only by root
# Format: username:password_hash:last_change:min:max:warn:inactive:expire
root:$6$rounds=5000$salt$hash...:19500:0:99999:7:::
analyst:$6$rounds=5000$salt$hash...:19500:0:99999:7:::

# Check PAM configuration for SSH
cat /etc/pam.d/sshd

# Look for unexpected PAM modules
find /lib/security/ -name "*.so" -newer /lib/security/pam_unix.so

# Check all authorized_keys files on the system
find / -name "authorized_keys" -type f 2>/dev/null

# Review root's authorized keys
cat /root/.ssh/authorized_keys

# Check for keys with command restrictions or unusual options
grep -r "command=" /home/*/.ssh/authorized_keys 2>/dev/null

# View recent authentication events
tail -100 /var/log/auth.log

# Search for failed SSH logins
grep "Failed password" /var/log/auth.log

# Search for successful SSH logins from unusual sources
grep "Accepted" /var/log/auth.log | grep -v "10.0."

# Query journald for SSH-related events
journalctl -u sshd --since "2 hours ago" --no-pager

# View login history
last -20

# View failed login attempts
lastb -20

# View last login for all users
lastlog

# Search audit log for executed commands
ausearch -m EXECVE --start recent

# System-wide crontab
cat /etc/crontab

# Per-user crontabs
ls -la /var/spool/cron/crontabs/

# Cron directories (scripts here run automatically)
ls -la /etc/cron.d/
ls -la /etc/cron.daily/
ls -la /etc/cron.hourly/

# List all cron jobs for all users
for user in $(cut -f1 -d: /etc/passwd); do echo "--- $user ---"; crontab -u $user -l 2>/dev/null; done

# Check for recently created service files
find /etc/systemd/system/ -name "*.service" -mtime -7

# Check for services in user directories
find /home -path "*/.config/systemd/user/*.service" 2>/dev/null

# List all enabled services
systemctl list-unit-files --state=enabled

# Check for suspicious service properties
systemctl cat suspicious-service.service

[Unit]
Description=System Health Monitor

[Service]
ExecStart=/usr/local/bin/.health-check
Restart=always
RestartSec=30

[Install]
WantedBy=multi-user.target

# Check init.d scripts modified recently
find /etc/init.d/ -mtime -7

# Check rc.local (runs at boot)
cat /etc/rc.local

# Files that execute on user login
cat /home/*/.bashrc
cat /home/*/.bash_profile
cat /home/*/.profile
cat /root/.bashrc

# Look for unusual commands (reverse shells, downloaders)
grep -r "curl\|wget\|nc\|/dev/tcp\|python.*import.*socket" /home/*/.*rc /home/*/.*profile /root/.*rc 2>/dev/null

# Check for system-wide LD_PRELOAD
cat /etc/ld.so.preload

# Check for LD_PRELOAD in environment files
grep -r "LD_PRELOAD" /etc/environment /etc/profile.d/ /home/*/.*rc 2>/dev/null

# List pending at jobs
atq

# View the contents of a specific at job
at -c [job_number]

# Current network connections
ss -tulnp

# All running processes with full command lines
ps auxww

# Open files and network connections per process
lsof -i -n -P

# Currently logged-in users
who
w

# Login history
last -25

# Check for new or modified user accounts
awk -F: '($3 == 0) {print $1}' /etc/passwd

# Users with login shells
grep -v "nologin\|false" /etc/passwd

# Sudo group members
getent group sudo
getent group wheel

# All authorized_keys files
find / -name "authorized_keys" 2>/dev/null

# Recent sudoers changes
stat /etc/sudoers
ls -la /etc/sudoers.d/

# All cron jobs
for u in $(cut -f1 -d: /etc/passwd); do echo "=== $u ==="; crontab -u $u -l 2>/dev/null; done
ls -la /etc/cron.d/ /etc/cron.daily/ /etc/cron.hourly/

# Recently created systemd services
find /etc/systemd/ /lib/systemd/ -name "*.service" -mtime -30

# init.d and rc.local
ls -la /etc/init.d/
cat /etc/rc.local 2>/dev/null

# Shell profile backdoors
grep -rn "curl\|wget\|nc \|ncat\|/dev/tcp" /home/*/.*rc /home/*/.*profile /root/.*rc 2>/dev/null

# LD_PRELOAD
cat /etc/ld.so.preload 2>/dev/null

# at jobs
atq

# Files modified in the last 7 days in critical directories
find /etc /usr/local/bin /usr/bin -mtime -7 -type f

# SUID/SGID binaries (compare against baseline)
find / -perm -4000 -o -perm -2000 -type f 2>/dev/null | sort

# Hidden files in /tmp and /dev/shm
ls -la /tmp/.*  2>/dev/null
ls -la /dev/shm/.* 2>/dev/null

# Files with unusual names (spaces, dots, unicode)
find /tmp /var/tmp /dev/shm -name ".*" -o -name "* *" 2>/dev/null

# World-writable executables
find / -perm -o+w -type f -executable 2>/dev/null

# Failed authentication attempts
grep "Failed\|failure\|FAILED" /var/log/auth.log | tail -50

# Successful SSH logins
grep "Accepted" /var/log/auth.log | tail -50

# Sudo usage
grep "sudo:" /var/log/auth.log | tail -50

# Check for log gaps (attacker may have cleared logs)
ls -la /var/log/auth.log*
stat /var/log/auth.log