etc

~ scripts and random things
git clone https://github.com/djnnvx/etc.git
build_fuzzers.sh (17597B)
      1 #!/bin/bash
      2 # Build TTYD fuzzing harnesses
      3 #
      4 # What this script does:
      5 #   1. Installs build dependencies
      6 #   2. Clones ttyd source + runs cmake (for validation & compile_commands.json)
      7 #   3. Generates mock headers/stubs so ttyd's .c files compile without real lws/uv
      8 #   4. Compiles instrumented ttyd objects (http.c, protocol.c, utils.c)
      9 #   5. Builds AFL++ harnesses that #include the real ttyd source files
     10 #   6. Sets up AFL++ dictionaries (download from AFL++ repo, fall back to custom)
     11 #   7. Generates seed corpus
     12 #   8. Installs a crontab entry to back up crashes every 15 minutes
     13 #
     14 # Usage: ./build_fuzzers.sh [afl|libfuzzer]
     15 #
     16 # Note: gen-corpus.sh is merged into this script and is no longer separate.
     17 
     18 set -e
     19 
     20 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
     21 MODE="${1:-afl}"
     22 SANITIZER="address,undefined"
     23 
     24 TTYD_SRC="$SCRIPT_DIR/ttyd-src"
     25 TTYD_OBJ="$SCRIPT_DIR/ttyd-obj"
     26 FUZZ_INCLUDE="$SCRIPT_DIR/fuzz-include"
     27 CRONTAB_MARKER="# ttyd-fuzz-crash-backup"
     28 BACKUP_SCRIPT="$SCRIPT_DIR/backup_crashes.sh"
     29 
     30 echo "[*] Building TTYD fuzzing harnesses"
     31 echo "[*] Mode:      $MODE"
     32 echo "[*] Sanitizer: $SANITIZER"
     33 echo "[*] Script:    $SCRIPT_DIR"
     34 echo
     35 
     36 # ─── 1. dependency install ────────────────────────────────────────────────────
     37 
     38 install_deps() {
     39     if [ "$(id -u)" -ne 0 ]; then
     40         echo "[!] Warning: Not running as root. Skipping apt-get installs."
     41         echo "    If builds fail, run: sudo apt-get install afl++ libjson-c-dev"
     42         echo "    cmake git libwebsockets-dev libuv1-dev pkg-config"
     43         return
     44     fi
     45 
     46     local DEPS="libjson-c-dev cmake git pkg-config"
     47     if [ "$MODE" = "afl" ]; then
     48         DEPS="$DEPS afl++"
     49     else
     50         DEPS="$DEPS clang"
     51     fi
     52 
     53     local MISSING=""
     54     for pkg in $DEPS; do
     55         dpkg -s "$pkg" &>/dev/null || MISSING="$MISSING $pkg"
     56     done
     57 
     58     if [ -n "$MISSING" ]; then
     59         echo "[*] Installing:$MISSING"
     60         apt-get install -y --no-install-recommends $MISSING
     61         echo "[+] Dependencies installed"
     62     fi
     63 
     64     # System tuning for AFL++
     65     if [ "$MODE" = "afl" ]; then
     66         local CORE_PATTERN
     67         CORE_PATTERN=$(cat /proc/sys/kernel/core_pattern 2>/dev/null || echo "")
     68         if [[ "$CORE_PATTERN" == "|"* ]]; then
     69             echo core > /proc/sys/kernel/core_pattern
     70             echo "[+] core_pattern set to 'core'"
     71         fi
     72         for gov in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do
     73             [ -f "$gov" ] && echo performance > "$gov" 2>/dev/null || true
     74         done
     75     fi
     76 }
     77 
     78 # ─── 2. clone + cmake ttyd ────────────────────────────────────────────────────
     79 
     80 setup_ttyd_source() {
     81     # Clone
     82     if [ ! -d "$TTYD_SRC/.git" ]; then
     83         echo "[*] Cloning ttyd source..."
     84         git clone --depth=1 https://github.com/tsl0922/ttyd "$TTYD_SRC"
     85         echo "[+] Cloned ttyd to $TTYD_SRC"
     86     else
     87         echo "[*] ttyd source already present — pulling latest..."
     88         git -C "$TTYD_SRC" pull --ff-only 2>/dev/null || \
     89             echo "[!] Pull skipped (uncommitted local changes or offline)"
     90     fi
     91 
     92     # cmake configure (validates deps, produces compile_commands.json for IDEs)
     93     local BUILD_DIR="$TTYD_SRC/build"
     94     mkdir -p "$BUILD_DIR"
     95     echo "[*] Running cmake configure..."
     96     (
     97         cd "$BUILD_DIR"
     98         # Use the real compiler here (not AFL++) — this is only for validation/headers
     99         cmake -DCMAKE_BUILD_TYPE=Debug \
    100               -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
    101               .. 2>&1 | tail -5 || true
    102     )
    103     echo "[+] cmake done (compile_commands.json at $BUILD_DIR/compile_commands.json)"
    104 
    105     # ── verify mock headers/sources exist ────────────────────────────────
    106     # These files are checked into the repo alongside the harnesses.
    107     # (Previously they were generated inline by this script.)
    108     for f in "$FUZZ_INCLUDE/libwebsockets.h" "$FUZZ_INCLUDE/uv.h" \
    109              "$SCRIPT_DIR/fuzz_lws_mock.h" "$SCRIPT_DIR/fuzz_lws_mock.c" \
    110              "$SCRIPT_DIR/mock_server_globals.c" "$SCRIPT_DIR/mock_pty_stubs.c"; do
    111         if [ ! -f "$f" ]; then
    112             echo "[!] Missing required file: $f"
    113             echo "    These files should be checked in alongside the harnesses."
    114             exit 1
    115         fi
    116     done
    117     echo "[+] Mock headers and stubs verified"
    118 
    119     # ── compile instrumented ttyd objects ────────────────────────────────
    120     mkdir -p "$TTYD_OBJ"
    121     local JSON_CFLAGS
    122     JSON_CFLAGS=$(pkg-config --cflags json-c 2>/dev/null || echo "")
    123 
    124     local OBJ_CC OBJ_CFLAGS
    125     if [ "$MODE" = "afl" ]; then
    126         OBJ_CC="afl-clang-fast"
    127     else
    128         OBJ_CC="clang"
    129     fi
    130     OBJ_CFLAGS="-fsanitize=$SANITIZER -g -O1 -D_GNU_SOURCE -std=gnu99"
    131     OBJ_CFLAGS="$OBJ_CFLAGS -I$FUZZ_INCLUDE -I$TTYD_SRC/src $JSON_CFLAGS"
    132     # Suppress warnings from upstream code we don't control
    133     OBJ_CFLAGS="$OBJ_CFLAGS -Wno-implicit-function-declaration -Wno-int-conversion"
    134 
    135     echo
    136     echo "[*] Compiling instrumented ttyd objects..."
    137     # Only utils.c is pre-compiled as a shared object.
    138     # http.c and protocol.c are compiled directly into each harness via
    139     # #include — pre-compiling them here would cause multiple-definition
    140     # linker errors.
    141     for src in utils; do
    142         echo "    $src.c → ttyd-obj/$src.o"
    143         $OBJ_CC $OBJ_CFLAGS -c "$TTYD_SRC/src/$src.c" -o "$TTYD_OBJ/$src.o"
    144     done
    145 
    146     # Mock objects
    147     for src in fuzz_lws_mock mock_server_globals mock_pty_stubs; do
    148         echo "    $src.c → ttyd-obj/$src.o"
    149         $OBJ_CC $OBJ_CFLAGS -c "$SCRIPT_DIR/$src.c" -o "$TTYD_OBJ/$src.o"
    150     done
    151 
    152     echo "[+] ttyd objects compiled to $TTYD_OBJ/"
    153 }
    154 
    155 # ─── 3. dictionaries ──────────────────────────────────────────────────────────
    156 
    157 setup_dictionaries() {
    158     local DICT_DIR="$SCRIPT_DIR/dictionaries"
    159     mkdir -p "$DICT_DIR"
    160 
    161     # Try system AFL++ dictionaries first
    162     for sys_dir in /usr/share/afl++/dictionaries /usr/local/share/afl++/dictionaries; do
    163         if [ -d "$sys_dir" ]; then
    164             [ -f "$sys_dir/http.dict" ] && [ ! -f "$DICT_DIR/http.dict" ] && \
    165                 cp "$sys_dir/http.dict" "$DICT_DIR/http.dict" && \
    166                 echo "[+] Copied http.dict from $sys_dir"
    167             [ -f "$sys_dir/json.dict" ] && [ ! -f "$DICT_DIR/json.dict" ] && \
    168                 cp "$sys_dir/json.dict" "$DICT_DIR/json.dict" && \
    169                 echo "[+] Copied json.dict from $sys_dir"
    170             break
    171         fi
    172     done
    173 
    174     # Download from AFL++ GitHub if still missing (http.dict and json.dict
    175     # ship under those exact names in the AFL++ stable branch)
    176     local BASE="https://raw.githubusercontent.com/AFLplusplus/AFLplusplus/stable/dictionaries"
    177     local DL=""
    178     command -v wget  &>/dev/null && DL="wget -q -O"
    179     command -v curl  &>/dev/null && [ -z "$DL" ] && DL="curl -fsSL -o"
    180 
    181     for dict in http json; do
    182         if [ ! -f "$DICT_DIR/$dict.dict" ] && [ -n "$DL" ]; then
    183             echo "[*] Downloading $dict.dict from AFL++ repo..."
    184             $DL "$DICT_DIR/$dict.dict" "$BASE/$dict.dict" && \
    185                 echo "[+] Downloaded $dict.dict" || \
    186                 rm -f "$DICT_DIR/$dict.dict"
    187         fi
    188     done
    189 
    190     # Minimal fallback if still missing (from checked-in fallback files)
    191     if [ ! -f "$DICT_DIR/http.dict" ] && [ -f "$SCRIPT_DIR/dictionaries/http_fallback.dict" ]; then
    192         cp "$SCRIPT_DIR/dictionaries/http_fallback.dict" "$DICT_DIR/http.dict"
    193         echo "[+] Created dictionaries/http.dict (fallback)"
    194     fi
    195 
    196     if [ ! -f "$DICT_DIR/json.dict" ] && [ -f "$SCRIPT_DIR/dictionaries/json_fallback.dict" ]; then
    197         cp "$SCRIPT_DIR/dictionaries/json_fallback.dict" "$DICT_DIR/json.dict"
    198         echo "[+] Created dictionaries/json.dict (fallback)"
    199     fi
    200 
    201     echo "[+] Dictionaries: $(ls "$DICT_DIR")"
    202 }
    203 
    204 # ─── 4. corpus generation ─────────────────────────────────────────────────────
    205 
    206 generate_corpus() {
    207     echo "[*] Running gen_corpus.py..."
    208     python3 "$SCRIPT_DIR/gen_corpus.py" "$SCRIPT_DIR"
    209 }
    210 
    211 # ─── 5. crash backup crontab ──────────────────────────────────────────────────
    212 
    213 setup_crash_backup() {
    214     local BACKUP_DIR="$SCRIPT_DIR/crashes-backup"
    215 
    216     # Write the backup script from template
    217     sed -e "s|%%CRONTAB_MARKER%%|$CRONTAB_MARKER|g" \
    218         -e "s|%%SCRIPT_DIR%%|$SCRIPT_DIR|g" \
    219         -e "s|%%BACKUP_DIR%%|$BACKUP_DIR|g" \
    220         "$SCRIPT_DIR/backup_crashes.sh.in" > "$BACKUP_SCRIPT"
    221     chmod +x "$BACKUP_SCRIPT"
    222     echo "[+] Created $BACKUP_SCRIPT"
    223 
    224     # Install crontab entry (idempotent — remove stale entry first)
    225     local TMPFILE
    226     TMPFILE=$(mktemp)
    227     crontab -l 2>/dev/null | grep -v "$CRONTAB_MARKER" | grep -v "backup_crashes.sh" > "$TMPFILE" || true
    228     {
    229         cat "$TMPFILE"
    230         echo "$CRONTAB_MARKER"
    231         echo "*/15 * * * * $BACKUP_SCRIPT >> $SCRIPT_DIR/crashes-backup/backup.log 2>&1"
    232     } | crontab -
    233     rm -f "$TMPFILE"
    234     echo "[+] Crontab entry installed (every 15 min)"
    235     crontab -l | grep -A1 "$CRONTAB_MARKER"
    236 }
    237 
    238 # ─── 6. select compiler ───────────────────────────────────────────────────────
    239 
    240 case "$MODE" in
    241     afl)
    242         if ! command -v afl-clang-fast &>/dev/null; then
    243             echo "[!] afl-clang-fast not found. Install AFL++ first."
    244             exit 1
    245         fi
    246         CC="afl-clang-fast"
    247         CFLAGS="-fsanitize=$SANITIZER -g -O1"
    248         SUFFIX=""
    249         ;;
    250     libfuzzer)
    251         if ! command -v clang &>/dev/null; then
    252             echo "[!] clang not found. Install clang first."
    253             exit 1
    254         fi
    255         CC="clang"
    256         CFLAGS="-fsanitize=fuzzer,$SANITIZER -g -O1"
    257         SUFFIX="_lf"
    258         ;;
    259     *)
    260         echo "[!] Unknown mode: $MODE  (use 'afl' or 'libfuzzer')"
    261         exit 1
    262         ;;
    263 esac
    264 
    265 # ─── find json-c ─────────────────────────────────────────────────────────────
    266 
    267 JSON_C_CFLAGS=$(pkg-config --cflags json-c 2>/dev/null || echo "")
    268 JSON_C_LIBS=$(pkg-config --libs   json-c 2>/dev/null || echo "-ljson-c")
    269 if [ -z "$JSON_C_CFLAGS" ]; then
    270     for d in /usr/include /usr/local/include; do
    271         [ -f "$d/json-c/json.h" ] && JSON_C_CFLAGS="-I$d" && break
    272     done
    273 fi
    274 if [ -z "$JSON_C_CFLAGS" ]; then
    275     echo "[!] json-c headers not found. Run: sudo apt-get install libjson-c-dev"
    276     exit 1
    277 fi
    278 
    279 # ─── run setup stages ────────────────────────────────────────────────────────
    280 
    281 echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    282 echo "[1/7] Installing dependencies"
    283 echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    284 install_deps
    285 
    286 echo
    287 echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    288 echo "[2/7] Setting up ttyd source + instrumented objects"
    289 echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    290 setup_ttyd_source
    291 
    292 # ─── build harnesses ─────────────────────────────────────────────────────────
    293 
    294 echo
    295 echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    296 echo "[3/7] Building harnesses (with real ttyd code)"
    297 echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    298 
    299 # Common flags for all harnesses: our mock includes take priority over system
    300 COMMON_CFLAGS="$CFLAGS -I$FUZZ_INCLUDE -I$TTYD_SRC/src $JSON_C_CFLAGS"
    301 COMMON_CFLAGS="$COMMON_CFLAGS -Wno-implicit-function-declaration -Wno-int-conversion"
    302 
    303 # http.c and protocol.c are compiled into each harness via #include — do NOT
    304 # include their pre-compiled objects here or every symbol will be defined twice.
    305 MOCK_OBJS="$TTYD_OBJ/utils.o"
    306 MOCK_OBJS="$MOCK_OBJS $TTYD_OBJ/fuzz_lws_mock.o $TTYD_OBJ/mock_server_globals.o $TTYD_OBJ/mock_pty_stubs.o"
    307 
    308 # http.c decompresses gzip-encoded HTML using zlib
    309 EXTRA_LIBS="-lz"
    310 
    311 echo "[*] Building fuzz_websocket_auth..."
    312 $CC $COMMON_CFLAGS fuzz_websocket_auth.c $MOCK_OBJS $JSON_C_LIBS $EXTRA_LIBS -o fuzz_websocket_auth$SUFFIX
    313 echo "[+] Built: fuzz_websocket_auth$SUFFIX"
    314 
    315 echo "[*] Building fuzz_http_parsing..."
    316 $CC $COMMON_CFLAGS fuzz_http_parsing.c $MOCK_OBJS $JSON_C_LIBS $EXTRA_LIBS -o fuzz_http_parsing$SUFFIX
    317 echo "[+] Built: fuzz_http_parsing$SUFFIX"
    318 
    319 echo "[*] Building fuzz_ws_fragments..."
    320 $CC $COMMON_CFLAGS fuzz_ws_fragments.c $MOCK_OBJS $JSON_C_LIBS $EXTRA_LIBS -o fuzz_ws_fragments$SUFFIX
    321 echo "[+] Built: fuzz_ws_fragments$SUFFIX"
    322 
    323 echo "[*] Building fuzz_prefs_message..."
    324 $CC $COMMON_CFLAGS fuzz_prefs_message.c $MOCK_OBJS $JSON_C_LIBS $EXTRA_LIBS -o fuzz_prefs_message$SUFFIX
    325 echo "[+] Built: fuzz_prefs_message$SUFFIX"
    326 
    327 echo "[*] Building fuzz_json_recv..."
    328 $CC $COMMON_CFLAGS fuzz_json_recv.c $MOCK_OBJS $JSON_C_LIBS $EXTRA_LIBS -o fuzz_json_recv$SUFFIX
    329 echo "[+] Built: fuzz_json_recv$SUFFIX"
    330 
    331 # CmpLog variants (AFL++ only)
    332 if [ "$MODE" = "afl" ]; then
    333     echo
    334     echo "[*] Building CmpLog variants..."
    335     AFL_LLVM_CMPLOG=1 $CC $COMMON_CFLAGS fuzz_websocket_auth.c $MOCK_OBJS $JSON_C_LIBS $EXTRA_LIBS \
    336         -o fuzz_websocket_auth_cmplog
    337     echo "[+] Built: fuzz_websocket_auth_cmplog"
    338     AFL_LLVM_CMPLOG=1 $CC $COMMON_CFLAGS fuzz_http_parsing.c $MOCK_OBJS $JSON_C_LIBS $EXTRA_LIBS \
    339         -o fuzz_http_parsing_cmplog
    340     echo "[+] Built: fuzz_http_parsing_cmplog"
    341     AFL_LLVM_CMPLOG=1 $CC $COMMON_CFLAGS fuzz_ws_fragments.c $MOCK_OBJS $JSON_C_LIBS $EXTRA_LIBS \
    342         -o fuzz_ws_fragments_cmplog
    343     echo "[+] Built: fuzz_ws_fragments_cmplog"
    344     AFL_LLVM_CMPLOG=1 $CC $COMMON_CFLAGS fuzz_prefs_message.c $MOCK_OBJS $JSON_C_LIBS $EXTRA_LIBS \
    345         -o fuzz_prefs_message_cmplog
    346     echo "[+] Built: fuzz_prefs_message_cmplog"
    347     AFL_LLVM_CMPLOG=1 $CC $COMMON_CFLAGS fuzz_json_recv.c $MOCK_OBJS $JSON_C_LIBS $EXTRA_LIBS \
    348         -o fuzz_json_recv_cmplog
    349     echo "[+] Built: fuzz_json_recv_cmplog"
    350 
    351     echo
    352     echo "[*] Building HTTP custom mutator..."
    353     cc -shared -fPIC -O2 http_mutator.c -o http_mutator.so
    354     echo "[+] Built: http_mutator.so"
    355 fi
    356 
    357 echo
    358 echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    359 echo "[4/7] Setting up dictionaries"
    360 echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    361 setup_dictionaries
    362 
    363 echo
    364 echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    365 echo "[5/7] Generating seed corpus"
    366 echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    367 generate_corpus
    368 
    369 echo
    370 echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    371 echo "[6/7] Installing crash-backup crontab"
    372 echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    373 setup_crash_backup
    374 
    375 echo
    376 echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    377 echo "[7/7] Done"
    378 echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    379 echo
    380 echo "Ready to fuzz:"
    381 if [ "$MODE" = "afl" ]; then
    382     echo "  ./run-fuzzers.sh          # launch all targets in tmux"
    383     echo "  afl-fuzz -i corpus/websocket_auth -o output/websocket_auth -x dictionaries/json.dict ./fuzz_websocket_auth"
    384 else
    385     echo "  ./fuzz_auth_header_lf corpus/auth_header/"
    386 fi
    387 echo
    388 echo "Crashes backed up automatically every 15 min → crashes-backup/"
    389 echo "To stop backup: ./clean.sh"