← Back to DevBytes
Developer Tutorial July 09, 2026 12 min read

Fix 'Segmentation fault' in C/C++ in Production: Root Cause Analysis

Understanding Segmentation Faults in Production

A segmentation fault (signal SIGSEGV) occurs when a process attempts to access a memory location that it is not allowed to access. The operating system's memory management unit detects the violation and sends the SIGSEGV signal, which by default terminates the process with the dreaded message: Segmentation fault (core dumped). In production environments, this means downtime, lost transactions, and frustrated users.

What Actually Triggers a Segmentation Fault?

The CPU, in coordination with the OS, triggers a segfault when your program violates memory access rules. Common causes include:

Why Segmentation Faults Are Catastrophic in Production

Unlike exceptions in managed languages, a segfault in C/C++ is unrecoverable by default. The process dies instantly. In a production server handling thousands of requests per second, this means:

Worse, the crash may be intermittent—manifesting only under specific load conditions, timing, or input patterns—making it notoriously difficult to reproduce in a debugger.

Root Cause Analysis Strategy: A Systematic Approach

🚀 Deploy your AI agent in 10 minutes

Managed Hermes hosting. Zero DevOps. 100M tokens/mo included.

Try it free →

Root cause analysis for production segfaults requires a disciplined, evidence-based methodology. You cannot attach a debugger to a live production process that crashes unpredictably. Instead, you must gather forensic artifacts and reconstruct the failure post-mortem.

Step 1: Enable Core Dumps in Production

Core dumps are the single most powerful tool for segfault root cause analysis. A core file is a snapshot of the process's memory at the moment of the crash, including the stack, registers, and heap. Without it, you are debugging blind.

# Check current core dump configuration
ulimit -c
# Set unlimited core dump size for the current session
ulimit -c unlimited

# Make it permanent by adding to /etc/security/limits.conf
# *                soft    core            unlimited
# *                hard    core            unlimited

# Configure core dump location and naming pattern
echo '/var/coredumps/core.%e.%p.%t' | sudo tee /proc/sys/kernel/core_pattern

# Ensure the directory exists and is writable
sudo mkdir -p /var/coredumps
sudo chmod 777 /var/coredumps

For a systemd-based service, also configure the service unit to allow core dumps:

# In the service override file /etc/systemd/system/your-service.service.d/override.conf
[Service]
LimitCORE=infinity
# Optional: set the working directory for core dumps
WorkingDirectory=/var/coredumps

Step 2: Capture the Crash with a Custom Signal Handler

While core dumps are essential, you should also install a SIGSEGV handler that logs critical diagnostic information before the process terminates. This gives you immediate context without waiting to extract core file data.

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <signal.h>
#include <execinfo.h>   /* for backtrace functions */
#include <unistd.h>
#include <time.h>

/* Thread-local crash context buffer */
static __thread char crash_context[4096] = {0};

/* Register a crash context string that will be printed on segfault */
void register_crash_context(const char *context) {
    if (context) {
        strncpy(crash_context, context, sizeof(crash_context) - 1);
        crash_context[sizeof(crash_context) - 1] = '\0';
    }
}

/* Signal handler for SIGSEGV, SIGABRT, SIGFPE */
static void critical_signal_handler(int sig, siginfo_t *info, void *ucontext) {
    /* Avoid recursive signals — reset handlers immediately */
    signal(sig, SIG_DFL);
    
    /* Write crash report to stderr and a dedicated log file */
    FILE *log = fopen("/var/log/app_crash.log", "a");
    if (!log) log = stderr;
    
    time_t now = time(NULL);
    fprintf(log, "=== CRASH REPORT [%s]", ctime(&now));
    fprintf(log, "Signal: %d (%s)\n", sig, strsignal(sig));
    fprintf(log, "Fault address: %p\n", info->si_addr);
    fprintf(log, "Fault code: %d\n", info->si_code);
    
    if (crash_context[0] != '\0') {
        fprintf(log, "Context: %s\n", crash_context);
    }
    
    /* Capture and print backtrace */
    void *bt_buffer[128];
    int bt_size = backtrace(bt_buffer, 128);
    char **bt_symbols = backtrace_symbols(bt_buffer, bt_size);
    
    fprintf(log, "Backtrace (%d frames):\n", bt_size);
    for (int i = 0; i < bt_size; i++) {
        fprintf(log, "  #%02d %s\n", i, bt_symbols[i]);
    }
    free(bt_symbols);
    
    /* Print register state from ucontext on x86_64 */
    #if defined(__x86_64__) && defined(REG_RIP)
    if (ucontext) {
        ucontext_t *uc = (ucontext_t *)ucontext;
        fprintf(log, "Registers:\n");
        fprintf(log, "  RIP: 0x%016lx  RSP: 0x%016lx\n",
                uc->uc_mcontext.gregs[REG_RIP],
                uc->uc_mcontext.gregs[REG_RSP]);
        fprintf(log, "  RAX: 0x%016lx  RBX: 0x%016lx\n",
                uc->uc_mcontext.gregs[REG_RAX],
                uc->uc_mcontext.gregs[REG_RBX]);
    }
    #endif
    
    fprintf(log, "=== END REPORT ===\n\n");
    fflush(log);
    if (log != stderr) fclose(log);
    
    /* Re-raise the signal with default handler to generate core dump */
    signal(sig, SIG_DFL);
    raise(sig);
}

void install_crash_handlers(void) {
    struct sigaction sa;
    memset(&sa, 0, sizeof(sa));
    sa.sa_sigaction = critical_signal_handler;
    sa.sa_flags = SA_SIGINFO | SA_RESETHAND;
    
    sigaction(SIGSEGV, &sa, NULL);
    sigaction(SIGABRT, &sa, NULL);
    sigaction(SIGFPE,  &sa, NULL);
    /* SIGBUS on some platforms */
    sigaction(SIGBUS,  &sa, NULL);
}

Register meaningful context strings at critical points in your code:

void process_order(Order *order) {
    register_crash_context("Processing order");
    
    // ... validation logic ...
    
    register_crash_context("Validating order items");
    for (int i = 0; i < order->item_count; i++) {
        char ctx[256];
        snprintf(ctx, sizeof(ctx), "Processing item %d/%d, SKU=%s",
                 i, order->item_count, order->items[i].sku);
        register_crash_context(ctx);
        
        // This is where the crash might happen
        validate_item(&order->items[i]);
    }
}

Step 3: Analyze the Core Dump with GDB

Once you have a core file, GDB is your primary analysis tool. The following sequence extracts maximum information:

# Load the executable and core file
gdb /path/to/your/binary /var/coredumps/core.your-app.12345.1680000000

# 1. Get basic crash information
(gdb) info signals
(gdb) info program

# 2. Examine the signal frame — shows exactly where the crash occurred
(gdb) frame 0
(gdb) info registers
(gdb) disassemble $rip-20,$rip+20

# 3. Print the faulting instruction and its operands
(gdb) x/i $rip
# Examine the memory address being accessed (e.g., if RAX held the bad pointer)
(gdb) info symbol $rax
(gdb) x/16gx $rax-32   # Look at memory around the fault address

# 4. Full backtrace with all frames, local variables, and arguments
(gdb) bt full
(gdb) thread apply all bt full

# 5. Examine heap metadata for corruption clues
(gdb) info malloc    # if glibc malloc debug symbols available
(gdb) heap check     # if compiled with heap debugging

# 6. Check if the fault was in shared library code
(gdb) info sharedlibrary
(gdb) frame 0
(gdb) info scope

Step 4: Use AddressSanitizer in Staging for Memory Error Detection

If you can reproduce the crash in a staging or canary environment, compile with AddressSanitizer (ASan). It detects the exact root cause at the point of error, not just at the point of crash. This is invaluable for finding use-after-free and buffer overflow bugs that manifest as segfaults later.

# Compile with ASan instrumentation
gcc -fsanitize=address -fno-omit-frame-pointer -g -O1 \
    -o app_asan your_source.c -lasan

# Or with clang
clang -fsanitize=address -fno-omit-frame-pointer -g -O1 \
    -o app_asan your_source.c

# Run the instrumented binary
./app_asan

# Typical ASan output for a heap buffer overflow:
# ==12345==ERROR: AddressSanitizer: heap-buffer-overflow on address 0x60300000e054
# READ of size 8 at 0x60300000e054 thread T0
#     #0 0x4c3a2e in process_record src/parser.c:142
#     #1 0x4c3f1a in parse_file src/parser.c:89
# 0x60300000e054 is located 4 bytes after 80-byte region [0x60300000e000,0x60300000e050)
# allocated by thread T0 here:
#     #0 0x7f8e3c in malloc (.../libasan.so)
#     #1 0x4c3a8a in create_record src/parser.c:67

The beauty of ASan is that it tells you both where the invalid access happened AND where the memory was originally allocated—giving you the complete lifecycle of the problematic memory.

Step 5: Static Analysis for Pattern Recognition

Not all segfaults can be reproduced. In those cases, static analysis of the source code around the crash site is essential. Look for common patterns:

/* PATTERN 1: Missing NULL check after allocation */
void vulnerable_function_1(void) {
    MyStruct *ptr = malloc(sizeof(MyStruct));
    /* If malloc fails, ptr is NULL — next line crashes */
    ptr->field = 42;   // SEGFAULT if malloc returned NULL
}

/* PATTERN 2: Use-after-free */
void vulnerable_function_2(void) {
    char *buffer = malloc(1024);
    process_data(buffer);
    free(buffer);
    /* ... many lines later, possibly in a callback ... */
    printf("%s", buffer);  // SEGFAULT: buffer is dangling
}

/* PATTERN 3: Stack buffer overflow */
void vulnerable_function_3(const char *input) {
    char local_buffer[64];
    /* If input is longer than 63 chars, stack gets corrupted */
    strcpy(local_buffer, input);  // Use strncpy or strlcpy instead
}

/* PATTERN 4: Off-by-one in array access */
void vulnerable_function_4(void) {
    int array[100];
    for (int i = 0; i <= 100; i++) {  // Bug: should be i < 100
        array[i] = compute_value(i);  // SEGFAULT or stack smash at i=100
    }
}

/* PATTERN 5: Double free */
void vulnerable_function_5(void) {
    char *p = malloc(256);
    /* ... code that might free p in an error path ... */
    cleanup_1(p);  // May free p
    cleanup_2(p);  // May free p again — corrupts heap, eventual SEGFAULT
}

Step 6: Correlate with Production Metrics

Segfaults often correlate with specific runtime conditions. Instrument your application to capture:

/* Lightweight runtime instrumentation for correlation */
typedef struct {
    uint64_t timestamp_ms;
    uint32_t thread_id;
    uint32_t sequence;
    const char *operation;
    const char *file;
    int line;
    void *allocation_ptr;
    size_t allocation_size;
} trace_event_t;

/* Ring buffer for recent events — survives in core dump */
#define TRACE_BUFFER_SIZE 8192
static trace_event_t trace_buffer[TRACE_BUFFER_SIZE];
static uint32_t trace_index = 0;

void record_trace(const char *operation, const char *file, int line,
                  void *ptr, size_t size) {
    uint32_t idx = __atomic_fetch_add(&trace_index, 1, __ATOMIC_RELAXED);
    idx = idx % TRACE_BUFFER_SIZE;
    trace_event_t *ev = &trace_buffer[idx];
    ev->timestamp_ms = get_monotonic_ms();
    ev->thread_id = (uint32_t)pthread_self();
    ev->sequence = idx;
    ev->operation = operation;
    ev->file = file;
    ev->line = line;
    ev->allocation_ptr = ptr;
    ev->allocation_size = size;
}

/* Use macros to automatically capture file/line */
#define TRACE_ALLOC(ptr, size) \
    record_trace("alloc", __FILE__, __LINE__, ptr, size)
#define TRACE_FREE(ptr) \
    record_trace("free", __FILE__, __LINE__, ptr, 0)
#define TRACE_ACCESS(ptr) \
    record_trace("access", __FILE__, __LINE__, ptr, 0)

When examining a core dump, inspect the trace buffer in GDB:

(gdb) print trace_index
(gdb) print trace_buffer[trace_index-1]
(gdb) print trace_buffer[trace_index-2]
# Or dump the last 50 events
(gdb) set $i = (trace_index - 50) % TRACE_BUFFER_SIZE
(gdb) while $i < trace_index
 > print trace_buffer[$i % TRACE_BUFFER_SIZE]
 > set $i = $i + 1
 > end

Step 7: Thread Sanitizer for Concurrent Segfaults

Segfaults in multi-threaded code often stem from data races that corrupt pointers. ThreadSanitizer (TSan) detects these races:

# Compile with ThreadSanitizer
clang -fsanitize=thread -g -O1 -o app_tsan your_source.c -lpthread

# Run with TSan
./app_tsan

# Example TSan output:
# WARNING: ThreadSanitizer: data race
#   Write of size 8 at 0x7b0400000018 by thread T2:
#     #0 update_shared_state src/worker.c:45
#   Previous read of size 8 at 0x7b0400000018 by thread T1:
#     #0 read_shared_state src/worker.c:23
#   Location is global 'g_shared_ptr' of size 8

Real-World Case Studies

Case 1: The Phantom Segfault in Production

A trading system crashed every few days with a segfault in strcmp(). The core dump showed:

(gdb) bt
#0  0x00007f8e3c in __strcmp_avx2 () from /lib64/libc.so.6
#1  0x0000004c3a2e in find_symbol (name=0x7f8e4000 "EURUSD") at symbols.c:89
#2  0x0000004c3b8a in resolve_trade (trade=0x603000000) at trade.c:234

(gdb) frame 1
(gdb) info locals
symbol_table = 0x60300000e000
name = 0x7f8e4000 "EURUSD"
# symbol_table pointer looks valid, but...

(gdb) x/32gx symbol_table
0x60300000e000: 0x0000000000000000 0x0000000000000000  # ALL ZEROES
# The symbol_table was freed and zeroed, but the pointer was still cached

The root cause: A background thread refreshed the symbol table by freeing the old one and allocating a new one. But active trading threads held cached pointers to the old table. The fix involved reference counting or RCU-based synchronization.

Case 2: Stack Exhaustion Under Load

A web server segfaulted only when handling more than 500 concurrent connections. The core dump revealed:

(gdb) bt
#0  0x00007f... in __libc_write () from /lib64/libc.so.6
#1  ...
#200 0x00004c... in process_request (req=0x...) at handler.c:156
#201 0x00004c... in process_request (req=0x...) at handler.c:156
#202 0x00004c... in process_request (req=0x...) at handler.c:156
... 2000+ identical frames ...

(gdb) frame 2000
(gdb) info locals
buffer = 0x7ffd9a3c1000  # Stack address dangerously low
(gdb) p/x $rsp - $rbp
# Frame size was 8KB due to a large on-stack buffer

The recursive function process_request() had an 8KB stack buffer and under heavy nesting exhausted the 8MB stack. The fix: move large buffers to the heap and add a recursion depth limit.

Best Practices for Preventing Production Segfaults

Automated Core Dump Analysis Pipeline

For large-scale deployments, manually analyzing every core dump is impractical. Build an automated pipeline:

#!/bin/bash
# automated_core_analyzer.sh — triggered by systemd on core dump

CORE_FILE="$1"
EXECUTABLE="/usr/bin/your-app"
REPORT_DIR="/var/crash-reports"
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
REPORT_FILE="$REPORT_DIR/crash_$TIMESTAMP.txt"

mkdir -p "$REPORT_DIR"

# Generate automated GDB report
gdb --batch --quiet -ex "thread apply all bt full" \
    -ex "info registers" \
    -ex "info frame" \
    -ex "info signals" \
    -ex "quit" \
    "$EXECUTABLE" "$CORE_FILE" > "$REPORT_FILE" 2>&1

# Extract key facts
FAULT_ADDR=$(grep -oP 'si_addr.*0x[0-9a-f]+' "$REPORT_FILE")
TOP_FUNCTION=$(grep -A1 '^#0' "$REPORT_FILE" | tail -1)

# Upload to central analysis service
curl -X POST https://crash-analysis.internal/api/report \
     -F "report=@$REPORT_FILE" \
     -F "fault_addr=$FAULT_ADDR" \
     -F "top_function=$TOP_FUNCTION" \
     -F "binary_version=$(md5sum $EXECUTABLE | cut -d' ' -f1)"

# Compress and archive core file (they're large)
gzip -9 "$CORE_FILE"
mv "$CORE_FILE.gz" "/var/coredumps/archive/"

Conclusion

Segmentation faults in production C/C++ applications are inevitable if you push the boundaries of performance and complexity. The key to handling them is not to prevent every possible crash—that's unrealistic—but to build a robust forensic infrastructure that captures enough data at the moment of failure to identify the root cause quickly. Core dumps are non-negotiable. Custom signal handlers with backtrace logging bridge the gap between the crash and your awareness of it. Sanitizers in staging catch memory errors before they reach production. And a systematic root cause analysis process—examining the faulting instruction, the memory state, the allocation history, and the thread context—turns a terrifying segfault into a solved problem. The difference between hours of guesswork and a five-minute diagnosis is the quality of the artifacts you collect before and during the crash. Invest in that infrastructure, and segfaults become just another manageable failure mode rather than a production nightmare.

🚀 Need a reliable AI agent for your project?

Deploy Hermes Agent in 10 minutes. Managed hosting, zero DevOps.

Get Started — $23.99/mo
← Back to all articles