Web Application Firewall (WAF) Setup: ModSecurity & Nginx (2026)

A Web Application Firewall (WAF) sits between your web server and the internet, inspecting HTTP traffic and blocking malicious requests before they reach your application. ModSecurity is the most widely deployed open-source WAF, and when combined with the OWASP Core Rule Set (CRS), it provides comprehensive protection against the OWASP Top 10 vulnerabilities.

1. Installing ModSecurity with Nginx

# Install dependencies
sudo apt install build-essential git libpcre3-dev libxml2-dev \
  libcurl4-openssl-dev libyajl-dev libgeoip-dev

# Clone and build ModSecurity v3
git clone --depth 1 https://github.com/SpiderLabs/ModSecurity
cd ModSecurity
git submodule init && git submodule update
./build.sh && ./configure
make -j$(nproc) && sudo make install

# Build Nginx connector module
git clone --depth 1 https://github.com/SpiderLabs/ModSecurity-nginx
# Compile Nginx with the module
nginx -V 2>&1 | grep arguments | sed "s/.*arguments: //" > /tmp/nginx-args
./configure $(cat /tmp/nginx-args) --add-dynamic-module=../ModSecurity-nginx
make modules
sudo cp objs/ngx_http_modsecurity_module.so /etc/nginx/modules/

2. Configuring ModSecurity

# /etc/nginx/modsec/modsecurity.conf

SecRuleEngine On
SecRequestBodyAccess On
SecResponseBodyAccess Off
SecRequestBodyLimit 13107200
SecRequestBodyNoFilesLimit 131072

# Logging
SecAuditEngine RelevantOnly
SecAuditLogRelevantStatus "^(?:5|4(?!04))"
SecAuditLogParts ABCEFHJKZ
SecAuditLogType Serial
SecAuditLog /var/log/modsec_audit.log

# Temp files
SecTmpDir /tmp
SecDataDir /tmp

# Unicode mapping
SecUnicodeMapFile unicode.mapping 20127

3. OWASP Core Rule Set

# Download OWASP CRS
cd /etc/nginx/modsec
git clone https://github.com/coreruleset/coreruleset
cp coreruleset/crs-setup.conf.example coreruleset/crs-setup.conf

# Include in ModSecurity config
echo "Include /etc/nginx/modsec/coreruleset/crs-setup.conf" >> modsecurity.conf
echo "Include /etc/nginx/modsec/coreruleset/rules/*.conf" >> modsecurity.conf

CRS Configuration

# /etc/nginx/modsec/coreruleset/crs-setup.conf

# Paranoia Level (1-4, higher = more strict)
SecAction "id:900000,phase:1,pass,t:none,nolog,setvar:tx.paranoia_level=2"

# Anomaly scoring thresholds
SecAction "id:900110,phase:1,pass,t:none,nolog,\
  setvar:tx.inbound_anomaly_score_threshold=10,\
  setvar:tx.outbound_anomaly_score_threshold=5"

# Allowed HTTP methods
SecAction "id:900200,phase:1,pass,t:none,nolog,\
  setvar:tx.allowed_methods=GET HEAD POST OPTIONS PUT DELETE"

4. Nginx Integration

# /etc/nginx/nginx.conf
load_module modules/ngx_http_modsecurity_module.so;

http {
    modsecurity on;
    modsecurity_rules_file /etc/nginx/modsec/modsecurity.conf;
    
    server {
        listen 443 ssl;
        server_name example.com;
        
        # WAF enabled for all locations
        modsecurity on;
        
        location / {
            proxy_pass http://127.0.0.1:8080;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        }
        
        # Disable WAF for specific paths (API uploads etc.)
        location /api/upload {
            modsecurity off;
            proxy_pass http://127.0.0.1:8080;
        }
    }
}

sudo nginx -t && sudo systemctl reload nginx

5. Custom WAF Rules

# /etc/nginx/modsec/custom-rules.conf

# Block known bad user agents
SecRule REQUEST_HEADERS:User-Agent "@pmFromFile bad-user-agents.txt" \
  "id:10001,phase:1,deny,status:403,msg:\"Blocked bad user agent\""

# Rate limit login attempts
SecRule REQUEST_URI "@streq /login" \
  "id:10002,phase:1,pass,nolog,setvar:ip.login_count=+1"
SecRule IP:LOGIN_COUNT "@gt 10" \
  "id:10003,phase:1,deny,status:429,msg:\"Login rate limit exceeded\",\
   setvar:ip.login_count=0,expirevar:ip.login_count=300"

# Block file upload of dangerous types
SecRule FILES_TMPNAMES "@rx \.(?:php|phtml|php3|php4|php5|exe|bat|cmd|sh)$" \
  "id:10004,phase:2,deny,status:403,msg:\"Dangerous file upload blocked\""

# Detect directory traversal
SecRule REQUEST_URI "@rx (?:\.{2}[/\\]){2,}" \
  "id:10005,phase:1,deny,status:403,msg:\"Directory traversal attempt\""

6. Testing Your WAF

# Test SQL injection blocking
curl -v "https://example.com/search?q=1%27%20OR%201=1--"
# Expected: 403 Forbidden

# Test XSS blocking
curl -v "https://example.com/page?name="
# Expected: 403 Forbidden

# Test path traversal blocking
curl -v "https://example.com/../../etc/passwd"
# Expected: 403 Forbidden

# Check WAF logs
tail -f /var/log/modsec_audit.log | grep -A5 "403"

7. Performance Optimization

# Whitelist static assets (no WAF inspection needed)
SecRule REQUEST_URI "@rx \.(css|js|png|jpg|gif|ico|woff2?|svg|webp)$" \
  "id:10100,phase:1,pass,nolog,ctl:ruleEngine=Off"

# Skip body inspection for GET requests
SecRule REQUEST_METHOD "@streq GET" \
  "id:10101,phase:1,pass,nolog,ctl:requestBodyAccess=Off"

# Cache compiled rules
SecTmpSaveUploadedFiles Off
SecRequestBodyInMemoryLimit 131072

Download our WAF Configuration Cheat Sheet for a printable quick-reference guide.