Nginx问题排查-故障诊断

📁 36563688 📅 2026-07-25 12:47:05 👤 admin 👁️ 2678 ❤️ 559
Nginx问题排查-故障诊断

Nginx问题排查-故障诊断业务场景引入在运维一个高流量的电商平台时，运维团队经常遇到各种突发问题：

服务不可用：用户反馈网站无法访问，购物车功能异常性能下降：页面加载速度明显变慢，用户购买流程卡顿安全攻击：遭受DDoS攻击或恶意爬虫骚扰配置错误：新上线的功能导致部分用户访问异常资源耗尽：服务器CPU或内存使用率过高导致服务不稳定SSL证书问题：证书过期或配置错误导致HTTPS访问失败这些问题需要快速定位和解决，以最小化对业务的影响。掌握系统的问题排查方法和故障诊断技巧，是保障系统稳定运行的关键。

常见故障类型及诊断方法服务不可用故障诊断步骤检查Nginx进程状态# 检查Nginx进程

ps aux | grep nginx

# 检查Nginx服务状态

systemctl status nginx

# 检查端口监听状态

netstat -tlnp | grep :80

netstat -tlnp | grep :443检查配置文件语法# 测试配置文件语法

nginx -t

# 详细测试配置

nginx -T查看错误日志# 查看错误日志

tail -f /var/log/nginx/error.log

# 查看最近的错误

tail -n 50 /var/log/nginx/error.log常见原因及解决方案# 1. 端口被占用

# 错误信息：bind() to 0.0.0.0:80 failed (98: Address already in use)

# 解决方案：查找并终止占用端口的进程

lsof -i :80

kill -9

# 2. 配置文件语法错误

# 错误信息：nginx: [emerg] unexpected "}" in /etc/nginx/nginx.conf:xx

# 解决方案：修复配置文件语法错误

nginx -t

# 根据错误提示定位并修复问题

# 3. 权限问题

# 错误信息：open() "/etc/nginx/nginx.conf" failed (13: Permission denied)

# 解决方案：检查文件权限

ls -la /etc/nginx/nginx.conf

chmod 644 /etc/nginx/nginx.conf性能问题诊断性能监控命令# 实时监控系统资源

top

htop

# 监控网络连接

netstat -an | grep :80 | awk '{print $6}' | sort | uniq -c

# 监控Nginx状态

watch -n 1 'curl -s http://localhost:8080/nginx_status'

# 分析访问日志

tail -f /var/log/nginx/access.log性能分析工具# 使用ab进行压力测试

ab -n 10000 -c 100 http://example.com/

# 使用wrk进行高性能测试

wrk -t12 -c400 -d30s http://example.com/

# 使用siege进行负载测试

siege -c100 -t60s http://example.com/常见性能问题及优化# 1. 连接数过多

# 检查当前连接数

curl -s http://localhost:8080/nginx_status | head -3

# 优化配置

events {

worker_connections 65535;

use epoll;

multi_accept on;

}

# 2. 响应时间过长

# 在日志中添加响应时间

log_format main '$remote_addr - $remote_user [$time_local] "$request" '

'$status $body_bytes_sent "$http_referer" '

'"$http_user_agent" "$http_x_forwarded_for" '

'request_time=$request_time '

'upstream_response_time=$upstream_response_time';

# 3. 缓冲区不足

# 优化缓冲区配置

http {

client_body_buffer_size 128k;

client_max_body_size 10m;

client_header_buffer_size 1k;

large_client_header_buffers 4 4k;

proxy_buffering on;

proxy_buffer_size 128k;

proxy_buffers 4 256k;

proxy_busy_buffers_size 256k;

}SSL/TLS相关问题常见SSL错误及解决# 1. 证书过期

# 检查证书有效期

openssl x509 -in /etc/nginx/ssl/example.com.crt -noout -dates

# 2. 证书链不完整

# 检查证书链

openssl verify -CAfile /etc/nginx/ssl/ca-bundle.crt /etc/nginx/ssl/example.com.crt

# 3. 协议不匹配

# 测试SSL连接

openssl s_client -connect example.com:443 -servername example.comSSL故障排查配置# 启用详细的SSL错误日志

error_log /var/log/nginx/ssl_error.log debug;

server {

listen 443 ssl;

server_name example.com;

# SSL配置

ssl_certificate /etc/nginx/ssl/example.com.crt;

ssl_certificate_key /etc/nginx/ssl/example.com.key;

# 启用SSL调试

ssl_protocols TLSv1.2 TLSv1.3;

ssl_ciphers HIGH:!aNULL:!MD5;

# 错误页面处理

error_page 495 496 497 /ssl_error.html;

location = /ssl_error.html {

root /var/www/error;

internal;

}

}代理和负载均衡问题后端服务故障诊断# 1. 检查后端服务连通性

curl -I http://backend-server:8080/health

# 2. 检查上游服务器状态

nginx -T | grep -A 10 "upstream"

# 3. 分析代理错误日志

grep "upstream" /var/log/nginx/error.log负载均衡配置问题# 负载均衡故障排查配置

upstream backend {

server 192.168.1.10:8080 max_fails=3 fail_timeout=30s;

server 192.168.1.11:8080 max_fails=3 fail_timeout=30s;

server 192.168.1.12:8080 backup;

# 启用健康检查日志

keepalive 32;

}

server {

location / {

proxy_pass http://backend;

proxy_set_header Host $host;

proxy_set_header X-Real-IP $remote_addr;

proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

# 详细的代理错误处理

proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;

proxy_next_upstream_timeout 10s;

proxy_next_upstream_tries 3;

}

}日志分析与故障定位错误日志分析# 1. 按错误类型统计

awk '/^\[error\]/ {print $0}' /var/log/nginx/error.log | \

awk -F'FastCGI|upstream|SSL|open\(\)' '{print $1}' | \

sort | uniq -c | sort -nr

# 2. 按时间分析错误趋势

awk '/^\[error\]/ {print $2}' /var/log/nginx/error.log | \

cut -d: -f1,2 | sort | uniq -c

# 3. 查找特定IP的错误

grep "192.168.1.100" /var/log/nginx/error.log访问日志分析# 1. 统计状态码分布

awk '{print $9}' /var/log/nginx/access.log | sort | uniq -c | sort -nr

# 2. 找出最耗时的请求

awk '{print $NF, $7}' /var/log/nginx/access.log | \

awk -F'=' '{print $2, $1}' | sort -nr | head -20

# 3. 统计IP访问频率

awk '{print $1}' /var/log/nginx/access.log | sort | uniq -c | sort -nr | head -20

# 4. 分析用户代理

awk -F'"' '{print $6}' /var/log/nginx/access.log | sort | uniq -c | sort -nr | head -10自定义日志分析脚本#!/bin/bash

# Nginx日志分析脚本

LOG_FILE=${1:-"/var/log/nginx/access.log"}

OUTPUT_DIR="/tmp/nginx_analysis_$(date +%Y%m%d_%H%M%S)"

mkdir -p $OUTPUT_DIR

# 生成分析报告

cat > $OUTPUT_DIR/analysis_report.txt << EOF

Nginx Log Analysis Report

Generated at: $(date)

Log file: $LOG_FILE

Total requests: $(wc -l < $LOG_FILE)

=== Top 20 IP Addresses ===

$(awk '{print $1}' $LOG_FILE | sort | uniq -c | sort -nr | head -20)

=== Status Code Distribution ===

$(awk '{print $9}' $LOG_FILE | sort | uniq -c | sort -nr)

=== Top 20 Requested URLs ===

$(awk '{print $7}' $LOG_FILE | sort | uniq -c | sort -nr | head -20)

=== Top 10 User Agents ===

$(awk -F'"' '{print $6}' $LOG_FILE | sort | uniq -c | sort -nr | head -10)

=== Response Time Analysis ===

Average response time: $(awk '{print $NF}' $LOG_FILE | awk -F'=' '{sum+=$2; count++} END {if (count > 0) print sum/count "s"}')

Slowest requests: $(awk '{print $NF, $7}' $LOG_FILE | awk -F'=' '{print $2, $1}' | sort -nr | head -10)

EOF

echo "Analysis report generated: $OUTPUT_DIR/analysis_report.txt"安全问题排查恶意访问检测# 1. 检测SQL注入尝试

grep -i "select\|union\|insert\|delete\|drop\|create\|alter" /var/log/nginx/access.log

# 2. 检测XSS攻击尝试

grep -i "

# 3. 检测扫描器活动

grep -i "nmap\|nessus\|nikto\|sqlmap" /var/log/nginx/access.log

# 4. 统计高频访问IP

awk '{print $1}' /var/log/nginx/access.log | sort | uniq -c | sort -nr | head -20安全加固配置# 安全防护配置

server {

listen 80;

server_name example.com;

# 限制请求频率

limit_req_zone $binary_remote_addr zone=one:10m rate=1r/s;

limit_req_zone $binary_remote_addr zone=two:10m rate=30r/m;

# 限制连接数

limit_conn_zone $binary_remote_addr zone=conn_limit_per_ip:10m;

location / {

# 应用频率限制

limit_req zone=one burst=5 nodelay;

limit_conn conn_limit_per_ip 10;

# 安全头设置

add_header X-Frame-Options "SAMEORIGIN" always;

add_header X-Content-Type-Options "nosniff" always;

add_header X-XSS-Protection "1; mode=block" always;

root /var/www/html;

index index.html;

}

# 保护敏感路径

location ~* /(admin|wp-admin|phpmyadmin) {

allow 192.168.1.0/24;

deny all;

auth_basic "Restricted Area";

auth_basic_user_file /etc/nginx/.htpasswd;

}

# 阻止访问敏感文件

location ~* \.(env|config|sql|log|sh|pl|cgi)$ {

deny all;

access_log off;

log_not_found off;

}

}调试工具使用Nginx调试命令# 1. 测试配置文件

nginx -t

# 2. 显示配置文件

nginx -T

# 3. 重新加载配置

nginx -s reload

# 4. 优雅停止服务

nginx -s quit

# 5. 强制停止服务

nginx -s stop

# 6. 指定配置文件启动

nginx -c /path/to/nginx.conf网络调试工具# 1. 抓包分析

tcpdump -i any -n port 80 or port 443

# 2. 连接跟踪

ss -tuln | grep :80

ss -tuln | grep :443

# 3. 网络延迟测试

ping backend-server

traceroute backend-server

# 4. DNS解析测试

nslookup example.com

dig example.com故障恢复与预防自动化故障检测脚本#!/bin/bash

# Nginx故障自动检测和恢复脚本

NGINX_STATUS_URL="http://localhost:8080/nginx_status"

HEALTH_CHECK_URL="http://localhost/health"

LOG_FILE="/var/log/nginx/monitor.log"

ALERT_EMAIL="admin@example.com"

# 检查函数

check_nginx() {

# 检查进程

if ! pgrep nginx > /dev/null; then

echo "$(date): Nginx process not running" >> $LOG_FILE

return 1

fi

# 检查端口

if ! netstat -tlnp | grep -q :80; then

echo "$(date): Port 80 not listening" >> $LOG_FILE

return 1

fi

# 检查状态页面

if ! curl -s --max-time 5 $NGINX_STATUS_URL > /dev/null; then

echo "$(date): Status page not accessible" >> $LOG_FILE

return 1

fi

return 0

}

# 恢复函数

recover_nginx() {

echo "$(date): Attempting to recover Nginx" >> $LOG_FILE

# 测试配置

if ! nginx -t >> $LOG_FILE 2>&1; then

echo "$(date): Configuration test failed" >> $LOG_FILE

return 1

fi

# 重启服务

systemctl restart nginx

sleep 5

# 验证恢复

if check_nginx; then

echo "$(date): Nginx recovered successfully" >> $LOG_FILE

echo "Nginx service recovered at $(date)" | mail -s "Nginx Recovery" $ALERT_EMAIL

return 0

else

echo "$(date): Nginx recovery failed" >> $LOG_FILE

echo "Nginx recovery failed at $(date)" | mail -s "Nginx Recovery Failed" $ALERT_EMAIL

return 1

fi

}

# 主监控循环

while true; do

if ! check_nginx; then

echo "$(date): Nginx health check failed" >> $LOG_FILE

recover_nginx

fi

sleep 60

done预防性维护脚本#!/bin/bash

# Nginx预防性维护脚本

LOG_DIR="/var/log/nginx"

BACKUP_DIR="/backup/nginx"

CONFIG_DIR="/etc/nginx"

# 日志轮转检查

check_log_rotation() {

echo "Checking log rotation..."

find $LOG_DIR -name "*.log" -size +100M -exec ls -lh {} \;

}

# 配置备份

backup_config() {

echo "Backing up configuration..."

BACKUP_NAME="nginx_config_$(date +%Y%m%d_%H%M%S)"

tar -czf $BACKUP_DIR/${BACKUP_NAME}.tar.gz $CONFIG_DIR

# 清理旧备份

find $BACKUP_DIR -name "nginx_config_*.tar.gz" -mtime +30 -delete

}

# 证书检查

check_certificates() {

echo "Checking SSL certificates..."

find $CONFIG_DIR -name "*.crt" -exec openssl x509 -in {} -noout -dates \;

}

# 磁盘空间检查

check_disk_space() {

echo "Checking disk space..."

df -h $LOG_DIR $CONFIG_DIR

}

# 执行维护任务

check_log_rotation

backup_config

check_certificates

check_disk_space

echo "Preventive maintenance completed at $(date)"最佳实践总结故障排查流程快速响应

确认故障现象和影响范围收集相关日志和监控数据通知相关团队和用户系统检查

检查Nginx进程和服务状态验证端口监听和网络连通性测试配置文件语法日志分析

查看错误日志定位问题根源分析访问日志了解故障模式使用工具进行深度分析问题解决

根据诊断结果制定解决方案实施修复措施并验证效果记录问题和解决方案预防措施

分析故障原因防止复发完善监控和告警机制更新文档和操作手册常见问题避免❌ 忽略日志监控和分析❌ 缺乏配置备份和恢复机制❌ 没有建立完善的监控体系❌ 忽视安全防护和访问控制❌ 缺乏自动化故障检测和恢复通过建立系统化的问题排查和故障诊断机制，可以快速定位和解决Nginx运行中的各种问题，保障服务的稳定性和可靠性。同时，通过预防性维护和持续优化，可以有效降低故障发生概率，提升系统的整体运维水平。

相关推荐