深夜处理 bug:重启 openresty(nginx)后, 403 forbidden,连服务器 ip 都无法访问了 - V2EX
V2EX = way to explore
V2EX 是一个关于分享和探索的地方
现在注册
已注册用户请  登录
kmdd33
V2EX    NGINX

深夜处理 bug:重启 openresty(nginx)后, 403 forbidden,连服务器 ip 都无法访问了

  •  
  •   kmdd33 2018-06-29 03:02:36 +08:00 7582 次点击
    这是一个创建于 2664 天前的主题,其中的信息可能已经有所发展或是发生改变。

    手贱重启了 openresty,突然站点就 403 forbidden。搞了一天了,真心累,请教同学们。

    配置环境:centos7+openresty(nginx)+php+mysql systemctl status openresty ,systemctl status php-fpm,systemctl status mysql,都是 active status,绿色。

    lsof -i :80

    COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME

    nginx 32255 root 6u IPv4 170338 0t0 TCP localhost:http (LISTEN)

    nginx 32332 root 6u IPv4 170338 0t0 TCP localhost:http (LISTEN)

    netstat -an |grep 80

    tcp 0 0 127.0.0.1:80 0.0.0.0:* LISTEN

    unix 2 [ ACC ] STREAM LISTENING 28071 /var/lib/mysql/mysql.sock

    unix 3 [ ] STREAM CONNECTED 11800

    unix 3 [ ] STREAM CONNECTED 11801 /run/systemd/journal/stdout

    ps aux | grep php

    root 19363 0.0 3.1 470292 31572 ? Ss 00:20 0:04 php-fpm: master process (/etc/ph-fpm.conf)

    nginx 19364 0.0 0.7 470292 7228 ? S 00:20 0:00 php-fpm: pool www

    nginx 19365 0.0 0.7 470292 7228 ? S 00:20 0:00 php-fpm: pool www

    nginx 19366 0.0 0.7 470292 7228 ? S 00:20 0:00 php-fpm: pool www

    nginx 19367 0.0 0.7 470292 7228 ? S 00:20 0:00 php-fpm: pool www

    nginx 19368 0.0 0.7 470292 7232 ? S 00:20 0:00 php-fpm: pool www

    nginx 19369 0.0 0.7 470292 7232 ? S 00:20 0:00 php-fpm: pool www

    nginx 19370 0.0 0.7 470292 7232 ? S 00:20 0:00 php-fpm: pool www

    nginx 19371 0.0 0.7 470292 7232 ? S 00:20 0:00 php-fpm: pool www

    root 32204 0.0 0.0 112704 968 pts/0 R+ 17:57 0:00 grep --color=auto php

    netstat -plant

    Active Internet connections (servers and established)

    Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name

    tcp 0 0 127.0.0.1:80 0.0.0.0:* LISTEN 32255/nginx: master

    tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 730/sshd

    tcp 0 0 45.79.66.57:22 113.118.253.226:4315 ESTABLISHED 32434/sshd: root@pt

    tcp 0 36 45.79.66.57:22 113.118.253.226:4860 ESTABLISHED 32584/sshd: root@pt

    tcp 0 0 45.79.66.57:22 113.118.253.226:3505 ESTABLISHED 32115/sshd: root@pt

    tcp6 0 0 :::9000 :::* LISTEN 19363/php-fpm: mast

    tcp6 0 0 :::3306 :::* LISTEN 3299/mysqld

    tcp6 0 0 :::22 :::* LISTEN 730/sshd

    /usr/local/openresty/nginx/conf/nginx.conf

    user root;

    worker_processes 1;

    #error_log logs/error.log;

    #error_log logs/error.log notice;

    #error_log logs/error.log info;

    #pid logs/nginx.pid;

    events { worker_connections 1024; }

    http { include mime.types; default_type application/octet-stream;

    #log_format main '$remote_addr - $remote_user [$time_local] "$request" ' # '$status $body_bytes_sent "$http_referer" ' # '"$http_user_agent" "$http_x_forwarded_for"'; #access_log logs/access.log main; sendfile on; #tcp_nopush on; #keepalive_timeout 0; keepalive_timeout 65; client_max_body_size 8m; #允许客户端请求的最大单文件字节数 client_body_buffer_size 2m; #缓冲区代理缓冲用户端请求的最大字节 fastcgi_buffers 8 16k; fastcgi_buffer_size 32k; fastcgi_connect_timeout 300; fastcgi_send_timeout 300; fastcgi_read_timeout 300; gzip on; #WAF lua_shared_dict limit 50m; lua_shared_dict guard_dict 100m; lua_shared_dict dict_captcha 70m; lua_package_path "/usr/local/openresty/nginx/conf/waf/?.lua"; init_by_lua_file "/usr/local/openresty/nginx/conf/waf/init.lua"; access_by_lua_file "/usr/local/openresty/nginx/conf/waf/access.lua"; server { listen 127.0.0.1; server_name kiwikiwifly.com; #charset koi8-r; #access_log logs/host.access.log main; location / { root html; index index.php index.html index.htm; } #error_page 404 /404.html; # redirect server error pages to the static page /50x.html # error_page 500 502 503 504 /50x.html; location = /50x.html { root html; } # proxy the PHP scripts to Apache listening on 127.0.0.1:80 # #location ~ \.php$ { # proxy_pass http://127.0.0.1; #} # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000 # location ~ \.php$ { root html; fastcgi_pass 127.0.0.1:9000; fastcgi_index index.php; fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; include fastcgi_params; } # deny access to .htaccess files, if Apache's document root # concurs with nginx's one # #location ~ /\.ht { # deny all; #} } # another virtual host using mix of IP-, name-, and port-based configuration # #server { # listen 8000; # listen somename:8080; # server_name somename alias another.alias; # location / { # root html; # index index.html index.htm; # } #} # HTTPS server # #server { # listen 443 ssl; # server_name localhost; # ssl_certificate cert.pem; # ssl_certificate_key cert.key; # ssl_session_cache shared:SSL:1m; # ssl_session_timeout 5m; # ssl_ciphers HIGH:!aNULL:!MD5; # ssl_prefer_server_ciphers on; # location / { # root html; # index index.html index.htm; # } #} 

    }

    and /usr/local/openresty/nginx/logs/error.log

    2018/06/26 21:27:10 [alert] 740#740: *7927 open socket #10 left in connection 12

    2018/06/26 21:27:10 [alert] 740#740: *7926 open socket #3 left in connection 13

    2018/06/26 21:27:10 [alert] 740#740: aborting

    只有到 26 号 error.log

    10 条回复    2018-06-29 18:02:42 +08:00
    Livid
        1
    Livid  
    MOD
    PRO
       2018-06-29 03:06:20 +08:00 via iPhone
    1. 给 server 那段单独写一个 error_log
    2. 试试把 WAF 相关的配置先注释掉
    kmdd33
        2
    kmdd33  
    OP
       2018-06-29 05:40:35 +08:00
    @Livid 2.已经注释了,1 给 server 那段单独写一个 error_log ?请问这个如何操作?
    yaxin
        3
    yaxin  
       2018-06-29 07:54:23 +08:00 via iPhone
    看一下 nginx 有没有 web 根路径的读权限,如果有,在根路径下创建一个 xxxx.php 文件,里面随便输出点字符串,然后访问,看能否访问,能,说明你站点 php 逻辑问题,不能,nginx 问题,通过日志把问题,还是找不到,直接上 strace
    est
        4
    est  
       2018-06-29 09:06:09 +08:00
    livid 的办法是正解。

    开 error_log,debug 级别。就能看到为啥出错了。
    sajesemuy
        5
    sajesemuy  
       2018-06-29 09:37:56 +08:00
    你这边怎么又是 403,另一个帖子 502。listen 127.0.0.1,你是直接 在本地 curl 127.0.0.1 出现 403/502 ?
    如果是访问你的站点的公网地址,前面那一层怎么访问 127.0.0.1 ?
    wqsfree
        6
    wqsfree  
       2018-06-29 09:43:41 +08:00
    user nobody nobody;
    worker_processes 2;
    error_log /var/log/nginx/error.log debug ; 这是错误日志的路径,会了么?
    pid /var/run/nginx/nginx.pid;
    kmdd33
        7
    kmdd33  
    OP
       2018-06-29 16:14:12 +08:00
    @Livid @sajesemuy @est Sorry,Sorry,是 502 bad gateway 的提示。
    kmdd33
        8
    kmdd33  
    OP
       2018-06-29 17:21:05 +08:00
    @wqsfree

    你好,按照您的建议,把你提供的这个写在了 http 段外面,重新 reload openresty 成功,error_log /var/log/nginx/error.log 内容:

    2018/06/29 09:06:08 [notice] 10008#10008: using the "epoll" event method

    2018/06/29 09:06:08 [notice] 10008#10008: openresty/1.13.6.2

    2018/06/29 09:06:08 [notice] 10008#10008: built by gcc 4.8.5 20150623 (Red Hat 4.8.5-16) (GCC)

    2018/06/29 09:06:08 [notice] 10008#10008: OS: Linux 3.10.0-862.3.2.el7.x86_64

    2018/06/29 09:06:08 [notice] 10008#10008: getrlimit(RLIMIT_NOFILE): 1024:4096

    2018/06/29 09:06:08 [notice] 10009#10009: start worker processes

    2018/06/29 09:06:08 [notice] 10009#10009: start worker process 10011

    2018/06/29 09:14:52 [notice] 10009#10009: signal 1 (SIGHUP) received from 10206, reconfiguring

    2018/06/29 09:14:52 [notice] 10009#10009: reconfiguring

    2018/06/29 09:14:52 [notice] 10009#10009: using the "epoll" event method

    2018/06/29 09:14:52 [notice] 10009#10009: start worker processes

    2018/06/29 09:14:52 [notice] 10009#10009: start worker process 10207

    2018/06/29 09:14:52 [notice] 10011#10011: gracefully shutting down

    2018/06/29 09:14:52 [notice] 10011#10011: exiting

    2018/06/29 09:14:52 [notice] 10011#10011: exit

    2018/06/29 09:14:52 [notice] 10009#10009: signal 17 (SIGCHLD) received from 10011

    2018/06/29 09:14:52 [notice] 10009#10009: worker process 10011 exited with code 0

    2018/06/29 09:14:52 [notice] 10009#10009: signal 29 (SIGIO) received

    2018/06/29 09:14:58 [notice] 10009#10009: signal 1 (SIGHUP) received from 10209, reconfiguring

    2018/06/29 09:14:58 [notice] 10009#10009: reconfiguring

    2018/06/29 09:14:58 [notice] 10009#10009: using the "epoll" event method

    2018/06/29 09:14:58 [notice] 10009#10009: start worker processes

    2018/06/29 09:14:58 [notice] 10009#10009: start worker process 10210

    2018/06/29 09:14:58 [notice] 10207#10207: gracefully shutting down

    2018/06/29 09:14:58 [notice] 10207#10207: exiting

    2018/06/29 09:14:58 [notice] 10207#10207: exit

    2018/06/29 09:14:58 [notice] 10009#10009: signal 17 (SIGCHLD) received from 10207

    2018/06/29 09:14:58 [notice] 10009#10009: worker process 10207 exited with code 0

    2018/06/29 09:14:58 [notice] 10009#10009: signal 29 (SIGIO) received
    kmdd33
        9
    kmdd33  
    OP
       2018-06-29 17:23:05 +08:00
    @wqsfree
    /var/run/nginx/nginx.pid 仅仅输出了

    10009

    这个提示。请问我应该修改什么地方呢?服务器 ip 还是无法访问
    kmdd33
        10
    kmdd33  
    OP
       2018-06-29 18:02:42 +08:00
    @Livid @est @sajesemuy @wqsfree @yaxin 最新的 /usr/local/openresty/nginx/log/error.log


    8/06/28 18:04:18 [notice] 32331#32331: signal process started
    2018/06/28 22:19:32 [emerg] 2711#2711: bind() to 127.0.0.1:80 failed (98: Address already in use)
    2018/06/28 22:19:32 [emerg] 2711#2711: bind() to 127.0.0.1:80 failed (98: Address already in use)
    2018/06/28 22:19:32 [emerg] 2711#2711: bind() to 127.0.0.1:80 failed (98: Address already in use)
    2018/06/28 22:19:32 [emerg] 2711#2711: bind() to 127.0.0.1:80 failed (98: Address already in use)
    2018/06/28 22:19:32 [emerg] 2711#2711: bind() to 127.0.0.1:80 failed (98: Address already in use)
    2018/06/28 22:19:32 [emerg] 2711#2711: still could not bind()
    2018/06/29 08:38:40 [emerg] 9553#9553: open() "/var/run/nginx/nginx.pid" failed (2: No such file or directory)
    2018/06/29 08:39:35 [emerg] 9589#9589: open() "/var/run/nginx/nginx.pid" failed (2: No such file or directory)
    2018/06/29 08:51:43 [emerg] 9757#9757: open() "/var/log/nginx/error.log" failed (2: No such file or directory)
    2018/06/29 08:55:53 [notice] 9801#9801: signal process started
    2018/06/29 08:55:53 [error] 9801#9801: invalid PID number "" in "/var/run/nginx/nginx.pid"
    2018/06/29 09:09:34 [notice] 10112#10112: signal process started
    2018/06/29 09:09:34 [error] 10112#10112: invalid PID number "" in "/usr/local/openresty/nginx/logs/nginx.pid"
    2018/06/29 09:14:52 [notice] 10206#10206: signal process started
    2018/06/29 09:14:58 [notice] 10209#10209: signal process started
    2018/06/29 09:30:18 [notice] 10444#10444: signal process started
    2018/06/29 09:32:03 [notice] 10470#10470: signal process started
    2018/06/29 09:44:49 [notice] 10614#10614: signal process started
    2018/06/29 09:44:49 [alert] 10614#10614: kill(10009, 1) failed (3: No such process)
    2018/06/29 09:45:05 [notice] 10616#10616: signal process started
    2018/06/29 09:45:05 [alert] 10616#10616: kill(10009, 1) failed (3: No such process)
    2018/06/29 09:50:38 [notice] 10681#10681: signal process started
    2018/06/29 09:50:38 [alert] 10681#10681: kill(10009, 1) failed (3: No such process)
    关于     帮助文档     自助推广系统     博客     API     FAQ     Solana     5632 人在线   最高记录 6679       Select Language
    创意工作者们的社区
    World is powered by solitude
    VERSION: 3.9.8.5 27ms UTC 06:31 PVG 14:31 LAX 23:31 JFK 02:31
    Do have faith in what you're doing.
    ubao snddm index pchome yahoo rakuten mypaper meadowduck bidyahoo youbao zxmzxm asda bnvcg cvbfg dfscv mmhjk xxddc yybgb zznbn ccubao uaitu acv GXCV ET GDG YH FG BCVB FJFH CBRE CBC GDG ET54 WRWR RWER WREW WRWER RWER SDG EW SF DSFSF fbbs ubao fhd dfg ewr dg df ewwr ewwr et ruyut utut dfg fgd gdfgt etg dfgt dfgd ert4 gd fgg wr 235 wer3 we vsdf sdf gdf ert xcv sdf rwer hfd dfg cvb rwf afb dfh jgh bmn lgh rty gfds cxv xcv xcs vdas fdf fgd cv sdf tert sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf sdf shasha9178 shasha9178 shasha9178 shasha9178 shasha9178 liflif2 liflif2 liflif2 liflif2 liflif2 liblib3 liblib3 liblib3 liblib3 liblib3 zhazha444 zhazha444 zhazha444 zhazha444 zhazha444 dende5 dende denden denden2 denden21 fenfen9 fenf619 fen619 fenfe9 fe619 sdf sdf sdf sdf sdf zhazh90 zhazh0 zhaa50 zha90 zh590 zho zhoz zhozh zhozho zhozho2 lislis lls95 lili95 lils5 liss9 sdf0ty987 sdft876 sdft9876 sdf09876 sd0t9876 sdf0ty98 sdf0976 sdf0ty986 sdf0ty96 sdf0t76 sdf0876 df0ty98 sf0t876 sd0ty76 sdy76 sdf76 sdf0t76 sdf0ty9 sdf0ty98 sdf0ty987 sdf0ty98 sdf6676 sdf876 sd876 sd876 sdf6 sdf6 sdf9876 sdf0t sdf06 sdf0ty9776 sdf0ty9776 sdf0ty76 sdf8876 sdf0t sd6 sdf06 s688876 sd688 sdf86