Discover Centmin Mod today
Register Now

Nginx Nginx down when DDoS

Discussion in 'Nginx, PHP-FPM & MariaDB MySQL' started by Oxide, Apr 18, 2016.

  1. Oxide

    Oxide Active Member

    504
    29
    28
    Mar 19, 2015
    Ratings:
    +54
    Local Time:
    10:25 AM
    When someone are performing a PingBack attack on my server, it goes down.

    The CPU is literally around 0.05 constantly, and i barely recieve any incoming bandwidth.

    I use this to block the attack:
    if ($http_user_agent ~* "PHP|curl|Wget|HTTrack|Nmap|Verifying|PingBack|Pingdom|Joomla|Wordpress") { return 444; }

    I can confirm it's working, however it seems that Nginx still goes down? It's not even reaching my PHP, or anything. It makes no sense. What could be wrong?

    If there are more than enough resources, what could be causing this?

    I changed it to 403 also, it uses zero CPU - and site is still down. Barely any network, something must be wrong with.. nginx?
     
    Last edited: Apr 18, 2016
  2. Oxide

    Oxide Active Member

    504
    29
    28
    Mar 19, 2015
    Ratings:
    +54
    Local Time:
    10:25 AM
    Code:
    user              nginx nginx;
    worker_processes 3;
    worker_priority -10;
    
    worker_rlimit_nofile 260000;
    timer_resolution 100ms;
    
    pcre_jit on;
    
    pid         logs/nginx.pid;
    
    events {
    worker_connections  4096;
    accept_mutex on;
    accept_mutex_delay 200ms;
    use epoll;
    #multi_accept on;
    }
    
    http {
    
    set_real_ip_from 199.27.128.0/21;
    set_real_ip_from 173.245.48.0/20;
    set_real_ip_from 103.21.244.0/22;
    set_real_ip_from 103.22.200.0/22;
    set_real_ip_from 103.31.4.0/22;
    set_real_ip_from 141.101.64.0/18;
    set_real_ip_from 108.162.192.0/18;
    set_real_ip_from 190.93.240.0/20;
    set_real_ip_from 188.114.96.0/20;
    set_real_ip_from 197.234.240.0/22;
    set_real_ip_from 198.41.128.0/17;
    set_real_ip_from 162.158.0.0/15;
    set_real_ip_from 104.16.0.0/12;
    real_ip_header CF-Connecting-IP;
    
    
    include /usr/local/nginx/conf/vts_http.conf;
    include /usr/local/nginx/conf/geoip.conf;
    #include /usr/local/nginx/conf/pagespeedadmin.conf;
    include /usr/local/nginx/conf/fastcgi_param_https_map.conf;
    
    log_format      main    '$remote_addr - $remote_user [$time_local] $request '
    '"$status" $body_bytes_sent "$http_referer" '
    '"$http_user_agent" "$http_x_forwarded_for" "$gzip_ratio"'
    ' "$connection" "$connection_requests" "$request_time"';
    
    access_log  logs/access.log combined buffer=128k flush=5m;
    error_log   logs/error.log warn;
    
    index  index.php index.html index.htm;
    include       mime.types;
    default_type  application/octet-stream;
    charset utf-8;
    
    sendfile on;
    sendfile_max_chunk 512k;
    tcp_nopush  on;
    tcp_nodelay on;
    server_tokens off;
    server_name_in_redirect off;
    
    keepalive_timeout  8;
    keepalive_requests 1000;
    lingering_time 20s;
    lingering_timeout 5s;
    keepalive_disable msie6;
    
    gzip on;
    gzip_vary   on;
    gzip_disable "MSIE [1-6]\.";
    gzip_static on;
    gzip_min_length   1400;
    gzip_buffers      32 8k;
    gzip_http_version 1.0;
    gzip_comp_level 5;
    gzip_proxied    any;
    gzip_types text/plain text/css text/xml application/javascript application/x-javascript application/xml application/xml+rss application/ecmascript application/json image/svg+xml;
    
    client_body_buffer_size 256k;
    client_body_in_file_only off;
    client_body_timeout 10s;
    client_header_buffer_size 64k;
    ## how long a connection has to complete sending
    ## it's headers for request to be processed
    client_header_timeout  8s;
    client_max_body_size 50m;
    connection_pool_size  512;
    directio  4m;
    ignore_invalid_headers on;  
    large_client_header_buffers 8 64k;
    output_buffers   8 256k;
    postpone_output  1460;
    proxy_temp_path  /tmp/nginx_proxy/;
    request_pool_size  32k;
    reset_timedout_connection on;
    send_timeout     15s;
    types_hash_max_size 2048;
    server_names_hash_bucket_size 64;
    
    # for nginx proxy backends to prevent redirects to backend port
    # port_in_redirect off;
    
    open_file_cache max=50000 inactive=60s;
    open_file_cache_valid 120s;
    open_file_cache_min_uses 2;
    open_file_cache_errors off;
    open_log_file_cache max=10000 inactive=30s min_uses=2;
    
    ## limit number of concurrency connections per ip to 16
    ## add to your server {} section the next line
    ## limit_conn limit_per_ip 16;
    ## uncomment below line allows 500K sessions
    # limit_conn_log_level error;
    #######################################
    # use limit_zone for Nginx <v1.1.7 and lower
    # limit_zone $binary_remote_addr zone=limit_per_ip:16m;
    #######################################
    # use limit_conn_zone for Nginx >v1.1.8 and higher
    # limit_conn_zone $binary_remote_addr zone=limit_per_ip:16m;
    #######################################
    
    include /usr/local/nginx/conf/conf.d/*.conf;
    }
    
    There has to be a optimization issue somewhere that I can change, to accept more connections?

    It's like the attackers are eating up all connections, so when accessing web server it times out. All other services, on the machine works fine.

    Code:
    [root@main-ovh ~]# grep processor /proc/cpuinfo | wc -l
    8
    [root@main-ovh ~]# ulimit -n
    262144
    
    Intel(R) Xeon(R) CPU E5-1630 v3 @ 3.70GHz

    /etc/sysctl.conf
    Code:
    # Disable IPv6 autoconf
    net.ipv6.conf.all.autoconf = 0
    net.ipv6.conf.default.autoconf = 0
    net.ipv6.conf.eth0.autoconf = 0
    net.ipv6.conf.all.accept_ra = 0
    net.ipv6.conf.default.accept_ra = 0
    net.ipv6.conf.eth0.accept_ra = 0
    # centminmod added
    fs.nr_open=12000000
    fs.file-max=9000000
    net.core.wmem_max=16777216
    net.core.rmem_max=16777216
    net.ipv4.tcp_rmem=8192 87380 16777216
    net.ipv4.tcp_wmem=8192 65536 16777216
    net.core.netdev_max_backlog=8192
    net.core.somaxconn=65536
    net.core.optmem_max=8192
    net.ipv4.tcp_fin_timeout=10
    net.ipv4.tcp_keepalive_intvl=30
    net.ipv4.tcp_keepalive_probes=3
    net.ipv4.tcp_keepalive_time=240
    net.ipv4.tcp_max_syn_backlog=8192
    net.ipv4.tcp_sack=1
    net.ipv4.tcp_syn_retries=3
    net.ipv4.tcp_synack_retries = 2
    net.ipv4.tcp_tw_recycle = 1
    net.ipv4.tcp_tw_reuse = 1
    vm.swappiness=10
    vm.min_free_kbytes=65536
    net.ipv4.ip_local_port_range=1024 65535
    
    net.ipv4.conf.all.accept_redirects = 0
    net.ipv4.conf.all.accept_source_route = 0
    net.ipv4.conf.all.log_martians = 1
    net.ipv4.conf.all.rp_filter = 1
    net.ipv4.conf.all.secure_redirects = 0
    net.ipv4.conf.all.send_redirects = 0
    net.ipv4.conf.default.accept_redirects = 0
    net.ipv4.conf.default.accept_source_route = 0
    net.ipv4.conf.default.log_martians = 1
    net.ipv4.conf.default.rp_filter = 1
    net.ipv4.conf.default.secure_redirects = 0
    net.ipv4.conf.default.send_redirects = 0
    net.ipv4.icmp_echo_ignore_broadcasts = 1
    net.ipv4.icmp_ignore_bogus_error_responses = 1
     
    Last edited: Apr 18, 2016
  3. eva2000

    eva2000 Administrator Staff Member

    30,178
    6,786
    113
    May 24, 2014
    Brisbane, Australia
    Ratings:
    +10,139
    Local Time:
    10:25 AM
    Nginx 1.13.x
    MariaDB 5.5
  4. Oxide

    Oxide Active Member

    504
    29
    28
    Mar 19, 2015
    Ratings:
    +54
    Local Time:
    10:25 AM
    I tried increasing them to like insane amounts (worker connections: 262144) and worker_processes to auto.. no luck.

    CPU was literally at 0% usage, i had 100% left

    it was a pingback attack, same that you suffered from - i am on centos 7 also if that matters.

    Code:
    user              nginx nginx;
    worker_processes auto;
    worker_priority -10;
    
    worker_rlimit_nofile 260000;
    timer_resolution 100ms;
    
    pcre_jit on;
    
    pid         logs/nginx.pid;
    
    events {
    worker_connections  262144;
    accept_mutex on;
    accept_mutex_delay 200ms;
    use epoll;
    #multi_accept on;
    }
    Code:
    [root@main-ovh ~]# ulimit -n
    262144
     
  5. eva2000

    eva2000 Administrator Staff Member

    30,178
    6,786
    113
    May 24, 2014
    Brisbane, Australia
    Ratings:
    +10,139
    Local Time:
    10:25 AM
    Nginx 1.13.x
    MariaDB 5.5
    Nginx debug mode and troubleshooting in general.

    Nginx Debug Mode



    Latest update on September 14, 2015 to Centmin Mod 1.2.3-eva2000.08 stable (123.08stable) and higher has added Nginx debug mode support. Nginx debug mode is disabled by default.

    To enable, edit centmin.sh and change variable NGINX_DEBUG=n default to NGINX_DEBUG=y and recompile Nginx via centmin.sh menu option 4 and then setting error_log in nginx vhosts to debug as outlined at nginx.org/en/docs/debugging_log.html & wiki.nginx.org/Debugging.

    You do not want to leave Nginx debug mode running forever, so after debugging, set NGINX_DEBUG=n in centmin.sh and recompile Nginx again via centmin.sh menu option 4 and remove error_log debugging mode to disable Nginx debug mode again.

    To update your Centmin Mod builds code for Nginx debug mode support if you do not have a NGINX_DEBUG variable in centmin.sh, follow instructions at centminmod.com/upgrade.html and respective version threads below:
    Centmin Mod is provided as is, but you can try debugging mode for Nginx for further troubleshooting if you have problems with Nginx (i.e. segfaults / signal 11 issues) as outlined at nginx.org/en/docs/debugging_log.html & wiki.nginx.org/Debugging.

    Troubleshooting Tools


    Tools and commands you will want to read up on and learn for basic system admin tasks and troubleshooting.
    Notes:
     
  6. Oxide

    Oxide Active Member

    504
    29
    28
    Mar 19, 2015
    Ratings:
    +54
    Local Time:
    10:25 AM
    Are there no other way other than enabling debug mode in nginx? Rather not re-compile it. Not even knowing if that will help
     
  7. eva2000

    eva2000 Administrator Staff Member

    30,178
    6,786
    113
    May 24, 2014
    Brisbane, Australia
    Ratings:
    +10,139
    Local Time:
    10:25 AM
    Nginx 1.13.x
    MariaDB 5.5
    nope.. debug mode needs nginx recompile A debugging log
     
  8. Oxide

    Oxide Active Member

    504
    29
    28
    Mar 19, 2015
    Ratings:
    +54
    Local Time:
    10:25 AM
    Okay.

    I will post logs here.

    Got a friend of mine that can test the attack on me, also noticed I got a test server that I can test it on :D
     
  9. eva2000

    eva2000 Administrator Staff Member

    30,178
    6,786
    113
    May 24, 2014
    Brisbane, Australia
    Ratings:
    +10,139
    Local Time:
    10:25 AM
    Nginx 1.13.x
    MariaDB 5.5
    no no.. I can't help you with troublshooting the logs as Centmin Mod is provided as is. Just pointing you to how you can get the info yourself to troubleshoot it.
     
  10. Oxide

    Oxide Active Member

    504
    29
    28
    Mar 19, 2015
    Ratings:
    +54
    Local Time:
    10:25 AM
    so i paid $60+ just for dbbackup.sh? damn lol would think you could be friendly and provide some basic help if you knew what were wrong..
     
  11. eva2000

    eva2000 Administrator Staff Member

    30,178
    6,786
    113
    May 24, 2014
    Brisbane, Australia
    Ratings:
    +10,139
    Local Time:
    10:25 AM
    Nginx 1.13.x
    MariaDB 5.5
    Centmin Mod is provide as is, so short of scripted related bugs or issues, any further optimisation to the web stack components - nginx, php-fpm, mariadb mysql, csf firewall etc or web app specific configurations are left to the Centmin Mod user to deal with. So I do not provide any free support for such.

    However, Centmin Mod users are free to help each other out and ask questions or give answers on this community forum. My hopes are that this community forum evolves so that more veteran long time Centmin Mod users help new Centmin Mod users out :)

    So yes other members are free to help if they want.
     
  12. eva2000

    eva2000 Administrator Staff Member

    30,178
    6,786
    113
    May 24, 2014
    Brisbane, Australia
    Ratings:
    +10,139
    Local Time:
    10:25 AM
    Nginx 1.13.x
    MariaDB 5.5
    from Premium User Membership Explained | Centmin Mod Community
    if you need paid help, you'd want to hire someone specifically.
     
  13. Oxide

    Oxide Active Member

    504
    29
    28
    Mar 19, 2015
    Ratings:
    +54
    Local Time:
    10:25 AM
    yes but the way you responded "NO NO" was pretty cocky, i didn't even direct to you - i've been here long enough to know that you will respond with your default txt
     
  14. eva2000

    eva2000 Administrator Staff Member

    30,178
    6,786
    113
    May 24, 2014
    Brisbane, Australia
    Ratings:
    +10,139
    Local Time:
    10:25 AM
    Nginx 1.13.x
    MariaDB 5.5
    sorry if it came off that way to you, the replies I made are not just for you but for anyone reading this post i.e. other members or visitors so they know the policy as well. That's why the replies specifically mention other members can help.

    As I am only 1 person manning the forums, rather than answer every same or similar question over and over, the default text which applies to the topic is used so that everyone reading knows. Bare in mind every topic posted on a forum is not just for your benefit, but for everyone that reads it. That is what cuts down the amount of effort and time I spent on support which I do for free.
     
    • Like Like x 1
  15. Oxide

    Oxide Active Member

    504
    29
    28
    Mar 19, 2015
    Ratings:
    +54
    Local Time:
    10:25 AM
    I checked now.

    I
    Code:
    host: "51.255.203.181"
    2016/04/18 18:45:08 [notice] 7194#7194: *20947 "PHP|curl|Wget|HTTrack|Verifying|PingBack|Joomla|Wordpress" matches "WordPress/3.7.5; http://legestart.ro; verifying pingback from 185.103.252.174", client: 93.190.146.134, server: vps236531.ovh.net, request: "GET / HTTP/1.0", host: "51.255.203.181"
    2016/04/18 18:45:08 [notice] 7194#7194: *21031 "PHP|curl|Wget|HTTrack|Verifying|PingBack|Joomla|Wordpress" matches "WordPress/4.0.1; http://lunawalkstudios.com; verifying pingback from 185.103.252.174", client: 52.76.216.152, server: vps236531.ovh.net, request: "GET / HTTP/1.0", host: "51.255.203.181"
    2016/04/18 18:45:08 [notice] 7192#7192: *21032 "PHP|curl|Wget|HTTrack|Verifying|PingBack|Joomla|Wordpress" matches "WordPress/3.1.3; http://www.izuminki.com", client: 188.120.236.141, server: vps236531.ovh.net, request: "GET / HTTP/1.0", host: "51.255.203.181"
    2016/04/18 18:45:08 [notice] 7194#7194: *21033 "PHP|curl|Wget|HTTrack|Verifying|PingBack|Joomla|Wordpress" matches "WordPress/3.6.1; http://yunev.com", client: 5.9.153.84, server: vps236531.ovh.net, request: "GET / HTTP/1.0", host: "51.255.203.181"
    2016/04/18 18:45:08 [notice] 7194#7194: *21001 "PHP|curl|Wget|HTTrack|Verifying|PingBack|Joomla|Wordpress" matches "WordPress/4.1.10; http://kshownow.net; verifying pingback from 185.106.92.140", client: 162.253.32.117, server: vps236531.ovh.net, request: "GET / HTTP/1.0", host: "51.255.203.181"
    2016/04/18 18:45:08 [notice] 7192#7192: *20967 "PHP|curl|Wget|HTTrack|Verifying|PingBack|Joomla|Wordpress" matches "WordPress/3.5.2; http://dogtv.com", client: 95.183.5.149, server: vps236531.ovh.net, request: "GET / HTTP/1.0", host: "51.255.203.181"
    2016/04/18 18:45:08 [notice] 7192#7192: *21034 "PHP|curl|Wget|HTTrack|Verifying|PingBack|Joomla|Wordpress" matches "WordPress/3.6.1; http://www.automationgame.com", client: 192.99.45.38, server: vps236531.ovh.net, request: "GET / HTTP/1.0", host: "51.255.203.181"
    2016/04/18 18:45:08 [notice] 7194#7194: *21035 "PHP|curl|Wget|HTTrack|Verifying|PingBack|Joomla|Wordpress" matches "WordPress/3.0.4; http://semimd.com", client: 173.192.66.163, server: vps236531.ovh.net, request: "GET / HTTP/1.0", host: "51.255.203.181"
    2016/04/18 18:45:08 [notice] 7194#7194: *21036 "PHP|curl|Wget|HTTrack|Verifying|PingBack|Joomla|Wordpress" matches "WordPress/3.4.2; http://www.fialka17.ru", client: 90.156.197.82, server: vps236531.ovh.net, request: "GET / HTTP/1.0", host: "51.255.203.181"
    2016/04/18 18:45:08 [notice] 7194#7194: *21037 "PHP|curl|Wget|HTTrack|Verifying|PingBack|Joomla|Wordpress" matches "WordPress/3.9; http://veedio.info; verifying pingback from 162.158.94.105", client: 64.237.36.34, server: vps236531.ovh.net, request: "GET / HTTP/1.0", host: "51.255.203.181"
    2016/04/18 18:45:08 [notice] 7192#7192: *21038 "PHP|curl|Wget|HTTrack|Verifying|PingBack|Joomla|Wordpress" matches "WordPress/4.1.10; http://fischeraudio.com; verifying pingback from 185.103.252.174", client: 90.156.197.57, server: vps236531.ovh.net, request: "GET / HTTP/1.0", host: "51.255.203.181"
    2016/04/18 18:45:08 [notice] 7192#7192: *21039 "PHP|curl|Wget|HTTrack|Verifying|PingBack|Joomla|Wordpress" matches "WordPress/4.1.10; http://www.iceandmixedfestival.co.nz; verifying pingback from 185.106.92.140", client: 54.252.142.51, server: vps236531.ovh.net, request: "GET / HTTP/1.0", host: "51.255.203.181"
    2016/04/18 18:45:08 [notice] 7194#7194: *21040 "PHP|curl|Wget|HTTrack|Verifying|PingBack|Joomla|Wordpress" matches "WordPress/4.1.10; http://appledigger.ru; verifying pingback from 185.103.252.174", client: 87.242.64.228, server: vps236531.ovh.net, request: "GET / HTTP/1.0", host: "51.255.203.181"
    2016/04/18 18:45:08 [notice] 7192#7192: *21041 "PHP|curl|Wget|HTTrack|Verifying|PingBack|Joomla|Wordpress" matches "WordPress/3.5.1; http://reviewcity.net", client: 69.175.87.226, server: vps236531.ovh.net, request: "GET / HTTP/1.0", host: "51.255.203.181"
    2016/04/18 18:45:08 [notice] 7194#7194: *21042 "PHP|curl|Wget|HTTrack|Verifying|PingBack|Joomla|Wordpress" matches "WordPress/3.6.1; http://www.disenchantedmusical.com", client: 64.71.32.26, server: vps236531.ovh.net, request: "GET / HTTP/1.0", host: "51.255.203.181"
    2016/04/18 18:45:08 [notice] 7194#7194: *21043 "PHP|curl|Wget|HTTrack|Verifying|PingBack|Joomla|Wordpress" matches "WordPress/3.9.11; http://olimpicastereo.com.co; verifying pingback from 185.103.252.173", client: 67.23.227.66, server: vps236531.ovh.net, request: "GET / HTTP/1.0", host: "51.255.203.181"
    2016/04/18 18:45:08 [notice] 7192#7192: *20405 "PHP|curl|Wget|HTTrack|Verifying|PingBack|Joomla|Wordpress" matches "WordPress/3.6; http://www.weightlossforall.com", client: 72.52.227.205, server: vps236531.ovh.net, request: "GET / HTTP/1.0", host: "51.255.203.181"
    2016/04/18 18:45:08 [notice] 7192#7192: *21045 "PHP|curl|Wget|HTTrack|Verifying|PingBack|Joomla|Wordpress" matches "WordPress/3.5; http://slado.cz", client: 78.24.14.178, server: vps236531.ovh.net, request: "GET / HTTP/1.0", host: "51.255.203.181"
    2016/04/18 18:45:08 [notice] 7194#7194: *21044 "PHP|curl|Wget|HTTrack|Verifying|PingBack|Joomla|Wordpress" matches "WordPress/3.0.4; http://semimd.com", client: 173.192.66.163, server: vps236531.ovh.net, request: "GET / HTTP/1.0", host: "51.255.203.181"
    2016/04/18 18:45:08 [notice] 7194#7194: *21024 "PHP|curl|Wget|HTTrack|Verifying|PingBack|Joomla|Wordpress" does not match "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.75 Safari/537.36", client: 91.34.26.106, server: vps236531.ovh.net, request: "GET / HTTP/1.1", host: "51.255.203.181"
    2016/04/18 18:45:08 [notice] 7194#7194: *21024 "PHP|curl|Wget|HTTrack|Verifying|PingBack|Joomla|Wordpress" does not match "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.75 Safari/537.36", client: 91.34.26.106, server: vps236531.ovh.net, request: "GET / HTTP/1.1", host: "51.255.203.181"
    91.34.26.106 is my ip, when i tried to connect - it timed out.
     
  16. Oxide

    Oxide Active Member

    504
    29
    28
    Mar 19, 2015
    Ratings:
    +54
    Local Time:
    10:25 AM
    Code:
    2016/04/18 18:45:31 [info] 7194#7194: *25 client 141.101.98.134 closed keepalive connection
    
    2016/04/18 18:45:32 [error] 7192#7192: *93 connect() failed (110: Connection timed out) while connecting to upstream, client: 5.187.21.58, server: skidtools.net, request: "GET /ajax/checkIPLogs.php?latest=1457408612 HTTP/1.1", upstream: "fastcgi://127.0.0.1:9000", host: "skidtools.net", referrer: "http://skidtools.net/iplogger"
    
    
    That's about it, nothing else..
     
  17. Oxide

    Oxide Active Member

    504
    29
    28
    Mar 19, 2015
    Ratings:
    +54
    Local Time:
    10:25 AM
    [​IMG]
     
  18. Oxide

    Oxide Active Member

    504
    29
    28
    Mar 19, 2015
    Ratings:
    +54
    Local Time:
    10:25 AM
    Does this mean that PHP, is timing out when Nginx is under attack?
     
  19. pamamolf

    pamamolf Well-Known Member

    2,725
    243
    63
    May 31, 2014
    Ratings:
    +434
    Local Time:
    3:25 AM
    Nginx-1.13.x
    MariaDB 10.1.x
  20. Oxide

    Oxide Active Member

    504
    29
    28
    Mar 19, 2015
    Ratings:
    +54
    Local Time:
    10:25 AM