Get the most out of your Centmin Mod LEMP stack
Become a Member

Wordpress Nginx returns 404 robots.txt

Discussion in 'Blogs & CMS usage' started by cloud9, Feb 15, 2024.

  1. cloud9

    cloud9 Premium Member Premium Member

    423
    117
    43
    Oct 6, 2015
    England
    Ratings:
    +216
    Local Time:
    2:07 PM
    1.25.3
    10.6.x
    Latest CMM
    AlmaLinux 8.x
    Latest Nginx

    Am using the AIOSEO plugin (but dont think its this) as it uses the native WP robots.txt virtual file

    So
    Code:
    https://mydomain.com/robots.txt returns a 404
    
    https://mydomain.com/?robots=1 returns the robots.txt file
    
    https://mydomain.com/index.php?robots=1 goes to https://mydomain.com/?robots=1 and returns the robots.txt file
    
    My config is here

    Code:
      location / {
      include /usr/local/nginx/conf/503include-only.conf;
    
      # Nginx level redis Wordpress
      # https://community.centminmod.com/posts/18828/
      try_files $uri $uri/ /index.php?$args;
    
      }
    
    location ~* /(wp-login\.php) {
        limit_req zone=xwplogin burst=1 nodelay;
    
        # https://community.centminmod.com/posts/18828/
        include /usr/local/nginx/conf/php-rediscache.conf;
    }
    
    location ~* /(xmlrpc\.php) {
        limit_req zone=xwprpc burst=45 nodelay;
    
        # https://community.centminmod.com/posts/18828/
        include /usr/local/nginx/conf/php-rediscache.conf;
    }
    
    location ~* /(xmlrpc\.php) {
        limit_req zone=xwprpc burst=45 nodelay;
        # https://jetpack.com/support/hosting-faq/
        include /usr/local/nginx/conf/jetpack_whitelist_ip.conf;
    
        # https://community.centminmod.com/posts/18828/
        include /usr/local/nginx/conf/php-rediscache.conf;
    }
    
    location ~* /wp-admin/(load-scripts\.php) {
        limit_req zone=xwprpc burst=5 nodelay;
    
        # https://community.centminmod.com/posts/18828/
        include /usr/local/nginx/conf/php-rediscache.conf;
    }
    
    location ~* /wp-admin/(load-scripts\.php) {
        limit_req zone=xwprpc burst=5 nodelay;
    
        # https://community.centminmod.com/posts/18828/
        include /usr/local/nginx/conf/php-rediscache.conf;
    }
    
    # HOTLINK PROTECTION
    invalid_referer and return “403: Access Forbidden” response.
        location ~ .(gif|png|psd|tiff|eps|ai|raw|jpe?g)$ {
          valid_referers none blocked mydomain.co.uk;
              if ($invalid_referer) {
              return 403;
              }
        }
    
    #To secure the admin area - if you have a static IP address allow only your IP and denay all other
        location /admin.php {
                include /usr/local/nginx/conf/php.conf;
                allow 127.0.0.1;
                allow IPS IN HERE;
                deny all;
        }
    
    location ~* /wp-admin/(load-styles\.php) {
        limit_req zone=xwprpc burst=5 nodelay;
    
        # https://community.centminmod.com/posts/18828/
        include /usr/local/nginx/conf/php-rediscache.conf;
    }
               
    
      include /usr/local/nginx/conf/wpincludes/xxxxxx.co.uk/wpsecure_xxxxxx.co.uk.conf;
    
      # https://community.centminmod.com/posts/18828/
      include /usr/local/nginx/conf/php-rediscache.conf;
      include /usr/local/nginx/conf/pre-staticfiles-local-xxxxxx.co.uk.conf;
      include /usr/local/nginx/conf/pre-staticfiles-global.conf;
      include /usr/local/nginx/conf/staticfiles.conf;
      include /usr/local/nginx/conf/drop.conf;
      include /usr/local/nginx/conf/vts_server.conf;
    }
    
    I have tried varies tryfiles and rewrites - including one that AIOSEO recommends


    Nothing I try will display the robots.txt file - just gives a 404

    Am I missing something ?
     
  2. cloud9

    cloud9 Premium Member Premium Member

    423
    117
    43
    Oct 6, 2015
    England
    Ratings:
    +216
    Local Time:
    2:07 PM
    1.25.3
    10.6.x
    If I add this in the server block (before the first location) then it works

    Code:
    #AIOSEO robots.txt file redirect ro ?robots=1 for Wordpress
      rewrite ^/robots.txt$ /index.php?robots=1 last;
      rewrite ^/([_0-9a-zA-Z-]+)/robots.txt$ /$1/index.php?robots=1;
    However I read somewhere rewrite isnt great and its better to use location and try files

    However cant find a working solution
     
  3. eva2000

    eva2000 Administrator Staff Member

    51,726
    11,944
    113
    May 24, 2014
    Brisbane, Australia
    Ratings:
    +18,436
    Local Time:
    11:07 PM
    Nginx 1.25.x
    MariaDB 10.x
    Probably best solution.