查看“Nginx location robots.txt”的源代码

 location =/robots.txt {
     root /data/wwwroot/xxx.com/;
     #单独rewrite一个文件要设置index等于这个文件
     index robots.txt;
  }

 location =/robots.txt {
        default_type text/html;
        add_header Content-Type "text/plain; charset=UTF-8";
        return 200 "User-Agent: *\nDisallow: /";
    }

 location = /robots.txt {
    if ($http_user_agent !~* "spider|bot|yahoo|sogou") {
        return 404;
    }
 }