{"id":8200,"date":"2018-03-29T16:57:00","date_gmt":"2018-03-29T08:57:00","guid":{"rendered":"https:\/\/www.awoo.com.tw\/blog\/?p=8200"},"modified":"2023-09-06T13:12:58","modified_gmt":"2023-09-06T05:12:58","slug":"robotstxt-crawl","status":"publish","type":"post","link":"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/","title":{"rendered":"Robots.txt\u7528\u9014\u8207\u4f7f\u7528\u7bc4\u4f8b\u6559\u5b78\uff0c\u91d0\u6e05SEO\u6536\u9304\u89c0\u5ff5\uff01"},"content":{"rendered":"<div id=\"ez-toc-container\" class=\"ez-toc-v2_0_78 counter-hierarchy ez-toc-counter ez-toc-grey ez-toc-container-direction\">\n<div class=\"ez-toc-title-container\">\n<p class=\"ez-toc-title\" style=\"cursor:inherit\">\u76ee\u9304<\/p>\n<span class=\"ez-toc-title-toggle\"><a href=\"#\" class=\"ez-toc-pull-right ez-toc-btn ez-toc-btn-xs ez-toc-btn-default ez-toc-toggle\" aria-label=\"\u986f\u793a\/\u96b1\u85cf\u5167\u5bb9\u76ee\u9304\"><span class=\"ez-toc-js-icon-con\"><span class=\"\"><span class=\"eztoc-hide\" style=\"display:none;\">Toggle<\/span><span class=\"ez-toc-icon-toggle-span\"><svg style=\"fill: #999;color:#999\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" class=\"list-377408\" width=\"20px\" height=\"20px\" viewBox=\"0 0 24 24\" fill=\"none\"><path d=\"M6 6H4v2h2V6zm14 0H8v2h12V6zM4 11h2v2H4v-2zm16 0H8v2h12v-2zM4 16h2v2H4v-2zm16 0H8v2h12v-2z\" fill=\"currentColor\"><\/path><\/svg><svg style=\"fill: #999;color:#999\" class=\"arrow-unsorted-368013\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"10px\" height=\"10px\" viewBox=\"0 0 24 24\" version=\"1.2\" baseProfile=\"tiny\"><path d=\"M18.2 9.3l-6.2-6.3-6.2 6.3c-.2.2-.3.4-.3.7s.1.5.3.7c.2.2.4.3.7.3h11c.3 0 .5-.1.7-.3.2-.2.3-.5.3-.7s-.1-.5-.3-.7zM5.8 14.7l6.2 6.3 6.2-6.3c.2-.2.3-.5.3-.7s-.1-.5-.3-.7c-.2-.2-.4-.3-.7-.3h-11c-.3 0-.5.1-.7.3-.2.2-.3.5-.3.7s.1.5.3.7z\"\/><\/svg><\/span><\/span><\/span><\/a><\/span><\/div>\n<nav><ul class='ez-toc-list ez-toc-list-level-1 ' ><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-1\" href=\"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/#%E7%B6%B2%E7%AB%99%E4%B8%8A%E7%B7%9A%EF%BC%8C%E5%8F%AA%E8%83%BD%E5%85%A8%E7%AB%99%E8%A2%AB%E6%94%B6%E9%8C%84%E5%97%8E%EF%BC%9F\" >\u7db2\u7ad9\u4e0a\u7dda\uff0c\u53ea\u80fd\u5168\u7ad9\u88ab\u6536\u9304\u55ce\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-2\" href=\"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/#%E6%90%9C%E5%B0%8B%E5%BC%95%E6%93%8E%E5%88%B0%E5%BA%95%E5%B0%8D%E7%B6%B2%E7%AB%99%E5%81%9A%E4%BA%86%E5%93%AA%E4%BA%9B%E4%BA%8B%EF%BC%9F\" >\u641c\u5c0b\u5f15\u64ce\u5230\u5e95\u5c0d\u7db2\u7ad9\u505a\u4e86\u54ea\u4e9b\u4e8b\uff1f<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-3\" href=\"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/#%E7%82%BA%E4%BD%95%E6%9C%89%E7%B6%B2%E9%A0%81%E4%B8%8D%E6%83%B3%E8%A2%AB%E6%94%B6%E9%8C%84%E7%9A%84%E5%8F%AF%E8%83%BD%EF%BC%9F\" >\u70ba\u4f55\u6709\u7db2\u9801\u4e0d\u60f3\u88ab\u6536\u9304\u7684\u53ef\u80fd\uff1f<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-4\" href=\"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/#Robotstxt_%E6%80%8E%E9%BA%BC%E5%81%9A%EF%BC%9F\" >Robots.txt \u600e\u9ebc\u505a\uff1f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-5\" href=\"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/#Robotstxt_%E6%80%8E%E9%BA%BC%E7%94%A8%EF%BC%9F\" >Robots.txt \u600e\u9ebc\u7528\uff1f<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-6\" href=\"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/#%E4%BD%BF%E7%94%A8%E7%AF%84%E4%BE%8B%E5%8F%83%E8%80%83\" >\u4f7f\u7528\u7bc4\u4f8b\u53c3\u8003<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-7\" href=\"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/#%E5%9F%BA%E6%9C%AC%E6%87%89%E7%94%A8\" >\u57fa\u672c\u61c9\u7528<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-8\" href=\"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/#%E9%80%B2%E9%9A%8E%E6%87%89%E7%94%A8\" >\u9032\u968e\u61c9\u7528<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-9\" href=\"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/#Robotstxt%E6%B8%AC%E8%A9%A6%E6%96%B9%E5%BC%8F\" >Robots.txt\u6e2c\u8a66\u65b9\u5f0f<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-10\" href=\"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/#Robotstxt%E5%B0%8F%E7%B4%B0%E7%AF%80\" >Robots.txt\u5c0f\u7d30\u7bc0<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-11\" href=\"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/#%E7%B5%90%E8%AB%96%EF%BC%8C%E6%89%80%E4%BB%A5Robotstxt%E5%88%B0%E5%BA%95%E9%87%8D%E4%B8%8D%E9%87%8D%E8%A6%81%EF%BC%9F\" >\u7d50\u8ad6\uff0c\u6240\u4ee5Robots.txt\u5230\u5e95\u91cd\u4e0d\u91cd\u8981\uff1f<\/a><\/li><\/ul><\/nav><\/div>\n<h2><span class=\"ez-toc-section\" id=\"%E7%B6%B2%E7%AB%99%E4%B8%8A%E7%B7%9A%EF%BC%8C%E5%8F%AA%E8%83%BD%E5%85%A8%E7%AB%99%E8%A2%AB%E6%94%B6%E9%8C%84%E5%97%8E%EF%BC%9F\"><\/span>\u7db2\u7ad9\u4e0a\u7dda\uff0c\u53ea\u80fd\u5168\u7ad9\u88ab\u6536\u9304\u55ce\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<p>\u7121\u8ad6\u662f\u81ea\u5df1\u67b6\u7ad9\uff0c\u4ea6\u6216\u662f\u5354\u52a9\u5ba2\u6236\u7db2\u7ad9\u554f\u984c\u6392\u9664\u6642\uff0c\u662f\u4e0d\u662f\u7e3d\u6703\u9047\u904e\u300e\u70ba\u4ec0\u9ebc\u6211\u7684\u7db2\u7ad9\u4e0a\u7dda\u9019\u9ebc\u4e45\uff0c\u537b\u90fd\u6c92\u6709\u597d\u7684\u6392\u540d\uff1f\u300f\u9019\u985e\u76f8\u95dc\u7591\u60d1\u5462\uff1f\u4f46\u5728\u6240\u8b02\u300e\u597d\u7684\u6392\u540d\u300f\u9019\u524d\u63d0\u5927\u65b9\u5411\uff0c\u4f60\u662f\u5426\u8a72\u5148\u6ce8\u610f\u300e\u6211\u7684\u7db2\u7ad9\u6709\u6c92\u6709\u88ab\u641c\u5c0b\u5f15\u64ce\u6536\u9304\u4e86\u300f\u9019\u4ef6\u4e8b\u5462\uff1f\u5982\u679c\u4f60\u7684\u7db2\u7ad9\u5728Google\u6216Bing\u4e0a\u9023\u6536\u9304\u90fd\u6c92\u6709\uff0c\u300e\u6392\u540d\u300f\u9019\u4ef6\u4e8b\u5c31\u8b8a\u6210\u662f\u7a7a\u60f3\u4e86\uff0c\u4e0d\u662f\u55ce\uff1f<br \/>\n\u90a3\u9ebc\u4f60\u662f\u5426\u66fe\u7d93\u60f3\u904e\uff0cGoogle\u6216Bing\u7b49\u641c\u5c0b\u5f15\u64ce\uff0c\u5f88\u76e1\u8cac\u7684\u5c07\u7dda\u4e0a\u904b\u4f5c\u7684\u7db2\u7ad9\u6293\u5230\u641c\u5c0b\u5f15\u64ce\u6e05\u55ae\u5167\u505a\u7d22\u5f15\uff0c\u4e26\u63d0\u4f9b\u7d66\u4f7f\u7528\u8005\u5c0b\u627e\u5404\u7a2e\u8cc7\u8a0a\u6642\uff0c\u4ec0\u9ebc\u60c5\u6cc1\u4e0b\u6703\u88ab\u62d2\u7d55\u7d22\u5f15\uff0c\u4ea6\u6216\u8005\u82e5\u6211\u7684\u7db2\u7ad9\u6709\u4e9b\u6771\u897f\u4e0d\u60f3\u88ab\u641c\u5c0b\u5230\u6216&#8221;\u66ab\u6642&#8221;\u4e0d\u60f3\u88ab\u641c\u5c0b\u5230\uff0c\u90a3\u6709\u6c92\u6709\u4ec0\u9ebc\u8fa6\u6cd5\u53ef\u4ee5\u963b\u6b62\u641c\u5c0b\u5f15\u64ce\u5462\uff1f<br \/>\n\u662f\u7684\uff0c\u65b9\u6cd5\u662f\u6709\u7684\uff0c\u5c31\u8b93\u6211\u5011awoo\u5718\u968a\u66ff\u4f60\u5a13\u5a13\u9053\u4f86\uff01<\/p>\n<ol>\n<li>\u662f\u7db2\u9801\u5167\u4f5c\u6cd5\uff0c\u4f7f\u7528\u6240\u8b02\u7684meta\u6a19\u7c64\u963b\u64cb\uff0c\u4f46\u58de\u8655\u662f\u5fc5\u9808\u6bcf\u4e00\u500b\u7db2\u9801\u90fd\u9700\u7368\u7acb\u8a2d\u7f6e\u4f7f\u7528\u3002<\/li>\n<li>\u800c\u7b2c\u4e8c\u7a2e\u65b9\u5f0f\uff0c\u5247\u662f\u53ef\u4ee5\u91dd\u5c0d\u6574\u500b\u7db2\u7ad9\u505a\u898f\u7bc4\u7684\u9650\u5236\uff0c\u9019\u5c31\u662f\u672c\u6b21\u8981\u8b1b\u7684\u4e3b\u984c\uff1aRobots.txt\u3002<\/li>\n<\/ol>\n<h2><span class=\"ez-toc-section\" id=\"%E6%90%9C%E5%B0%8B%E5%BC%95%E6%93%8E%E5%88%B0%E5%BA%95%E5%B0%8D%E7%B6%B2%E7%AB%99%E5%81%9A%E4%BA%86%E5%93%AA%E4%BA%9B%E4%BA%8B%EF%BC%9F\"><\/span>\u641c\u5c0b\u5f15\u64ce\u5230\u5e95\u5c0d\u7db2\u7ad9\u505a\u4e86\u54ea\u4e9b\u4e8b\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<p>\u641c\u5c0b\u5f15\u64ce\u7684\u904b\u4f5c\u539f\u7406\u662f\u4ec0\u9ebc\u5462\uff1f<br \/>\n\u7c21\u55ae\u4f86\u8aaa\uff0c\u641c\u5c0b\u5f15\u64ce\u6703\u5148\u91dd\u5c0d\u7db2\u7ad9\u505aCrawl\uff08\u6aa2\u7d22\uff09\u8207Index\uff08\u7d22\u5f15\uff09\uff0c\u7136\u5f8c\u5c07\u7db2\u7ad9\u8cc7\u8a0a\u6536\u9304\uff0c\u6839\u64da\u5404\u5bb6\u6f14\u7b97\u6cd5\u8a08\u7b97\u5f8c\u505a\u51fa\u6392\u5e8f\uff0c\u63d0\u4f9b\u641c\u5c0b\u7d50\u679c\u7d66\u4f7f\u7528\u8005\u67e5\u8a62\u3002\uff08\u7576\u7136\u5176\u4e2d\u9084\u6709\u5f88\u8907\u96dc\u7684\u5404\u7a2e\u8a08\u5206\u65b9\u5f0f\uff0c\u4f46\u9019\u908a\u5c31\u4e0d\u7d30\u8ac7\u4e86\uff09<br \/>\nRobots.txt\u5c31\u662f\u9019\u6642\u6d3e\u4e0a\u7528\u5834\uff0c\u4e3b\u8981\u884c\u70ba\u5c31\u662f\u5728\u641c\u5c0b\u5f15\u64ce\u6aa2\u7d22\u7db2\u7ad9\u6642\uff0c\u544a\u8a34\u5b83\u7db2\u7ad9\u54ea\u4e9b\u5167\u5bb9\u53ef\u4ee5\u88ab\u6aa2\u7d22\uff0c\u54ea\u4e9b\u5167\u5bb9\u53ef\u4ee5\u4e0d\u7528\u88ab\u6aa2\u7d22\u3002<br \/>\n\u4e0d\u904e\u9019\u908a\u6709\u4e00\u9ede\u5f88\u91cd\u8981\u9700\u8aaa\u660e\uff0c\u96d6\u7136 Google \u4e0d\u6703\u5c0d Robots.txt \u6240\u5c01\u9396\u7684\u5167\u5bb9\u9032\u884c\u6aa2\u7d22\u6216\u5efa\u7acb\u7d22\u5f15\uff0c\u4f46\u82e5\u6211\u5011\u900f\u904e\u7db2\u8def\u4e0a\u5176\u4ed6\u7db2\u9801\u7684\u9023\u7d50\u767c\u73fe\u5c01\u9396\u7684\u7db2\u5740\uff0c\u4ecd\u7136\u6703\u5efa\u7acb\u9019\u4e9b\u7db2\u5740\u7684\u7d22\u5f15\u3002\u56e0\u6b64\uff0c\u7db2\u9801\u7db2\u5740\u4ee5\u53ca\u5176\u4ed6\u53ef\u80fd\u7684\u516c\u958b\u8cc7\u8a0a (\u4f8b\u5982\u7db2\u9801\u9023\u7d50\u4e2d\u7684\u9328\u9ede\u6587\u5b57) \u4ecd\u7136\u6703\u51fa\u73fe\u5728 Google \u641c\u5c0b\u7d50\u679c\u4e2d\u3002\u5982\u8981\u5b8c\u5168\u907f\u514d\u9019\u7a2e\u60c5\u5f62\uff0c\u5efa\u8b70\u60a8\u4f7f\u7528\u5bc6\u78bc\u4fdd\u8b77\u4f3a\u670d\u5668\u4e0a\u7684\u6a94\u6848\uff0c\u6216\u662f\u4f7f\u7528 noindex \u4e2d\u7e7c\u6a19\u8a18\u6216\u56de\u61c9\u6a19\u982d (\u6216\u5b8c\u5168\u79fb\u9664\u7db2\u9801)\u3002<br \/>\n\u66f4\u591a\u7981\u6b62\u7db2\u9801\u88ab\u6536\u9304\u7684\u65b9\u6cd5\u4ea6\u53ef\u53c3\u8003\u6211\u5011awoo\u5148\u524d\u7684\u6587\u7ae0\uff1a<a href=\"https:\/\/www.awoo.ai\/zh-hant\/blog\/no-index\/\">\u4f7f\u7528noindex\u201d\u7b494\u7a2e\u65b9\u5f0f\u7981\u6b62\u7279\u5b9a\u7db2\u9801\u88ab\u6536\u9304\uff0c\u63d0\u5347\u7db2\u7ad9\u6574\u9ad4seo\u54c1\u8cea<\/a><\/p>\n<blockquote><p>\u82e5\u60f3\u8fd1\u4e00\u6b65\u4e86\u89e3\u6aa2\u7d22\u8207\u7d22\u5f15\uff0c\u53ef\u53c3\u8003Google\u63d0\u4f9b\u7684<a href=\"https:\/\/www.google.com\/intl\/zh-TW\/insidesearch\/howsearchworks\/crawling-indexing.html\">\u6aa2\u7d22\u8207\u5efa\u7acb\u7d22\u5f15<\/a><\/p><\/blockquote>\n<h2><\/h2>\n<h3><span class=\"ez-toc-section\" id=\"%E7%82%BA%E4%BD%95%E6%9C%89%E7%B6%B2%E9%A0%81%E4%B8%8D%E6%83%B3%E8%A2%AB%E6%94%B6%E9%8C%84%E7%9A%84%E5%8F%AF%E8%83%BD%EF%BC%9F\"><\/span>\u70ba\u4f55\u6709\u7db2\u9801\u4e0d\u60f3\u88ab\u6536\u9304\u7684\u53ef\u80fd\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<p>\u53ef\u80fd\u6709\u4eba\u597d\u5947\uff0c\u4ec0\u9ebc\u6642\u5019\u6216\u6709\u4e0a\u8ff0\u72c0\u6cc1\u767c\u751f\u5462\uff1f\u7db2\u7ad9\u90fd\u5b8c\u6210\u4e86\uff0c\u5c31\u662f\u5e0c\u671b\u4ed6\u53ef\u4ee5\u88ab\u8490\u9304\u8ddf\u4e0a\u7dda\u4e0d\u662f\u55ce\uff1f<br \/>\n\u9019\u72c0\u6cc1\u6bd4\u8f03\u53ef\u80fd\u51fa\u73fe\u7684\u60c5\u5883\u8207\u53d7\u773e\u6bd4\u8f03\u5e38\u898b\u7684\u53ef\u80fd\u6709\u9019\u4e9b<\/p>\n<ol>\n<li>\u5c1a\u672a\u5b8c\u6210\u7684\u7db2\u7ad9\u4f46\u9700\u4e0a\u7dda\u5be6\u6e2c\u7684\u7db2\u7ad9\uff1a\u6709\u4e9b\u7db2\u7ad9\u53ef\u80fd\u4e0a\u7dda\u662f\u70ba\u4e86\u5354\u4f5c\u6e2c\u8a66\uff0c\u4ea6\u6216\u8005\u7528\u5de5\u5177\u505a\u58d3\u529b\u6e2c\u8a66\uff0c\u4f46\u6e2c\u8a66\u968e\u6bb5\u53c8\u4e0d\u60f3\u88ab\u641c\u5c0b\u5f15\u64ce\u6aa2\u7d22\u5167\u5bb9\uff0c\u9019\u6642\u5c31\u53ef\u4ee5\u7528\u5230Robots.txt\u505a\u6392\u9664\u4e86\u3002\uff08\u4f46\u5728\u9019\u5efa\u8b70<a href=\"https:\/\/www.awoo.ai\/zh-hant\/blog\/no-index\/\">\u642d\u914dnoindex\u4f7f\u7528<\/a>\uff0c\u6548\u679c\u6700\u4f73\uff09<\/li>\n<li>\u7db2\u7ad9\u7ba1\u7406\u8005\u5f8c\u53f0\uff1a\u6709\u8a31\u591aCMS\uff08\u5167\u5bb9\u7ba1\u7406\u7cfb\u7d71\uff0c\u5982\uff1aWordpress\uff09\u8207\u81ea\u884c\u67b6\u8a2d\u7684\u7db2\u7ad9\u6703\u63d0\u4f9b\u7ba1\u7406\u8005\u5f8c\u53f0\u767b\u5165\u7684\u5165\u53e3\uff0c\u9019\u4e9b\u901a\u5e38\u662f\u70ba\u4e86\u7db2\u7ad9\u7dad\u8b77\u8207\u7ba1\u7406\u800c\u8a2d\u7f6e\u7684\u5165\u53e3\uff0c\u6c92\u6709\u88ab\u6aa2\u7d22\u7684\u5fc5\u8981\u3002<\/li>\n<li>\u7279\u5b9a\u8cc7\u6599\u593e\u5167\u6a94\u6848\uff1a\u7db2\u7ad9\u5e0c\u671b\u641c\u5c0b\u5f15\u64ce\u6aa2\u7d22\u7684\uff0c\u5f80\u5f80\u662f\u6709\u5167\u5bb9\u7684\u8cc7\u8a0a\uff0c\u8a31\u591a\u5f8c\u53f0\u4f7f\u7528\u7684\u6a94\u6848\uff0c\u5c31\u6703\u4ee5\u8cc7\u6599\u593e\u5f62\u5f0f\u6216\u6b63\u898f\u5b57\u5143\u65b9\u5f0f\uff08\u6b63\u898f\u5b57\u5143\u4f7f\u7528\u65b9\u5f0f\u53ef\u898b\u6b64\u7bc7\u5f8c\u7e8c\u7684\u9032\u968e\u4f7f\u7528\u8aaa\u660e\uff09\u505a\u6392\u9664\u6aa2\u7d22\u7684\u52d5\u4f5c\u3002<\/li>\n<\/ol>\n<p><img decoding=\"async\" src=\"https:\/\/lh4.googleusercontent.com\/kAnDYhi41Tn9JTLzbDoldUWY-nP9AwYLjv_wRu9Wdli1gkcqX4j2M99Dly3eMY_fOGuC0ryAynHsZB1EmlqhWA5wU950BtqJGr-Ku9SfdCsIwnEaGes611f-xS-qbRtryCnoZY8i\" alt=\"Apple\u7684Robots.txt\u6a94\u7bc4\u4f8b\" width=\"443\" height=\"509\" \/><\/p>\n<h2><span class=\"ez-toc-section\" id=\"Robotstxt_%E6%80%8E%E9%BA%BC%E5%81%9A%EF%BC%9F\"><\/span>Robots.txt \u600e\u9ebc\u505a\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<p>\u53ea\u8981\u6709\u6587\u5b57\u7de8\u8f2f\u5668\uff0c\u90fd\u53ef\u4ee5\u5b8c\u6210\uff0c\u6bd4\u8f03\u9700\u8981\u6ce8\u610f\u7684\u662f\u5fc5\u9808\u4f7f\u7528UTF-8 \u7de8\u78bc\u7684\u7d14\u6587\u5b57\u6a94\u624d\u53ef\u4ee5\uff0c\u5982\u679c\u4f7f\u7528\u7684\u5b57\u5143\u7de8\u78bc\u6703\u9020\u6210\u4f7f\u7528\u5230\u975e UTF-8 \u7684\u5b50\u96c6\u7684\u5b57\u5143\uff0c\u9019\u7a2e\u60c5\u6cc1\u53ef\u80fd\u6703\u5c0e\u81f4\u6a94\u6848\u5167\u5bb9\u7684\u5256\u6790\u4e0d\u6b63\u78ba\u3002\u8a73\u7d30\u8aaa\u660e\u898f\u7bc4\u53ef\u53c3\u8003Google\u7684<a href=\"https:\/\/developers.google.com\/search\/reference\/robots_txt?hl=zh-tw\">Robots.txt \u898f\u7bc4<\/a>\u4e2d\u7684\u6a94\u6848\u683c\u5f0f\u8aaa\u660e\u3002<br \/>\n\u6bd4\u8f03\u63a8\u85a6\u7684\u7b2c\u4e09\u65b9\u7de8\u8f2f\u5668\u4e5f\u53ef\u4ee5\u53c3\u8003\uff1a<\/p>\n<ul>\n<li><a href=\"https:\/\/www.sublimetext.com\/\">Sublime<\/a><\/li>\n<li><a href=\"https:\/\/atom.io\/\">Atom<\/a><\/li>\n<li><a href=\"https:\/\/notepad-plus-plus.org\/zh\/\">Notepad++<\/a><\/li>\n<li><a href=\"https:\/\/code.visualstudio.com\/\">Visual Studio Code<\/a><\/li>\n<\/ul>\n<p><img decoding=\"async\" src=\"https:\/\/lh6.googleusercontent.com\/c2YbHJcsIx8zrjqvHZORYss44AJPtCgVmLaySlnAeppTBDTJKcMBs55muqDNz3s9S7RcmOSSDD_5rJ_rSLG0LNkgWeRmsqy1nEe9oWxgF336JphR0KWlIEDlnUdUwW30M7A59ayA\" alt=\"Sublime\u7de8\u8f2fRobots.txt\u6a94\u622a\u5716\" width=\"610\" height=\"348\" \/><\/p>\n<h2><span class=\"ez-toc-section\" id=\"Robotstxt_%E6%80%8E%E9%BA%BC%E7%94%A8%EF%BC%9F\"><\/span>Robots.txt \u600e\u9ebc\u7528\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<p>\u57fa\u672c\u6703\u7528\u7684\u5e7e\u500b\u53c3\u6578\u5206\u5225\u5982\u4e0b\uff1a<\/p>\n<ul>\n<li>User-agent =&gt; \u5b9a\u7fa9\u4e0b\u8ff0\u898f\u5247\u5c0d\u54ea\u4e9b\u641c\u5c0b\u5f15\u64ce\u751f\u6548\uff0c\u5373\u662f\u5c0d\u8c61\u3002<\/li>\n<li>Disallow =&gt; \u6307\u5b9a\u54ea\u4e9b\u76ee\u9304\u6216\u6a94\u6848\u985e\u578b\u4e0d\u60f3\u88ab\u6aa2\u7d22\uff0c\u9700\u6307\u540d\u8def\u5f91\uff0c\u5426\u5247\u5c07\u6703\u88ab\u5ffd\u7565\u3002<\/li>\n<li>Allow =&gt; \u6307\u5b9a\u54ea\u4e9b\u76ee\u9304\u6216\u6a94\u6848\u985e\u578b\u53ef\u80fd\u88ab\u6aa2\u7d22\uff0c\u9700\u6307\u540d\u8def\u5f91\uff0c\u5426\u5247\u5c07\u6703\u88ab\u5ffd\u7565\u3002<\/li>\n<li>Sitemap =&gt; \u6307\u5b9a\u7db2\u7ad9\u5167\u7684sitemap\u6a94\u6848\u653e\u7f6e\u4f4d\u7f6e\uff0c\u9700\u4f7f\u7528\u7d55\u5c0d\u8def\u5f91\u3002<\/li>\n<\/ul>\n<p>\u5ef6\u4f38\u95b1\u8b80\uff1a<a href=\"https:\/\/www.awoo.ai\/zh-hant\/blog\/sitemap-xml\/\" target=\"_blank\" rel=\"noopener noreferrer\">sitemap.xml\u7db2\u7ad9\u5730\u5716\u662f\u4ec0\u9ebc\uff1f\u5f9e\u5de5\u5177\/\u7522\u751f\u5668\/\u7a0b\u5f0f\u88fd\u4f5c\u5230\u9032\u884c\u63d0\u4ea4\u6559\u5b78<\/a><\/p>\n<h3><span class=\"ez-toc-section\" id=\"%E4%BD%BF%E7%94%A8%E7%AF%84%E4%BE%8B%E5%8F%83%E8%80%83\"><\/span>\u4f7f\u7528\u7bc4\u4f8b\u53c3\u8003<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<p>\u5c31\u7b97\u77e5\u9053\u53c3\u6578\uff0c\u4f46\u5f80\u5f80\u770b\u8457\u53c3\u6578\u4f46\u537b\u7121\u5f9e\u4e0b\u624b\u7684\u72c0\u6cc1\u4e5f\u5f88\u983b\u7e41\uff0c\u6240\u4ee5\u5f9e\u7bc4\u4f8b\u4e0b\u624b\uff0c\u662f\u6700\u5bb9\u6613\u7406\u89e3\u4e14\u597d\u5165\u9580\u7684\u65b9\u6cd5\u3002<br \/>\n\u4ee5\u4e0b\u662f\u5e7e\u7a2e\u5e38\u898b\u53ca\u53ef\u80fd\u4f7f\u7528\u5230\u7684\u65b9\u5f0f\uff1a<\/p>\n<h3><span class=\"ez-toc-section\" id=\"%E5%9F%BA%E6%9C%AC%E6%87%89%E7%94%A8\"><\/span>\u57fa\u672c\u61c9\u7528<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<ol>\n<li>\u5141\u8a31\u6240\u6709\u641c\u5c0b\u5f15\u64ce\u6aa2\u7d22\u6240\u6709\u5167\u5bb9(\u901a\u5e38\u5efa\u8b70\u4f7f\u7528)<br \/>\nUser-agent: *<br \/>\nDisallow:<\/li>\n<li>\u62d2\u7d55\u6240\u6709\u641c\u5c0b\u5f15\u64ce\u6aa2\u7d22\u6240\u6709\u5167\u5bb9(\u6b63\u5f0f\u74b0\u5883\u8acb\u907f\u514d\u4f7f\u7528)<br \/>\nUser-agent: *<br \/>\nDisallow: \/<\/li>\n<li>\u62d2\u7d55\u6240\u6709\u641c\u5c0b\u5f15\u64ce\u6aa2\u7d22\/members\/\u5e95\u4e0b\u6240\u6709\u5167\u5bb9\u3002<br \/>\nUser-agent: *<br \/>\nDisallow: \/members\/<\/li>\n<li>\u62d2\u7d55Google\u641c\u5716\u7684\u722c\u87f2\u6aa2\u7d22\/images\/\u5e95\u4e0b\u6240\u6709\u5167\u5bb9\u3002<br \/>\nUser-agent: Googlebot-image<br \/>\nDisallow:\/images\/<\/li>\n<\/ol>\n<h3><span class=\"ez-toc-section\" id=\"%E9%80%B2%E9%9A%8E%E6%87%89%E7%94%A8\"><\/span>\u9032\u968e\u61c9\u7528<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<ol>\n<li>[\u842c\u7528\u5b57\u5143]\u62d2\u7d55\u6240\u6709\u641c\u5c0b\u5f15\u64ce\u6aa2\u7d22\u7db2\u7ad9\u5167png\u70ba\u526f\u6a94\u540d\u7684\u5716\u6a94\u3002<br \/>\nUser-agent: *<br \/>\nDisallow: *.png$<\/li>\n<li>[\u842c\u7528\u5b57\u5143]\u62d2\u7d55Bing\u641c\u5c0b\u5f15\u64ce\u6aa2\u7d22\u7db2\u7ad9\u5167\/wp-admin\u76ee\u9304\u5e95\u4e0b\u6240\u6709\u5167\u5bb9\u53ca\u7db2\u7ad9\u5167\u958b\u982d\u70batest\u7684\u6240\u6709\u6a94\u540d\u3002<br \/>\nUser-agent: bingbot<br \/>\nDisallow: \/wp-admin\/<br \/>\nDisallow: ^test*<\/li>\n<\/ol>\n<h3><span class=\"ez-toc-section\" id=\"Robotstxt%E6%B8%AC%E8%A9%A6%E6%96%B9%E5%BC%8F\"><\/span>Robots.txt\u6e2c\u8a66\u65b9\u5f0f<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<p>\u5982\u679c\u8981\u9a57\u8b49\u81ea\u5df1\u7684\u7db2\u7ad9\u6709\u7121Robots.txt\uff0c\u6700\u7c21\u55ae\u7684\u65b9\u5f0f\u662f\u76f4\u63a5\u5728\u7db2\u7ad9\u6839\u76ee\u9304\u4e0b\u8f38\u5165robotx.txt\u505a\u6e2c\u8a66\uff0c\u6aa2\u9a57\u662f\u5426\u5b58\u5728\u3002<br \/>\n\u4f8b\uff1a<a href=\"https:\/\/www.apple.com\/robots.txt\">https:\/\/www.apple.com\/robots.txt<\/a><br \/>\n\u800c\u82e5\u4e0d\u77e5\u9053\u76ee\u524drobots.txt\u8a9e\u6cd5\u662f\u5426\u6b63\u78ba\uff0c\u4e5f\u53ef\u4ee5\u501f\u52a9Google Search Console\u5de5\u5177\u4f86\u505a\u6e2c\u8a66\u3002<br \/>\n<img decoding=\"async\" src=\"https:\/\/lh3.googleusercontent.com\/CVaC0tt4Q5u1FNonG58HpiOU7zwkzBuDSccKpDLVvPfbdYmWiIculD2L_Rghlk7-N9dK5i8JsJgmMfrs5ap3z8SeqegydNq82wMADgv22NxVp7D1CVWr_cMfkdaPGquBJE1P0JuR\" alt=\"Robots.txt\u65bcGoogle Search Console\u8a9e\u6cd5\u6e2c\u8a66\" width=\"1200\" height=\"556\" \/><br \/>\n\u5728Search Console\u5167\uff0c\u5de6\u5074\u7684\u300e\u6aa2\u7d22\u300f\u5167\uff0c\u9ede\u9078\u300erobots.txt \u6e2c\u8a66\u5de5\u5177\u300f\uff0c\u53ef\u5728\u88e1\u982d\u6e2c\u8a66\u7576\u524drobots.txt\u6a94\u6848\u662f\u5426\u6709\u554f\u984c\uff0c\u4e5f\u53ef\u76f4\u63a5\u67e5\u770b\u5373\u6642robots.txt\u7684\u6a94\u6848\u5167\u5bb9\u3002<br \/>\n\u82e5\u662f\u9084\u662f\u64d4\u5fc3\u81ea\u5df1\u7684\u67d0\u4e9b\u7db2\u5740\u6703\u88abrobots.txt\u8a2d\u7f6e\u5c01\u9396\uff0c\u4e5f\u53ef\u7528\u5de5\u5177\u4e0b\u65b9\u7684\u63d0\u4ea4\u7db2\u5740\u65b9\u5f0f\u505a\u6e2c\u8a66\u3002<br \/>\n<img decoding=\"async\" src=\"https:\/\/lh5.googleusercontent.com\/6VHt2q6MfoXELJpJI42dIyNMcrS9B_4ceyXBqpC2FEyy3CiP_IeFsFAhQ5walW_XINDHzjdTCgKIs2N6mkvXYweAzbaKMOK-9cwIJT7MNMVc0CkwNs9b6GyXH6G_qjUR_oSFLVsw\" alt=\"\u5728Search Console\u5167\u6e2c\u8a66\u7db2\u5740\u662f\u5426\u88abRobots.txt\u6a94\u963b\u64cb\" width=\"963\" height=\"67\" \/><\/p>\n<p>\u540c\u6a23\u7684\u4e5f\u53ef\u4ee5\u5728Bing Webmaster\u5167\uff0c\u6e2c\u8a66\u662f\u5426\u6709\u88abrobots.txt\u6a94\u6848\u963b\u64cb\u7684\u6a94\u6848\u9805\u76ee\u3002<br \/>\n<img decoding=\"async\" src=\"https:\/\/media.awoo.ai\/live\/wp-content\/uploads\/5b86d3c9-b68d-4b89-a00d-00c9105e4454.jpeg\" alt=\"\u4f7f\u7528Bing Webmaster\u6e2c\u8a66\u662f\u5426\u6709\u6a94\u6848\u88abRobots.txt\u963b\u64cb\" width=\"696\" height=\"347\" \/><\/p>\n<h3><span class=\"ez-toc-section\" id=\"Robotstxt%E5%B0%8F%E7%B4%B0%E7%AF%80\"><\/span>Robots.txt\u5c0f\u7d30\u7bc0<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<p>Robots.txt\u7684\u898f\u7bc4\u4e2d\u4e5f\u6709\u8457\u5404\u7a2e\u7d30\u7bc0\u8ddf\u8a2d\u7f6e\uff0c\u96d6\u7136\u975e\u7d44\u6210\u7684\u5fc5\u8981\u5143\u7d20\uff0c\u4f46\u82e5\u61c2\u5f97\u5982\u4f55\u61c9\u7528\uff0c\u5c0d\u8a2d\u7f6eRobots.txt\u76f8\u4fe1\u6703\u66f4\u6709\u5fc3\u5f97\uff1a\uff09<\/p>\n<ol>\n<li>Sitemap\u5728Robots.txt\u5167\u662f\u5c6c\u65bcnon-group \u7d00\u9304\uff0c\u5373\u662f\u4f4d\u7f6e\u4e26\u6c92\u6709\u9650\u5236\uff0c\u4e26\u4e0d\u6703\u56e0\u70baUser-agent\u6216Disallow\u6240\u5f71\u97ff\uff0c\u53ef\u653e\u7f6e\u65bc\u4efb\u4f55\u4f4d\u7f6e\u3002<\/li>\n<\/ol>\n<p>User-agent: bingbot<br \/>\nDisallow: \/wp-admin\/<br \/>\nDisallow: ^test*<br \/>\nSitemap: <a href=\"http:\/\/www.abc.com\/sitemap.xml\">http:\/\/www.abc.com\/sitemap.xml<\/a><\/p>\n<ol start=\"2\">\n<li>Crawl-delay\u53c3\u6578\u53ef\u6307\u5b9a\u641c\u5c0b\u5f15\u64ce\u722c\u87f2\u4f86\u8a2a\u7684\u9593\u9694\u6642\u9593\uff08\u55ae\u4f4d\u70ba\u79d2\uff09\uff0c\u4e0b\u8ff0\u4f8b\u5b50\u5373\u8868\u793a\uff1a\u544a\u8a34\u722c\u87f2\uff0c\u518d\u6b21\u4f86\u8a2a\u7684\u6700\u77ed\u6642\u9593\u9593\u9694\u70ba20\u79d2\u3002<\/li>\n<\/ol>\n<p>User-agent: *<br \/>\nCrawl-deslay: 20<br \/>\n\u6ce8\u610f\uff1a\u767e\u5ea6\u516c\u958b\u8868\u793a\u4e0d\u652f\u63f4\u6b64\u8a9e\u6cd5<\/p>\n<ol start=\"3\">\n<li>\u76ee\u524d\u5927\u578b\u7684\u641c\u5c0b\u5f15\u64ce\uff0c\u5982\uff1aGoogle\u3001Ask\u3001Bing\u3001Yahoo \u5747\u652f\u63f4 Robotx.txt\u5167\u7684Sitemap\u6307\u5411\u3002<\/li>\n<\/ol>\n<p>4.Robots.txt\u7684\u6a94\u6848\u547d\u540d\u65b9\u5f0f\u5fc5\u9808\u70ba\u5c0f\u5beb\u5b57\u6bcd\uff08robots.txt\uff09\uff0c\u82e5\u547d\u540d\u70baRobots.txt\u6216robots.TXT\u7b49\u7686\u70ba\u4e0d\u6b63\u78ba\u7684\u547d\u540d\u65b9\u5f0f\uff0c\u5c07\u6703\u88ab\u641c\u7d22\u5f15\u64ce\u7684\u722c\u87f2\u5ffd\u7565\u3002<\/p>\n<ol start=\"5\">\n<li>Robot.txt\u6a94\u6848\u53ea\u80fd\u653e\u5728\u7db2\u7ad9\u7684\u6839\u76ee\u9304\u5e95\u4e0b\uff0c\u82e5\u975e\u6839\u76ee\u9304\u5247\u4e0d\u6703\u88ab\u767c\u73fe\u3002<\/li>\n<\/ol>\n<p>\u8a73\u7d30\u8cc7\u8a0a\u4e5f\u53ef\u4ee5\u5728Google\u7684<a href=\"https:\/\/developers.google.com\/search\/reference\/robots_txt?hl=zh-tw\">Robots.txt \u898f\u7bc4<\/a>\u78ba\u8a8d\u5176\u4ed6\u4f7f\u7528\u65b9\u5f0f\u8207\u9805\u76ee\u985e\u5225\u3002<\/p>\n<h2><span class=\"ez-toc-section\" id=\"%E7%B5%90%E8%AB%96%EF%BC%8C%E6%89%80%E4%BB%A5Robotstxt%E5%88%B0%E5%BA%95%E9%87%8D%E4%B8%8D%E9%87%8D%E8%A6%81%EF%BC%9F\"><\/span>\u7d50\u8ad6\uff0c\u6240\u4ee5Robots.txt\u5230\u5e95\u91cd\u4e0d\u91cd\u8981\uff1f<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<p>Robots.txt\uff0c\u4ed6\u771f\u7684\u5f88\u91cd\u8981\u55ce\uff1f\u6211\u60f3\uff0c\u7b54\u6848\u4e0d\u662f\u7d55\u5c0d\u7684\uff0c\u4f46\u80af\u5b9a\u7684\u662f\u300e\u5982\u679c\u8a2d\u7f6e\u932f\u4e86\uff0c\u5f71\u97ff\u5c64\u9762\u662f\u975e\u5e38\u6df1\u7684\u300f\u3002\u5728\u4e00\u822c\u6b63\u5e38\u7db2\u7ad9\u904b\u4f5c\u4e0b\uff0c\u6c92\u6709\u984d\u5916\u8a2d\u7f6eRobots.txt\u6a94\uff0c\u9802\u591a\u5f71\u97ff\u6703\u662f\u300e\u4e0d\u592a\u597d\u300f\uff1b\u4f46\u82e5\u6709\u4e0d\u5c11\u6e2c\u8a66\u9801\u9762\u6216\u5c1a\u672a\u5b8c\u6210\u7248\u9762\uff0c\u5fd8\u4e86\u8a2d\u7f6e\u4f7f\u7528Robots.txt\u505a\u6392\u9664\u5167\uff0c\u90a3\u5f71\u97ff\u4e0d\u610f\u5916\u7684\u80af\u5b9a\u300e\u6bd4\u8f03\u5927\u300f\uff1b\u800c\u7e5e\u56de\u6700\u4e00\u958b\u59cb\u8a0e\u8ad6\u7684\u5167\u5bb9\uff0c\u82e5\u4f60\u7684\u7db2\u7ad9\u6e96\u5099\u597d\u4e86\u4e5f\u4e0a\u7dda\u4e86\uff0c\u4f60\u7684\u76ee\u6a19\u662f\u5e0c\u671b\u6709\u597d\u7684\u6392\u540d\u6642\uff0c\u9019\u6642\u82e5\u932f\u8aa4\u4f7f\u7528Robots.txt\u963b\u64cb\u641c\u5c0b\u5f15\u64ce\u4f86\u505a\u7d22\u5f15\uff0c\u90a3\u5c31\u7d55\u5c0d\u662f\u300e\u975e\u5e38\u4e0d\u597d\u300f\u3002<br \/>\n\u5728\u9019\u500bSEO\u4e00\u76f4\u53cd\u8986\u5f37\u8abf\u5167\u5bb9\u70ba\u738b\u7684\u6642\u4ee3\uff0c\u8cc7\u8a0a\u67b6\u69cb\u7684\u5f71\u97ff\u771f\u7684\u53ef\u5927\u53ef\u5c0f\uff0c\u4f46\u4e0d\u53ef\u5426\u8a8d\u7684\u662f\uff0c\u82e5\u67b6\u69cb\u7684\u521d\u671f\u7684\u7b2c\u4e00\u6b65\u5c31\u8d70\u932f\u4e86\uff0c\u90a3\u5f71\u97ff\u7d55\u5c0d\u662f\u5f88\u6df1\u9060\u7684\u3002<\/p>\n<p>\u5982\u6709\u4efb\u4f55\u81ea\u7136\u6d41\u91cf\u589e\u52a0\u7684\u554f\u984c\uff0c\u6216\u60f3\u77ad\u89e3\u95dc\u65bc SEO \u4eba\u5de5\u667a\u6167\u89e3\u6c7a\u65b9\u6848\uff0c<a href=\"https:\/\/www.awoo.ai\/zh-hant\/contact\/\">\u6b61\u8fce\u7acb\u5373\u8aee\u8a62\uff0c\u5c07\u6709 awoo \u5c08\u696d\u9867\u554f\u70ba\u60a8\u670d\u52d9\u3002<\/a><\/p>\n<p><a href=\"https:\/\/www.awoo.ai\/zh-hant\/contact\/\" data-wplink-edit=\"true\"><img decoding=\"async\" class=\"wp-image-40156  alignnone\" src=\"https:\/\/media.awoo.ai\/live\/wp-content\/uploads\/image-1.png\" alt=\"\" width=\"257\" height=\"73\" data-pagespeed-url-hash=\"2088792711\" \/>\u00a0<\/a><\/p>\n<p>\u5ef6\u4f38\u95b1\u8b80\uff1a<a href=\"https:\/\/www.awoo.ai\/zh-hant\/blog\/rwd-and-awd-difference-for-seo\/\">RWD \u662f\u4ec0\u9ebc\uff1f\u8207 AWD \u6709\u4ec0\u9ebc\u4e0d\u540c\uff0c\u97ff\u61c9\u5f0f\u7db2\u7ad9\u8a2d\u8a08\u5c0d SEO \u597d\u8655\u4ecb\u7d39<\/a><\/p>\n<p>\u5ef6\u4f38\u95b1\u8b80\uff1a<a href=\"https:\/\/www.awoo.ai\/zh-hant\/blog\/seo-and-site-search-for-retail\/\">SEO \u8207\u7ad9\u5167\u641c\u5c0b\uff0c\u5982\u4f55\u5275\u96f6\u552e\u96fb\u5546\u7ad9\u5167\u641c\u5c0b\u6700\u5927\u50f9\u503c<\/a><\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u7db2\u7ad9\u4e0a\u7dda\uff0c\u53ea\u80fd\u5168\u7ad9\u88ab\u6536\u9304\u55ce\uff1f \u7121\u8ad6\u662f\u81ea\u5df1\u67b6\u7ad9\uff0c\u4ea6\u6216\u662f\u5354\u52a9\u5ba2\u6236\u7db2\u7ad9\u554f\u984c\u6392\u9664\u6642\uff0c\u662f\u4e0d\u662f\u7e3d\u6703\u9047\u904e\u300e\u70ba\u4ec0\u9ebc\u6211\u7684\u7db2\u7ad9\u4e0a\u7dda [&hellip;]<\/p>\n","protected":false},"author":71,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_acf_changed":false,"footnotes":""},"categories":[29,436],"tags":[382,59,72,131,266],"class_list":["post-8200","post","type-post","status-publish","format-standard","hentry","category-seo-guide","category-seo-advanced","tag-robots-txt","tag-seo-concept","tag-seo-tools","tag-seo","tag-seo-"],"acf":[],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v26.4 - https:\/\/yoast.com\/wordpress\/plugins\/seo\/ -->\n<title>Robots.txt\u7528\u9014\u8207\u4f7f\u7528\u7bc4\u4f8b\u6559\u5b78\uff0c\u91d0\u6e05SEO\u6536\u9304\u89c0\u5ff5\uff01 - awoo<\/title>\n<meta name=\"description\" content=\"\u5584\u7528Robots.txt\u53ef\u4ee5\u5e6b\u52a9\u641c\u5c0b\u5f15\u64ce\u6aa2\u7d22\u7db2\u7ad9\u6642\uff0c\u544a\u8a34\u5b83\u7db2\u7ad9\u54ea\u4e9b\u5167\u5bb9\u53ef\u4ee5\u88ab\u6aa2\u7d22\uff0c\u54ea\u4e9b\u5167\u5bb9\u53ef\u4ee5\u4e0d\u7528\u88ab\u6aa2\u7d22\u3002\u5728\u9019\u5167\u5bb9\u70ba\u738b\u7684\u6642\u4ee3\uff0cRobots.txt\u5373\u662f\u60a8\u7db2\u7ad9\u66dd\u5149\u7684\u7b2c\u4e00\u9053\u8def\u724c\u3002\" \/>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/\" \/>\n<meta property=\"og:locale\" content=\"zh_TW\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"Robots.txt\u7528\u9014\u8207\u4f7f\u7528\u7bc4\u4f8b\u6559\u5b78\uff0c\u91d0\u6e05SEO\u6536\u9304\u89c0\u5ff5\uff01 - awoo\" \/>\n<meta property=\"og:description\" content=\"\u5584\u7528Robots.txt\u53ef\u4ee5\u5e6b\u52a9\u641c\u5c0b\u5f15\u64ce\u6aa2\u7d22\u7db2\u7ad9\u6642\uff0c\u544a\u8a34\u5b83\u7db2\u7ad9\u54ea\u4e9b\u5167\u5bb9\u53ef\u4ee5\u88ab\u6aa2\u7d22\uff0c\u54ea\u4e9b\u5167\u5bb9\u53ef\u4ee5\u4e0d\u7528\u88ab\u6aa2\u7d22\u3002\u5728\u9019\u5167\u5bb9\u70ba\u738b\u7684\u6642\u4ee3\uff0cRobots.txt\u5373\u662f\u60a8\u7db2\u7ad9\u66dd\u5149\u7684\u7b2c\u4e00\u9053\u8def\u724c\u3002\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/\" \/>\n<meta property=\"og:site_name\" content=\"awoo\" \/>\n<meta property=\"article:published_time\" content=\"2018-03-29T08:57:00+00:00\" \/>\n<meta property=\"article:modified_time\" content=\"2023-09-06T05:12:58+00:00\" \/>\n<meta property=\"og:image\" content=\"https:\/\/lh4.googleusercontent.com\/kAnDYhi41Tn9JTLzbDoldUWY-nP9AwYLjv_wRu9Wdli1gkcqX4j2M99Dly3eMY_fOGuC0ryAynHsZB1EmlqhWA5wU950BtqJGr-Ku9SfdCsIwnEaGes611f-xS-qbRtryCnoZY8i\" \/>\n<meta name=\"author\" content=\"Fenix Li\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"\u4f5c\u8005:\" \/>\n\t<meta name=\"twitter:data1\" content=\"Fenix Li\" \/>\n\t<meta name=\"twitter:label2\" content=\"\u9810\u4f30\u95b1\u8b80\u6642\u9593\" \/>\n\t<meta name=\"twitter:data2\" content=\"2 \u5206\u9418\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/\",\"url\":\"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/\",\"name\":\"Robots.txt\u7528\u9014\u8207\u4f7f\u7528\u7bc4\u4f8b\u6559\u5b78\uff0c\u91d0\u6e05SEO\u6536\u9304\u89c0\u5ff5\uff01 - awoo\",\"isPartOf\":{\"@id\":\"https:\/\/www.awoo.ai\/zh-hant\/#website\"},\"primaryImageOfPage\":{\"@id\":\"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/#primaryimage\"},\"image\":{\"@id\":\"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/#primaryimage\"},\"thumbnailUrl\":\"https:\/\/lh4.googleusercontent.com\/kAnDYhi41Tn9JTLzbDoldUWY-nP9AwYLjv_wRu9Wdli1gkcqX4j2M99Dly3eMY_fOGuC0ryAynHsZB1EmlqhWA5wU950BtqJGr-Ku9SfdCsIwnEaGes611f-xS-qbRtryCnoZY8i\",\"datePublished\":\"2018-03-29T08:57:00+00:00\",\"dateModified\":\"2023-09-06T05:12:58+00:00\",\"author\":{\"@id\":\"https:\/\/www.awoo.ai\/zh-hant\/#\/schema\/person\/f81912c39618dbd764a5d473bbe66b73\"},\"description\":\"\u5584\u7528Robots.txt\u53ef\u4ee5\u5e6b\u52a9\u641c\u5c0b\u5f15\u64ce\u6aa2\u7d22\u7db2\u7ad9\u6642\uff0c\u544a\u8a34\u5b83\u7db2\u7ad9\u54ea\u4e9b\u5167\u5bb9\u53ef\u4ee5\u88ab\u6aa2\u7d22\uff0c\u54ea\u4e9b\u5167\u5bb9\u53ef\u4ee5\u4e0d\u7528\u88ab\u6aa2\u7d22\u3002\u5728\u9019\u5167\u5bb9\u70ba\u738b\u7684\u6642\u4ee3\uff0cRobots.txt\u5373\u662f\u60a8\u7db2\u7ad9\u66dd\u5149\u7684\u7b2c\u4e00\u9053\u8def\u724c\u3002\",\"breadcrumb\":{\"@id\":\"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/#breadcrumb\"},\"inLanguage\":\"zh-TW\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/\"]}]},{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-TW\",\"@id\":\"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/#primaryimage\",\"url\":\"https:\/\/lh4.googleusercontent.com\/kAnDYhi41Tn9JTLzbDoldUWY-nP9AwYLjv_wRu9Wdli1gkcqX4j2M99Dly3eMY_fOGuC0ryAynHsZB1EmlqhWA5wU950BtqJGr-Ku9SfdCsIwnEaGes611f-xS-qbRtryCnoZY8i\",\"contentUrl\":\"https:\/\/lh4.googleusercontent.com\/kAnDYhi41Tn9JTLzbDoldUWY-nP9AwYLjv_wRu9Wdli1gkcqX4j2M99Dly3eMY_fOGuC0ryAynHsZB1EmlqhWA5wU950BtqJGr-Ku9SfdCsIwnEaGes611f-xS-qbRtryCnoZY8i\"},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"\u9996\u9801\",\"item\":\"https:\/\/www.awoo.ai\/zh-hant\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"Robots.txt\u7528\u9014\u8207\u4f7f\u7528\u7bc4\u4f8b\u6559\u5b78\uff0c\u91d0\u6e05SEO\u6536\u9304\u89c0\u5ff5\uff01\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/www.awoo.ai\/zh-hant\/#website\",\"url\":\"https:\/\/www.awoo.ai\/zh-hant\/\",\"name\":\"awoo\",\"description\":\"\u53f0\u7063 AI MarTech \u9818\u5c0e\u54c1\u724c\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\/\/www.awoo.ai\/zh-hant\/?s={search_term_string}\"},\"query-input\":{\"@type\":\"PropertyValueSpecification\",\"valueRequired\":true,\"valueName\":\"search_term_string\"}}],\"inLanguage\":\"zh-TW\"},{\"@type\":\"Person\",\"@id\":\"https:\/\/www.awoo.ai\/zh-hant\/#\/schema\/person\/f81912c39618dbd764a5d473bbe66b73\",\"name\":\"Fenix Li\"}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"Robots.txt\u7528\u9014\u8207\u4f7f\u7528\u7bc4\u4f8b\u6559\u5b78\uff0c\u91d0\u6e05SEO\u6536\u9304\u89c0\u5ff5\uff01 - awoo","description":"\u5584\u7528Robots.txt\u53ef\u4ee5\u5e6b\u52a9\u641c\u5c0b\u5f15\u64ce\u6aa2\u7d22\u7db2\u7ad9\u6642\uff0c\u544a\u8a34\u5b83\u7db2\u7ad9\u54ea\u4e9b\u5167\u5bb9\u53ef\u4ee5\u88ab\u6aa2\u7d22\uff0c\u54ea\u4e9b\u5167\u5bb9\u53ef\u4ee5\u4e0d\u7528\u88ab\u6aa2\u7d22\u3002\u5728\u9019\u5167\u5bb9\u70ba\u738b\u7684\u6642\u4ee3\uff0cRobots.txt\u5373\u662f\u60a8\u7db2\u7ad9\u66dd\u5149\u7684\u7b2c\u4e00\u9053\u8def\u724c\u3002","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/","og_locale":"zh_TW","og_type":"article","og_title":"Robots.txt\u7528\u9014\u8207\u4f7f\u7528\u7bc4\u4f8b\u6559\u5b78\uff0c\u91d0\u6e05SEO\u6536\u9304\u89c0\u5ff5\uff01 - awoo","og_description":"\u5584\u7528Robots.txt\u53ef\u4ee5\u5e6b\u52a9\u641c\u5c0b\u5f15\u64ce\u6aa2\u7d22\u7db2\u7ad9\u6642\uff0c\u544a\u8a34\u5b83\u7db2\u7ad9\u54ea\u4e9b\u5167\u5bb9\u53ef\u4ee5\u88ab\u6aa2\u7d22\uff0c\u54ea\u4e9b\u5167\u5bb9\u53ef\u4ee5\u4e0d\u7528\u88ab\u6aa2\u7d22\u3002\u5728\u9019\u5167\u5bb9\u70ba\u738b\u7684\u6642\u4ee3\uff0cRobots.txt\u5373\u662f\u60a8\u7db2\u7ad9\u66dd\u5149\u7684\u7b2c\u4e00\u9053\u8def\u724c\u3002","og_url":"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/","og_site_name":"awoo","article_published_time":"2018-03-29T08:57:00+00:00","article_modified_time":"2023-09-06T05:12:58+00:00","og_image":[{"url":"https:\/\/lh4.googleusercontent.com\/kAnDYhi41Tn9JTLzbDoldUWY-nP9AwYLjv_wRu9Wdli1gkcqX4j2M99Dly3eMY_fOGuC0ryAynHsZB1EmlqhWA5wU950BtqJGr-Ku9SfdCsIwnEaGes611f-xS-qbRtryCnoZY8i","type":"","width":"","height":""}],"author":"Fenix Li","twitter_card":"summary_large_image","twitter_misc":{"\u4f5c\u8005:":"Fenix Li","\u9810\u4f30\u95b1\u8b80\u6642\u9593":"2 \u5206\u9418"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/","url":"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/","name":"Robots.txt\u7528\u9014\u8207\u4f7f\u7528\u7bc4\u4f8b\u6559\u5b78\uff0c\u91d0\u6e05SEO\u6536\u9304\u89c0\u5ff5\uff01 - awoo","isPartOf":{"@id":"https:\/\/www.awoo.ai\/zh-hant\/#website"},"primaryImageOfPage":{"@id":"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/#primaryimage"},"image":{"@id":"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/#primaryimage"},"thumbnailUrl":"https:\/\/lh4.googleusercontent.com\/kAnDYhi41Tn9JTLzbDoldUWY-nP9AwYLjv_wRu9Wdli1gkcqX4j2M99Dly3eMY_fOGuC0ryAynHsZB1EmlqhWA5wU950BtqJGr-Ku9SfdCsIwnEaGes611f-xS-qbRtryCnoZY8i","datePublished":"2018-03-29T08:57:00+00:00","dateModified":"2023-09-06T05:12:58+00:00","author":{"@id":"https:\/\/www.awoo.ai\/zh-hant\/#\/schema\/person\/f81912c39618dbd764a5d473bbe66b73"},"description":"\u5584\u7528Robots.txt\u53ef\u4ee5\u5e6b\u52a9\u641c\u5c0b\u5f15\u64ce\u6aa2\u7d22\u7db2\u7ad9\u6642\uff0c\u544a\u8a34\u5b83\u7db2\u7ad9\u54ea\u4e9b\u5167\u5bb9\u53ef\u4ee5\u88ab\u6aa2\u7d22\uff0c\u54ea\u4e9b\u5167\u5bb9\u53ef\u4ee5\u4e0d\u7528\u88ab\u6aa2\u7d22\u3002\u5728\u9019\u5167\u5bb9\u70ba\u738b\u7684\u6642\u4ee3\uff0cRobots.txt\u5373\u662f\u60a8\u7db2\u7ad9\u66dd\u5149\u7684\u7b2c\u4e00\u9053\u8def\u724c\u3002","breadcrumb":{"@id":"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/#breadcrumb"},"inLanguage":"zh-TW","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/"]}]},{"@type":"ImageObject","inLanguage":"zh-TW","@id":"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/#primaryimage","url":"https:\/\/lh4.googleusercontent.com\/kAnDYhi41Tn9JTLzbDoldUWY-nP9AwYLjv_wRu9Wdli1gkcqX4j2M99Dly3eMY_fOGuC0ryAynHsZB1EmlqhWA5wU950BtqJGr-Ku9SfdCsIwnEaGes611f-xS-qbRtryCnoZY8i","contentUrl":"https:\/\/lh4.googleusercontent.com\/kAnDYhi41Tn9JTLzbDoldUWY-nP9AwYLjv_wRu9Wdli1gkcqX4j2M99Dly3eMY_fOGuC0ryAynHsZB1EmlqhWA5wU950BtqJGr-Ku9SfdCsIwnEaGes611f-xS-qbRtryCnoZY8i"},{"@type":"BreadcrumbList","@id":"https:\/\/www.awoo.ai\/zh-hant\/blog\/robotstxt-crawl\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"\u9996\u9801","item":"https:\/\/www.awoo.ai\/zh-hant\/"},{"@type":"ListItem","position":2,"name":"Robots.txt\u7528\u9014\u8207\u4f7f\u7528\u7bc4\u4f8b\u6559\u5b78\uff0c\u91d0\u6e05SEO\u6536\u9304\u89c0\u5ff5\uff01"}]},{"@type":"WebSite","@id":"https:\/\/www.awoo.ai\/zh-hant\/#website","url":"https:\/\/www.awoo.ai\/zh-hant\/","name":"awoo","description":"\u53f0\u7063 AI MarTech \u9818\u5c0e\u54c1\u724c","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/www.awoo.ai\/zh-hant\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"zh-TW"},{"@type":"Person","@id":"https:\/\/www.awoo.ai\/zh-hant\/#\/schema\/person\/f81912c39618dbd764a5d473bbe66b73","name":"Fenix Li"}]}},"_links":{"self":[{"href":"https:\/\/www.awoo.ai\/zh-hant\/wp-json\/wp\/v2\/posts\/8200","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.awoo.ai\/zh-hant\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.awoo.ai\/zh-hant\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.awoo.ai\/zh-hant\/wp-json\/wp\/v2\/users\/71"}],"replies":[{"embeddable":true,"href":"https:\/\/www.awoo.ai\/zh-hant\/wp-json\/wp\/v2\/comments?post=8200"}],"version-history":[{"count":4,"href":"https:\/\/www.awoo.ai\/zh-hant\/wp-json\/wp\/v2\/posts\/8200\/revisions"}],"predecessor-version":[{"id":40800,"href":"https:\/\/www.awoo.ai\/zh-hant\/wp-json\/wp\/v2\/posts\/8200\/revisions\/40800"}],"wp:attachment":[{"href":"https:\/\/www.awoo.ai\/zh-hant\/wp-json\/wp\/v2\/media?parent=8200"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.awoo.ai\/zh-hant\/wp-json\/wp\/v2\/categories?post=8200"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.awoo.ai\/zh-hant\/wp-json\/wp\/v2\/tags?post=8200"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}