Управление индексацией сайта при помощи файла robots.txt
( , , ..) -, , . - : -, robots.txt. .
robots.txt - (, - txt, ), (, [url]http://filosofia.ru/[/url], robots.txt : [url]http://filosofia.ru/robots.txt)[/url]. : ! , - , , robots.txt ( ). , , . robots.txt. , ( :-)).
robots.txt
robots.txt : . , , - ( ), . ( User-agent, robots.txt ):
User-agent: robot1 robot2 robot3, :
robot1, robot2 robot3 - . :
User-agent: WebCrawler Lycos StackRambler
, ( : Rambler "" StackRambler). , , . :
User-agent: *, "*" , ( ).
:
Disallow: , :
- URI, . ( URI, ). , Dissallow.
: , .
:
Disallow: /topsecret/ - , , .. topsecret.html , /topsecret/ - .
Disallow: /privat - , .
Disallow: /folder/file.htm - .
, , , Disallow. Disallow , URL . robots.txt , , Disallow. Disallow User-agent, User-agent Disallow.
robots.txt . .
1.
# robots.txt file for http://dengi.filosofia.ru/
# contact [email protected] for critics
User-agent: StackRambler Googlebot
Disallow:
User-agent: *
Disallow: /tmp/
Disallow: /logs/
# . .
tmp logs, StackRambler Googlebot .
2.
# I hate search engines!
User-agent: *
Disallow: /
.
3.
User-agent: *
Disallow: /private/letters/
# This is top secret directory.
Disallow: /cgi-bin/
Disallow: /anektods.html
, /private/letters/ /cgi-bin/, anektods.html.
4.
User-agent: *
Disallow:
. .
robots.txt . . - , , . " " . " ".
? - . , ? - , - "robots". .
Все статьи
|