aniroleplay.com/robots.txt:
User-agent: Mediapartners-Google
Disallow:
User-agent: Googlebot-Image
Disallow: /
User-Agent: Genieo
Disallow: /
# http://riskyinternet.com/what-is/web-robot/jikespider-6996/
User-agent: JikeSpider
Disallow: /
# http://riskyinternet.com/what-is/web-robot/yandex-media-6835/
User-agent: Yandex
Disallow: /
# http://riskyinternet.com/what-is/web-robot/yandex-std-crawler-6821/
User-agent: YandexBot
Disallow: /
# http://riskyinternet.com/what-is/web-robot/yandex-images-6828/
User-agent: YandexImages
Disallow: /
# http://riskyinternet.com/what-is/web-robot/baiduspider-5925/
User-agent: Baiduspider Baiduspider-video Baiduspider-image Baiduspider-video Baiduspider-news Baiduspider-favo Baiduspider-cpro Baiduspider-ads
Disallow: /
# http://riskyinternet.com/what-is/web-robot/naverbot-crawler-6443/
User-agent: NaverBot
Disallow: /
# http://help.naver.com/robots
User-agent: Yeti
Disallow: /
# http://riskyinternet.com/what-is/web-robot/spinn3rspinner-aka-tailrank-inc-7073/
# http://spinn3r.com/robot
User-agent: Spinn3r
Disallow: /
# http://riskyinternet.com/what-is/web-robot/new-sogou-spider-6562/
User-agent: sogou
Disallow: /
User-agent: sogou spider
Disallow: /
# http://riskyinternet.com/what-is/web-robot/internet-archiver-lift-all-internet-content-over-and-over-3517/
# http://archive.org/abot/exclude.php
# http://www.alexa.com/help/webmasters
User-agent: ia_archiver
Disallow:
# http://riskyinternet.com/what-is/web-robot/ahrefscom-bot-6002/
# https://ahrefs.com/robot/
User-agent: AhrefsBot
Disallow: /
# http://riskyinternet.com/what-is/web-robot/mj12-bot-4994/
# http://www.majestic12.co.uk/projects/dsearch/mj12bot.php
User-agent: MJ12bot
Disallow: /
# http://riskyinternet.com/what-is/web-robot/discoveryengine-robot-6142/
# http://discoveryengine.com/discobot.html
User-agent: discobot
Disallow: /
# http://uptimerobot.com
User-agent: Uptimebot
Disallow: /
# http://fulltext.sblog.cz/category/robot
User-agent: seznambot
Disallow: /
# http://riskyinternet.com/what-is/web-robot/dotnetdotcom-3573/
User-agent: Ezooms
Disallow: /
# http://www.acoon.de/robot.asp
User-agent: Acoon
Disallow: /
# http://riskyinternet.com/what-is/web-robot/mlbot-3559/
# http://www.metadatalabs.com/mlbot
User-agent: MLBot
Disallow: /
User-agent: Sosospider
Disallow: /
User-agent: *
Crawl-delay: 10
Disallow: /add_bookmark.php
Disallow: /add_comment.php
Disallow: /add_group.php
Disallow: /add_testimonial.php
Disallow: /advertise.php
Disallow: /albums.php
Disallow: /approve_friend.php
Disallow: /approve_test.php
Disallow: /blocked.php
Disallow: /block_user.php
Disallow: /blognotifications.php
Disallow: /bulletinnotifications.php
Disallow: /commentnotifications.php
Disallow: /delete_test.php
Disallow: /edit_comments.php
Disallow: /flag.php
Disallow: /forgot_password.php
Disallow: /forward_friend.php
Disallow: /friendnotifications.php
Disallow: /gallery.php
Disallow: /gallery/
Disallow: /groupnotifications.php
Disallow: /group_images/
Disallow: /home.php
Disallow: /images/
Disallow: /invite_friend.php
Disallow: /join_me.php
Disallow: /login.php
Disallow: /m/
Disallow: /m/images/
Disallow: /m/old_dirs/
Disallow: /m/old_files/
Disallow: /m/photos/
Disallow: /m/rss_blog.php
Disallow: /m/songs/
Disallow: /m/xspf/
Disallow: /make_default.php
Disallow: /mobile.php
Disallow: /new_friends.php
Disallow: /notice.php
Disallow: /old_dirs/
Disallow: /old_files/
Disallow: /online_friends.php
Disallow: /photos/
Disallow: /picturenotifications.php
Disallow: /profile_setting.php
Disallow: /rank_band.php
Disallow: /reply.php
Disallow: /rss_blog.php
Disallow: /search_profiles.php
Disallow: /send_message.php
Disallow: /send_message1.php
Disallow: /shared_layouts.php
Disallow: /songs/
Disallow: /statusnotifications.php
Disallow: /status_stream.php
Disallow: /subscribe_blog.php
Disallow: /user_images/
Disallow: /view_blogs_category.php
Disallow: /view_blog_layout.php
Disallow: /view_comments.php
Disallow: /view_friends.php
Disallow: /view_group.php
Disallow: /view_layout.php
Disallow: /view_message.php
Disallow: /view_messages.php
Disallow: /view_photo.php
Disallow: /view_post.php
Disallow: /view_testimonials.php
Disallow: /xspf/
в яндексе
Прочее:
Ping/Trace,
История в Веб-архиве,
История владельцев,
Валидность HTML,
Валидность CSS,
Проверка на вирусы.
Запрашиваемая страница недоступна!
|