User-agent: *
Crawl-delay: 1
Disallow: /cgi-bin/
Disallow: /i/
Disallow: /goto/
Disallow: /add.php
Disallow: /add
Disallow: /login
Disallow: /esearch.php
Disallow: /search.php
Disallow: /contxt/
Disallow: /keywords/
Disallow: /?m=
Disallow: /pmeta.php





User-agent: DotBot
Disallow: /archive/
Disallow: /catalog/
Disallow: /cgi-bin/
Disallow: /goto/
Disallow: /login



User-agent: Mail.Ru
Disallow: /archive/
Disallow: /catalog/
Disallow: /cgi-bin/
Disallow: /goto/
Disallow: /login



User-agent: AhrefsBot
Disallow: /

User-agent: NTENTbot
Disallow: /catalog/
Disallow: /archive/
Disallow: /search.php
Disallow: /cgi-bin/
Disallow: /i/
Disallow: /goto/
Disallow: /add
Disallow: /login

User-agent: NTENTbot-fetch
Disallow: /catalog/
Disallow: /archive/
Disallow: /search.php
Disallow: /cgi-bin/
Disallow: /i/
Disallow: /goto/
Disallow: /add
Disallow: /login


User-agent: ConveraCrawler
Disallow: /catalog/
Disallow: /archive/
Disallow: /search.php
Disallow: /cgi-bin/
Disallow: /i/
Disallow: /goto/
Disallow: /add
Disallow: /login



User-agent: WebBot-FindUrl
Disallow: /



User-agent: ia_archiver
Disallow: /catalog/
Disallow: /archive/
Disallow: /search.php
Disallow: /cgi-bin/
Disallow: /i/
Disallow: /goto/
Disallow: /add.php
Disallow: /add
Disallow: /login
Disallow: /esearch.php
Disallow: /contxt/
Disallow: /keywords/
Disallow: /?m=
Disallow: /pmeta.php


User-agent: MJ12bot
Disallow: /catalog/
Disallow: /archive/
Disallow: /goto/
Disallow: /search.php
Disallow: /cgi-bin/
Disallow: /i/
Disallow: /goto/
Disallow: /add.php
Disallow: /add
Disallow: /login
Disallow: /esearch.php
Disallow: /contxt/
Disallow: /keywords/
Disallow: /?m=
Disallow: /pmeta.php


User-agent: Webbot
Disallow: /



User-agent: SemrushBot
Disallow: /


User-agent: TurnitinBot
Disallow: /


User-agent: Sogou
Disallow: /


User-agent: CCBot
Disallow: /


User-agent: grapeshot
Disallow: /archive/
Disallow: /search/
Disallow: /tags/
Disallow: /search.php


User-agent: proximic
Disallow: /archive/
Disallow: /search/
Disallow: /tags/
Disallow: /search.php