# robots.txt file for all search engine

User-agent: *
Disallow: /test/
Disallow: /stats/
Disallow: /images/
Disallow: /help/
Disallow: /d.php
Disallow: /ls.php
Disallow: /cs.php
Disallow: /dredir.php
Disallow: /load.php
Disallow: /load1.php
Disallow: /load2.php
Disallow: /load3.php
Disallow: /load4.php


User-Agent: BaiDuSpider
Disallow: /

User-agent: dotbot
Disallow: / 

User-agent: CherryPicker
Disallow: / 

User-agent: CherryPicker/1.0
Disallow: / 

User-agent: CherryPickerSE/1.0 
Disallow: / 

User-agent: CherryPickerElite/1.0 
Disallow: / 

User-agent: Crescent_Internet_ToolPak
Disallow: /

User-agent: Crescent_Internet_ToolPak_HTTP_OLE_Control_v.1.0
Disallow: /

User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0
Disallow: /

User-Agent: DISCo Pump
Disallow: /

User-agent: EmailCollector/1.0 
Disallow: / 

User-agent: EmailSiphon
Disallow: /

User-agent: EmailWolf 1.00 
Disallow: / 

User-agent: ExtractorPro
Disallow: /

User-Agent: grub
Disallow: /

User-Agent: htdig
Disallow: /

User-Agent: HTTrack
Disallow: /

User-Agent: ia_archiver
Disallow: /

User-Agent: LNSpiderGuy
Disallow: /

User-Agent: lwp-trivial
Disallow: /

User-agent: Mozilla/2.0 (compatible; NEWT ActiveX; Win32) 
Disallow: /

User-agent: NICErsPRO
Disallow: /

User-Agent: Netprospector
Disallow: /

User-Agent: PerMan Surfer
Disallow: /

User-Agent: psbot
Disallow: /

User-agent: Telesoft/1.29
Disallow: /

User-agent: WebBandit/2.1
Disallow: /

User-agent: WebBandit/3.50
Disallow: /

User-agent: webbandit/4.00.0
Disallow: /

User-agent: webbandit
Disallow: /

User-agent: WebEMailExtractor/1.0B
Disallow: /

User-agent: Wget/1.8.2
Disallow: /