Sitemap: https://www.dgt.es/sitemap.xml

########################################################
#                          ___                         #
#     _____               /\__\                        #
#    /::\  \             /:/ _/_               ___     #
#   /:/\:\  \           /:/ /\  \             /\__\    #
#  /:/  \:\__\         /:/ /::\  \           /:/  /    #
# /:/__/ \:|__|       /:/__\/\:\__\         /:/__/     #
# \:\  \ /:/  /       \:\  \ /:/  /        /::\  \     #
#  \:\  /:/  /         \:\  /:/  /        /:/\:\  \    #
#   \:\/:/  /           \:\/:/  /         \/__\:\  \   #
#    \::/  /             \::/  /               \:\__\  #
#     \/__/               \/__/                 \/__/  #
#                                                      #
#    ____       _           _        _        _        #
#   |  _ \ ___ | |__   ___ | |_ ___ | |___  _| |_      # 
#   | |_) / _ \| '_ \ / _ \| __/ __|| __\ \/ / __|     #
#   |  _ < (_) | |_) | (_) | |_\__ \| |_ >  <| |_      #
#   |_| \_\___/|_.__/ \___/ \__|___(_)__/_/\_\\__|     #
########################################################
User-Agent: *
User-agent: Googlebot
User-agent: Googlebot-News
User-agent: Googlebot-Image
User-agent: Googlebot-Video

## Evitar rastreo de Bad Bots
User-agent: 008 Disallow: /
User-agent: AddThis.com Disallow: /
User-agent: admantx Disallow: /
User-agent: AhrefsBot Disallow: / 
User-agent: Alexibot Disallow: /
User-agent: Aqua_Products Disallow: /
User-agent: BDCbot Disallow: /
User-agent: Bender Disallow: /
User-agent: BIXOCRAWLER Disallow: / 
User-agent: bl.uk_lddc_bot Disallow: /
User-agent: BLEXBot Disallow: /
User-agent: BUbiNG Disallow: / 
User-agent: Cliqzbot Disallow: / 
User-agent: CNCDialer Disallow: / 
User-agent: crawler4j Disallow: / 
User-agent: CrystalSemanticsBot Disallow: /
User-Agent: CyberAlert Disallow: /
User-agent: DigExt Disallow: / 
User-agent: discobot Disallow: /
User-agent: discoverybot Disallow: /
User-agent: dloader Disallow: / 
User-agent: dloader(NaverRobot) Disallow: / 
User-agent: DOC Disallow: / 
User-agent: dotbot Disallow: / 
User-agent: Download Ninja Disallow: /  
User-agent: DTS Agent Disallow: / 
User-agent: Exabot Disallow: / 
User-agent: Ezooms Disallow: /
User-Agent: FairShare Disallow: / 
User-agent: Fetch Disallow: / 
User-agent: Flamingo_SearchEngine Disallow: / 
User-agent: Genieo Disallow: / 
User-agent: Gigabot Disallow: /
User-agent: grub-client Disallow: / 
User-agent: Heritrix Disallow: / 
User-agent: heritrix/3.3.0 Disallow: / 
User-agent: HTTrack Disallow: / 
User-agent: ia_archiver Disallow: / 
User-agent: integromedb Disallow: /
User-agent: IstellaBot Disallow: /
User-agent: JikeSpider Disallow: /
User-agent: Jyxobot Disallow: /
User-agent: k2spider Disallow: / 
User-agent: Kimengi Disallow: / 
User-agent: Kimengi/nineconnections.com Disallow: /
User-agent: larbin Disallow: / 
User-agent: LexxeBot/1.0 Disallow: /
User-agent: libwww Disallow: / 
User-agent: linko Disallow: / 
User-agent: Livelapbot Disallow: / 
User-agent: magpie-crawler Disallow: / 
User-agent: Maxthon Disallow: / 
User-agent: MetaURI Disallow: / 
User-agent: Microsoft.URL.Control Disallow: / 
User-agent: MJ12bot Disallow: / 
User-agent: Moreover Disallow: / 
User-agent: Moreoverbot Disallow: /
User-agent: MSIECrawler Disallow: / 
User-agent: nabot Disallow: / 
User-agent: NaverBot Disallow: / 
User-agent: NerdByNature.Bot Disallow: /
User-agent: netEstate NE Crawler Disallow: /
User-agent: NetSeer crawler Disallow: /
User-agent: Newscan Disallow: /
User-agent: NextGenSearchBot Disallow: /
User-agent: NPBot Disallow: / 
User-agent: nutch Disallow: / 
User-agent: Offline Explorer Disallow: / 
User-agent: omgilibot Disallow: / 
User-agent: Orthogaffe Disallow: / 
User-agent: PiplBot Disallow: / 
User-agent: Pixray-Seeker Disallow: /
User-agent: proximic Disallow: / 
User-agent: psbot Disallow: /
User-agent: QuerySeekerSpider Disallow: / 
User-agent: rogerbot Disallow: / 
User-agent: seokicks Disallow: /
User-agent: SEOkicks-Robot Disallow: / 
User-agent: SiteBot Disallow: /
User-agent: SiteBot/0.1 Disallow: /
User-agent: sitecheck.internetseer.com Disallow: / 
User-agent: SiteSnagger Disallow: / 
User-agent: Slurp Disallow: / 
User-agent: sogou Disallow: /
User-agent: Sosospider Disallow: /
User-agent: spbot Disallow: / 
User-agent: Spinn3r Disallow: /
User-agent: Teleport Disallow: / 
User-agent: TeleportPro Disallow: / 
User-Agent: trendictionbot Disallow: / 
User-agent: trovitBot Disallow: /
User-agent: TurnitinBot Disallow: /
User-agent: UbiCrawler Disallow: / 
User-agent: uMBot-LN Disallow: / 
User-agent: UnisterBot Disallow: /
User-agent: UniversalFeedParser Disallow: / 
User-agent: WBSearchBot Disallow: /
User-agent: WebCopier Disallow: / 
User-agent: WebReaper Disallow: /
User-agent: WebStripper Disallow: / 
User-agent: WebZIP Disallow: / 
User-agent: WeSEE:Search Disallow: /
User-agent: wget Disallow: /
User-agent: Wotbot Disallow: / 
User-agent: wotbox Disallow: /
User-agent: Xenu Disallow: / 
User-agent: Xenu's Link Sleuth 1.1c Disallow: /
User-agent: Xenu's Disallow: /
User-agent: Yasni Disallow: /
User-agent: Yahoo! Slurp China Disallow: /
User-agent: Zao Disallow: / 
User-agent: Zealbot Disallow: / 
User-agent: ZyBORG Disallow: / 
User-agent: Zeus Disallow: /
User-agent: Zeus Link Scout Disallow: /
User-agent: Zeus 32297 Webster Pro V2.9 Win32 Disallow: /

User-agent: b2w/0.1 Disallow: /
User-agent: BackDoorBot/1.0 Disallow: /
User-agent: Bookmark search tool Disallow: /
User-agent: BotALot Disallow: /
User-agent: BotRightHere Disallow: /
User-agent: BuiltBotTough Disallow: /
User-agent: Bullseye/1.0 Disallow: /
User-agent: BunnySlippers Disallow: /
User-agent: CherryPicker Disallow: /
User-agent: CherryPickerSE/1.0 Disallow: /
User-agent: CherryPickerElite/1.0 Disallow: /
User-agent: CheeseBot Disallow: /
User-agent: Copernic Disallow: /
User-agent: cosmos Disallow: /
User-agent: Crescent Disallow: /
User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0 Disallow: /
User-agent: DittoSpyder Disallow: /
User-agent: EmailCollector Disallow: /
User-agent: EmailSiphon Disallow: /
User-agent: EmailWolf Disallow: /
User-agent: EroCrawler Disallow: /
User-agent: ExtractorPro Disallow: /
User-agent: FairAd Client Disallow: /
User-agent: fast Disallow: /
User-agent: Flaming AttackBot Disallow: /
User-agent: Foobot Disallow: /
User-agent: Gaisbot Disallow: /
User-agent: GetRight/4.2 Disallow: /
User-agent: Harvest/1.5 Disallow: /
User-agent: hloader Disallow: /
User-agent: httplib Disallow: /
User-agent: HTTrack 3.0 Disallow: /
User-agent: humanlinks Disallow: /
User-agent: InfoNaviRobot Disallow: /
User-agent: Iron33/1.0.2 Disallow: /
User-agent: JennyBot Disallow: /
User-agent: Jyxobot/1 Disallow: /
User-agent: Kenjin Spider Disallow: /
User-agent: Keyword Density/0.9 Disallow: /
User-agent: LexiBot Disallow: /
User-agent: libWeb/clsHTTP Disallow: /
User-agent: LinkextractorPro Disallow: /
User-agent: LinkScan/8.1a Unix Disallow: /
User-agent: LinkWalker Disallow: /
User-agent: LNSpiderguy Disallow: /
User-agent: lwp-trivial/1.34 Disallow: /
User-agent: lwp-trivial Disallow: /
User-agent: Mata Hari Disallow: /
User-agent: MIIxpc Disallow: /
User-agent: MIIxpc/4.2 Disallow: /
User-agent: Mister PiX Disallow: /
User-agent: moget Disallow: /
User-agent: moget/2.1 Disallow: /
User-agent: NetAnts Disallow: /
User-agent: NICErsPRO Disallow: /
User-agent: Openbot Disallow: /
User-agent: Openfind data gatherer Disallow: /
User-agent: Openfind Disallow: /
User-agent: Oracle Ultra Search Disallow: /
User-agent: PerMan Disallow: /
User-agent: ProWebWalker Disallow: /
User-agent: ProPowerBot/2.14 Disallow: /
User-agent: Python-urllib Disallow: /
User-agent: QueryN Metasearch Disallow: /
User-agent: Radiation Retriever 1.1 Disallow: /
User-agent: RepoMonkey Bait & Tackle/v1.01 Disallow: /
User-agent: RepoMonkey Disallow: /
User-agent: RMA Disallow: /
User-agent: searchpreview Disallow: /
User-agent: SpankBot Disallow: /
User-agent: spanner Disallow: /
User-agent: suzuran Disallow: /
User-agent: Szukacz/1.4 Disallow: /
User-agent: Telesoft Disallow: /
User-agent: The Intraformant Disallow: /
User-agent: TightTwatBot Disallow: /
User-agent: TheNomad Disallow: /
User-agent: toCrawl/UrlDispatcher Disallow: /
User-agent: True_Robot/1.0 Disallow: /
User-agent: True_Robot Disallow: /
User-agent: turingos Disallow: /
User-agent: TurnitinBot/1.5 Disallow: /
User-agent: URL Control Disallow: /
User-agent: URL_Spider_Pro Disallow: /
User-agent: URLy Warning Disallow: /
User-agent: VCI WebViewer VCI WebViewer Win32 Disallow: /
User-agent: VCI Disallow: /
User-agent: WebBandit Disallow: /
User-agent: WebBandit/3.50 Disallow: /
User-agent: WebCapture 2.0 Disallow: /
User-agent: WebCopier v3.2a Disallow: /
User-agent: WebCopier v.2.2 Disallow: / 
User-agent: Web Image Collector Disallow: /
User-agent: WebEnhancer Disallow: /
User-agent: WebSauger Disallow: /
User-agent: Website Quester Disallow: /
User-agent: Webster Pro Disallow: /
User-agent: WebZip/4.0 Disallow: /
User-agent: WebZIP/4.21 Disallow: /
User-agent: WebZIP/5.0 Disallow: /
User-agent: Wget/1.6 Disallow: /
User-agent: Wget/1.5.3 Disallow: /
User-agent: Wget Disallow: /
User-agent: WWW-Collector-E Disallow: /


##conf especifica bots
User-agent: archive.org_bot
Visit-time: 0100-0700
User-agent: msnbot
Crawl-delay: 20
User-agent: Slurp
Crawl-delay: 10