# robots.txt

Sitemap: http://www.modernhistoryproject.org/mhp/Sitemap.php

User-agent: MJ12bot
Disallow: /

User-agent: Twiceler
Disallow: /

User-agent: Boitho
Disallow: /

User-agent: Charlotte
Disallow: /

User-agent: *
Crawl-delay: 15
Disallow: /mhp/NewsMgmt.php
Disallow: /mhp/ArticleMgmt.php
Disallow: /mhp/EntityMgmt.php
Disallow: /mhp/MessageMgmt.php
Disallow: /mhp/LinksMgmt.php
Disallow: /mhp/UserLogin.php
Disallow: /mhp/Admin/
Disallow: /mhp/Articles/
Disallow: /mhp/Functions/
Disallow: /mhp/Images/
Disallow: /mhp/Logs/
Disallow: /mhp/Offsite/
Disallow: /mhp-06/
Disallow: /mhp-07/