User-agent: *
Crawl-delay: 10
Sitemap: http://helda.helsinki.fi/htmlmap
# Uncomment the following line ONLY if sitemaps.org or HTML sitemaps are used
# and you have verified that your site is being indexed correctly.
Disallow: /dspace
Disallow: /jspui
Disallow: /browse
Disallow: /*/browse
Disallow: /*/browse?
Disallow: /browse-subject
Disallow: /browse-author
Disallow: /browse-title
Disallow: /browse-date
Disallow: /suggest
Disallow: /*/browse-subject
Disallow: /*/browse-author
Disallow: /*/browse-title
Disallow: /*/browse-date
Disallow: /*?locale-attribute=fi
Disallow: /*?locale-attribute=sv
Disallow: /*?locale-attribute=en
Disallow: /*/license.txt?
Disallow: /*/license.txt
Disallow: /*.txt?
Disallow: /feed
# You also may wish to disallow access to the following paths, in order
# to stop web spiders from accessing user-based content:
Disallow: /advanced-search
Disallow: /contact
Disallow: /feedback
Disallow: /forgot
Disallow: /login
Disallow: /register
Disallow: /search