User-agent: *
# Directories
Disallow: /demo/database/
Disallow: /demo/includes/
Disallow: /demo/misc/
Disallow: /demo/modules/
Disallow: /demo/sites/
Disallow: /demo/themes/
Disallow: /demo/scripts/
Disallow: /demo/ui/
Disallow: /demo/profiles/
#
# Files
Disallow: /demo/xmlrpc.php
Disallow: /demo/cron.php
Disallow: /demo/update.php
Disallow: /demo/install.php
Disallow: /demo/INSTALL.mysql.txt
Disallow: /demo/INSTALL.pgsql.txt
Disallow: /demo/CHANGELOG.txt
Disallow: /demo/MAINTAINERS.txt
Disallow: /demo/LICENSE.txt
Disallow: /demo/UPGRADE.txt
#
# Paths (clean URLs) -- modified from default!
Disallow: /demo/admin/
Disallow: /demo/aggregator/
Disallow: /demo/comment/
Disallow: /demo/contact
Disallow: /demo/logout
Disallow: /demo/search/
#
# block tracker pages
Disallow: /demo/tracker/
# block paginated tracker pages
Disallow: /demo/tracker?
#
# IMPORTANT: THE FOLLOWING LINE BLOCKS ALL /node* URLs -- only use it if you do not have content with URLs like http://example.com/node/10. Alternatively use the Global Redirect module.
Disallow: /demo/node
Disallow: /demo/node/
#
# If you DON'T want your user's pages crawled, use this, otherwise delete it:
Disallow: /demo/user
# Disallow robots from all but the main feed - requires Pathauto feed aliases turned on. Optionally only block Google from the following paths because it causes the most havoc with Google
Disallow: /demo/*/feed
# This prevents duplicate content caused by some modules like the Forums and Views modules
Disallow: /demo/*sort=
# Prevent duplicate content created by the Image module
Disallow: /demo/*size=
# If you have the Front module this will keep a duplicate front page from being indexed
Disallow: /demo/front_page
# WHN specific (Duplicate content in subsites)
Disallow: /demo/amsterdam/services
Disallow: /demo/amsterdam/bank
Disallow: /demo/amsterdam/financial
Disallow: /demo/amsterdam/taxonomy/term/91
Disallow: /demo/amsterdam/taxonomy/term/94
Disallow: /demo/amsterdam/gettingaround
Disallow: /demo/amsterdam/air
Disallow: /demo/amsterdam/taxituktuk
Disallow: /demo/amsterdam/image
Disallow: /demo/amsterdam/advice
Disallow: /demo/amsterdam/tools
Disallow: /demo/amsterdam/stories
Disallow: /demo/amsterdam/narrative
Disallow: /demo/amsterdam/whoonerstory
Disallow: /demo/amsterdam/buddylist
Disallow: /demo/amsterdam/news

Disallow: /demo/lasvegas/services
Disallow: /demo/lasvegas/bank
Disallow: /demo/lasvegas/financial
Disallow: /demo/lasvegas/taxonomy/term/91
Disallow: /demo/lasvegas/taxonomy/term/94
Disallow: /demo/lasvegas/gettingaround
Disallow: /demo/lasvegas/air
Disallow: /demo/lasvegasm/taxituktuk
Disallow: /demo/lasvegas/image
Disallow: /demo/lasvegas/advice
Disallow: /demo/lasvegas/tools
Disallow: /demo/lasvegas/stories
Disallow: /demo/lasvegas/narrative
Disallow: /demo/lasvegas/whoonerstory
Disallow: /demo/lasvegas/buddylist
Disallow: /demo/amsterdam/news

Disallow: /demo/losangeles/services
Disallow: /demo/losangeles/bank
Disallow: /demo/losangeles/financial
Disallow: /demo/losangeles/taxonomy/term/91
Disallow: /demo/losangeles/taxonomy/term/94
Disallow: /demo/losangeles/gettingaround
Disallow: /demo/losangeles/air
Disallow: /demo/losangeles/taxituktuk
Disallow: /demo/losangeles/image
Disallow: /demo/losangeles/advice
Disallow: /demo/losangeles/tools
Disallow: /demo/losangeles/stories
Disallow: /demo/losangeles/narrative
Disallow: /demo/losangeles/whoonerstory
Disallow: /demo/losangeles/buddylist
Disallow: /demo/losangeles/news

Disallow: /demo/bangkok/services
Disallow: /demo/bangkok/bank
Disallow: /demo/bangkok/financial
Disallow: /demo/bangkok/taxonomy/term/91
Disallow: /demo/bangkok/taxonomy/term/94
Disallow: /demo/bangkok/gettingaround
Disallow: /demo/bangkok/air
Disallow: /demo/bangkok/taxituktuk
Disallow: /demo/bangkok/image
Disallow: /demo/bangkok/advice
Disallow: /demo/bangkok/tools
Disallow: /demo/bangkok/stories
Disallow: /demo/bangkok/narrative
Disallow: /demo/bangkok/whoonerstory
Disallow: /demo/bangkok/buddylist
Disallow: /demo/bangkok/news

Disallow: /demo/havana/services
Disallow: /demo/havana/bank
Disallow: /demo/havana/financial
Disallow: /demo/havana/taxonomy/term/91
Disallow: /demo/havana/taxonomy/term/94
Disallow: /demo/havana/gettingaround
Disallow: /demo/havana/air
Disallow: /demo/havana/taxituktuk
Disallow: /demo/havana/image
Disallow: /demo/havana/advice
Disallow: /demo/havana/tools
Disallow: /demo/havana/stories
Disallow: /demo/havana/narrative
Disallow: /demo/havana/whoonerstory
Disallow: /demo/havana/buddylist
Disallow: /demo/havana/news

#other directories 
Disallow: /beta/
Disallow: /panama/
Disallow: /whn/