# To remove the staging sites from all user-agent from and prevent them crawling the staging sites, with the exceptions of the user-agents of ctc google search appliance and Akamai. User-agent: * Disallow: # Allow the CTC's google search appliance to access the sites User-agent: gsa-crawler # Any empty value, indicates that all URLs can be retrieved. Disallow: # Allow Akamai crawler to access the sites User-agent: Akamai-SiteSnapshot/* # Any empty value, indicates that all URLs can be retrieved. Disallow: