# Tells Scanning Robots Where They Are And Are Not Welcome # User-agent: can also specify by name; "*" is for everyone # Disallow: if this matches first part of requested path, # forget it User-agent: * # applies to all robots Allow: /index.html # allows index.html to be indexed Disallow: /php # disallow indexing of all pages Disallow: /images # disallow indexing of all pages Disallow: /priv_stats # disallow indexing of all pages Disallow: /tests # disallow indexing of all pages