Debian Codegraph @

doc/robots.txt.example

raw
1 # Simple robots.txt example, put this into your web root.
2 # See the complete reference at
3 # http://www.robotstxt.org/wc/norobots-rfc.html
4
5 # disallow cgi-bin access for all robots
6 User-agent: *
7 Disallow: /cgi-bin/
8
9 # All LinkChecker versions are not allowed to check anything
10 User-agent: LinkChecker
11 Disallow: /