site: added dummy robots.txt to handle those from crawling robots.

rhodecode/config/middleware.py

@@ -331,6 +331,13 @@ def includeme_first(config):
     config.add_view(favicon_redirect, route_name='favicon')
     config.add_route('favicon', '/favicon.ico')
 
+    def robots_redirect(context, request):
+        return HTTPFound(
+            request.static_path('rhodecode:public/robots.txt'))
+
+    config.add_view(robots_redirect, route_name='robots')
+    config.add_route('robots', '/robots.txt')
+
     config.add_static_view(
         '_static/deform', 'deform:static')
     config.add_static_view(
@@ -362,7 +369,6 @@ def wrap_app_in_wsgi_middlewares(pyramid_app, config):
         pyramid_app = make_gzip_middleware(
             pyramid_app, settings, compress_level=1)
 
-
     # this should be the outer most middleware in the wsgi stack since
     # middleware like Routes make database calls
     def pyramid_app_with_cleanup(environ, start_response):

rhodecode/public/robots.txt

@@ -0,0 +1,8 @@
+# See http://www.robotstxt.org/robotstxt.html for documentation on
+# how to use the robots.txt file
+#
+# To forbid all spiders from indexing, uncomment the next two lines:
+# User-Agent: *
+# Disallow: /
+
+User-Agent: *