From a01df6d465cc0b9f240344f85fc14312dad7d3ad Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Fri, 13 Jan 2017 11:32:41 +0100 Subject: [PATCH] site: added dummy robots.txt to handle those from crawling robots. --- rhodecode/config/middleware.py | 8 +++++++- rhodecode/public/robots.txt | 8 ++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 rhodecode/public/robots.txt diff --git a/rhodecode/config/middleware.py b/rhodecode/config/middleware.py index 112e1744..fb34a1bb 100644 --- a/rhodecode/config/middleware.py +++ b/rhodecode/config/middleware.py @@ -331,6 +331,13 @@ def includeme_first(config): config.add_view(favicon_redirect, route_name='favicon') config.add_route('favicon', '/favicon.ico') + def robots_redirect(context, request): + return HTTPFound( + request.static_path('rhodecode:public/robots.txt')) + + config.add_view(robots_redirect, route_name='robots') + config.add_route('robots', '/robots.txt') + config.add_static_view( '_static/deform', 'deform:static') config.add_static_view( @@ -362,7 +369,6 @@ def wrap_app_in_wsgi_middlewares(pyramid_app, config): pyramid_app = make_gzip_middleware( pyramid_app, settings, compress_level=1) - # this should be the outer most middleware in the wsgi stack since # middleware like Routes make database calls def pyramid_app_with_cleanup(environ, start_response): diff --git a/rhodecode/public/robots.txt b/rhodecode/public/robots.txt new file mode 100644 index 00000000..054de641 --- /dev/null +++ b/rhodecode/public/robots.txt @@ -0,0 +1,8 @@ +# See http://www.robotstxt.org/robotstxt.html for documentation on +# how to use the robots.txt file +# +# To forbid all spiders from indexing, uncomment the next two lines: +# User-Agent: * +# Disallow: / + +User-Agent: *