From e6833d2fd86f3ec7621d01273cb5114afd06be87 Mon Sep 17 00:00:00 2001 From: Daniel Dourvaris Date: Sat, 13 Aug 2016 02:56:48 +0300 Subject: [PATCH 01/10] db: move Session.remove to outer wsgi layer and also add it to hooks daemon to avoid leaving connections open in the db pool fixes #4173, refs #4166 --- rhodecode/config/middleware.py | 25 +++++++++++- rhodecode/lib/hooks_daemon.py | 6 ++- rhodecode/lib/middleware/simplevcs.py | 7 +++- rhodecode/tweens.py | 55 ++++++++++++--------------- 4 files changed, 59 insertions(+), 34 deletions(-) diff --git a/rhodecode/config/middleware.py b/rhodecode/config/middleware.py index 161c436c..028cc329 100644 --- a/rhodecode/config/middleware.py +++ b/rhodecode/config/middleware.py @@ -39,6 +39,7 @@ from routes.middleware import RoutesMiddleware import routes.util import rhodecode +from rhodecode.model import meta from rhodecode.config import patches from rhodecode.config.routing import STATIC_FILE_PREFIX from rhodecode.config.environment import ( @@ -158,6 +159,10 @@ def make_pyramid_app(global_config, **settings): pyramid_app = config.make_wsgi_app() pyramid_app = wrap_app_in_wsgi_middlewares(pyramid_app, config) pyramid_app.config = config + + # creating the app uses a connection - return it after we are done + meta.Session.remove() + return pyramid_app @@ -374,7 +379,25 @@ def wrap_app_in_wsgi_middlewares(pyramid_app, config): pyramid_app = make_gzip_middleware( pyramid_app, settings, compress_level=1) - return pyramid_app + + # this should be the outer most middleware in the wsgi stack since + # middleware like Routes make database calls + def pyramid_app_with_cleanup(environ, start_response): + try: + return pyramid_app(environ, start_response) + finally: + # Dispose current database session and rollback uncommitted + # transactions. + meta.Session.remove() + + # In a single threaded mode server, on non sqlite db we should have + # '0 Current Checked out connections' at the end of a request, + # if not, then something, somewhere is leaving a connection open + pool = meta.Base.metadata.bind.engine.pool + log.debug('sa pool status: %s', pool.status()) + + + return pyramid_app_with_cleanup def sanitize_settings_and_apply_defaults(settings): diff --git a/rhodecode/lib/hooks_daemon.py b/rhodecode/lib/hooks_daemon.py index dd86fe3f..50dd4f34 100644 --- a/rhodecode/lib/hooks_daemon.py +++ b/rhodecode/lib/hooks_daemon.py @@ -30,6 +30,7 @@ import Pyro4 import pylons import rhodecode +from rhodecode.model import meta from rhodecode.lib import hooks_base from rhodecode.lib.utils2 import ( AttributeDict, safe_str, get_routes_generator_for_server_url) @@ -64,7 +65,10 @@ class HooksHttpHandler(BaseHTTPRequestHandler): def _call_hook(self, method, extras): hooks = Hooks() - result = getattr(hooks, method)(extras) + try: + result = getattr(hooks, method)(extras) + finally: + meta.Session.remove() return result def log_message(self, format, *args): diff --git a/rhodecode/lib/middleware/simplevcs.py b/rhodecode/lib/middleware/simplevcs.py index 753b0cca..69601f1d 100644 --- a/rhodecode/lib/middleware/simplevcs.py +++ b/rhodecode/lib/middleware/simplevcs.py @@ -406,8 +406,11 @@ class SimpleVCS(object): yield chunk finally: # invalidate cache on push - if action == 'push': - self._invalidate_cache(repo_name) + try: + if action == 'push': + self._invalidate_cache(repo_name) + finally: + meta.Session.remove() def _get_repository_name(self, environ): """Get repository name out of the environmnent diff --git a/rhodecode/tweens.py b/rhodecode/tweens.py index 49958580..b9b66c7b 100644 --- a/rhodecode/tweens.py +++ b/rhodecode/tweens.py @@ -40,40 +40,35 @@ def pylons_compatibility_tween_factory(handler, registry): from pyramid. For example while rendering an old template that uses the 'c' or 'h' objects. This tween sets up the needed pylons globals. """ - try: - config = rhodecode.CONFIG - environ = request.environ - session = request.session - session_key = (config['pylons.environ_config'] - .get('session', 'beaker.session')) + config = rhodecode.CONFIG + environ = request.environ + session = request.session + session_key = (config['pylons.environ_config'] + .get('session', 'beaker.session')) - # Setup pylons globals. - pylons.config._push_object(config) - pylons.request._push_object(request) - pylons.session._push_object(session) - environ[session_key] = session - pylons.url._push_object(URLGenerator(config['routes.map'], - environ)) + # Setup pylons globals. + pylons.config._push_object(config) + pylons.request._push_object(request) + pylons.session._push_object(session) + environ[session_key] = session + pylons.url._push_object(URLGenerator(config['routes.map'], + environ)) - # TODO: Maybe we should use the language from pyramid. - translator = _get_translator(config.get('lang')) - pylons.translator._push_object(translator) + # TODO: Maybe we should use the language from pyramid. + translator = _get_translator(config.get('lang')) + pylons.translator._push_object(translator) - # Get the rhodecode auth user object and make it available. - auth_user = get_auth_user(environ) - request.user = auth_user - environ['rc_auth_user'] = auth_user + # Get the rhodecode auth user object and make it available. + auth_user = get_auth_user(environ) + request.user = auth_user + environ['rc_auth_user'] = auth_user - # Setup the pylons context object ('c') - context = ContextObj() - context.rhodecode_user = auth_user - attach_context_attributes(context, request) - pylons.tmpl_context._push_object(context) - return handler(request) - finally: - # Dispose current database session and rollback uncommitted - # transactions. - meta.Session.remove() + # Setup the pylons context object ('c') + context = ContextObj() + context.rhodecode_user = auth_user + attach_context_attributes(context, request) + pylons.tmpl_context._push_object(context) + return handler(request) return pylons_compatibility_tween From 8c01d385c8edf0bfc85ae28d4345fd95e91d31de Mon Sep 17 00:00:00 2001 From: Daniel Dourvaris Date: Mon, 15 Aug 2016 17:52:51 +0300 Subject: [PATCH 02/10] pyro: remove db Session when callback finishes to avoid leaving hanging connections --- rhodecode/lib/hooks_daemon.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rhodecode/lib/hooks_daemon.py b/rhodecode/lib/hooks_daemon.py index 50dd4f34..00fa57d6 100644 --- a/rhodecode/lib/hooks_daemon.py +++ b/rhodecode/lib/hooks_daemon.py @@ -265,6 +265,7 @@ class Hooks(object): } finally: pylons.url._pop_object() + meta.Session.remove() return { 'status': result.status, From 75af072019c9b5767db54a32f3f6120956fb81f7 Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Sat, 20 Aug 2016 18:52:37 +0200 Subject: [PATCH 03/10] files: pep8 fixes --- rhodecode/lib/diffs.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/rhodecode/lib/diffs.py b/rhodecode/lib/diffs.py index 91c04cdd..65dc33a0 100644 --- a/rhodecode/lib/diffs.py +++ b/rhodecode/lib/diffs.py @@ -46,6 +46,7 @@ class OPS(object): MOD = 'M' DEL = 'D' + def wrap_to_table(str_): return ''' @@ -57,8 +58,8 @@ def wrap_to_table(str_): def wrapped_diff(filenode_old, filenode_new, diff_limit=None, file_limit=None, - show_full_diff=False, ignore_whitespace=True, line_context=3, - enable_comments=False): + show_full_diff=False, ignore_whitespace=True, line_context=3, + enable_comments=False): """ returns a wrapped diff into a table, checks for cut_off_limit for file and whole diff and presents proper message @@ -79,8 +80,9 @@ def wrapped_diff(filenode_old, filenode_new, diff_limit=None, file_limit=None, f_gitdiff = get_gitdiff(filenode_old, filenode_new, ignore_whitespace=ignore_whitespace, context=line_context) - diff_processor = DiffProcessor(f_gitdiff, format='gitdiff', diff_limit=diff_limit, - file_limit=file_limit, show_full_diff=show_full_diff) + diff_processor = DiffProcessor( + f_gitdiff, format='gitdiff', diff_limit=diff_limit, + file_limit=file_limit, show_full_diff=show_full_diff) _parsed = diff_processor.prepare() diff = diff_processor.as_html(enable_comments=enable_comments) @@ -190,7 +192,8 @@ class DiffProcessor(object): # used for inline highlighter word split _token_re = re.compile(r'()(>|<|&|\W+?)') - def __init__(self, diff, format='gitdiff', diff_limit=None, file_limit=None, show_full_diff=True): + def __init__(self, diff, format='gitdiff', diff_limit=None, + file_limit=None, show_full_diff=True): """ :param diff: A `Diff` object representing a diff from a vcs backend :param format: format of diff passed, `udiff` or `gitdiff` From 464236b412b10e289408ae7f92f4395ac0f8af7c Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Sat, 20 Aug 2016 18:53:32 +0200 Subject: [PATCH 04/10] diffs: limit the file context to ~1mln lines. Fixes #4184 Something more than 1mln lines is anyway not usable in web browser. --- rhodecode/lib/diffs.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/rhodecode/lib/diffs.py b/rhodecode/lib/diffs.py index 65dc33a0..8abc1a05 100644 --- a/rhodecode/lib/diffs.py +++ b/rhodecode/lib/diffs.py @@ -40,6 +40,10 @@ from rhodecode.lib.utils2 import safe_unicode log = logging.getLogger(__name__) +# define max context, a file with more than this numbers of lines is unusable +# in browser anyway +MAX_CONTEXT = 1024 * 1014 + class OPS(object): ADD = 'A' @@ -117,6 +121,10 @@ def get_gitdiff(filenode_old, filenode_new, ignore_whitespace=True, context=3): """ # make sure we pass in default context context = context or 3 + # protect against IntOverflow when passing HUGE context + if context > MAX_CONTEXT: + context = MAX_CONTEXT + submodules = filter(lambda o: isinstance(o, SubModuleNode), [filenode_new, filenode_old]) if submodules: From 96bf68036cf21ea47e424337b0232a644d04271f Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Tue, 23 Aug 2016 11:07:32 +0200 Subject: [PATCH 05/10] docs: added 4.3.1 release docs --- docs/release-notes/release-notes-4.3.1.rst | 15 +++++++++++++++ docs/release-notes/release-notes.rst | 1 + 2 files changed, 16 insertions(+) create mode 100644 docs/release-notes/release-notes-4.3.1.rst diff --git a/docs/release-notes/release-notes-4.3.1.rst b/docs/release-notes/release-notes-4.3.1.rst new file mode 100644 index 00000000..17c48dc7 --- /dev/null +++ b/docs/release-notes/release-notes-4.3.1.rst @@ -0,0 +1,15 @@ +|RCE| 4.3.1 |RNS| +----------------- + +Release Date +^^^^^^^^^^^^ + +- 2016-08-23 + +Fixes +^^^^^ + +- Core: fixed database session cleanups. This will make sure RhodeCode can + function correctly after database server problems. Fixes #4173, refs #4166 +- Diffs: limit the file context to ~1mln lines. Fixes #4184, also make sure + this doesn't trigger Integer overflow for msgpack. \ No newline at end of file diff --git a/docs/release-notes/release-notes.rst b/docs/release-notes/release-notes.rst index 566c1c55..6b247876 100644 --- a/docs/release-notes/release-notes.rst +++ b/docs/release-notes/release-notes.rst @@ -9,6 +9,7 @@ Release Notes .. toctree:: :maxdepth: 1 + release-notes-4.3.1.rst release-notes-4.3.0.rst release-notes-4.2.1.rst release-notes-4.2.0.rst From 18080814d5b7a5e2fc0bde8c4340e5e9f73e81d2 Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Tue, 23 Aug 2016 12:03:01 +0200 Subject: [PATCH 06/10] docs: update SVN configuration docs --- docs/admin/svn-http.rst | 96 ++++++++++++++++++++++++++++++----------- 1 file changed, 70 insertions(+), 26 deletions(-) diff --git a/docs/admin/svn-http.rst b/docs/admin/svn-http.rst index 39121f11..73121bef 100644 --- a/docs/admin/svn-http.rst +++ b/docs/admin/svn-http.rst @@ -3,16 +3,21 @@ |svn| With Write Over HTTP -------------------------- -To use |svn| with write access, the currently supported method is over HTTP. -This requires you to configure your local machine so that it can access your -|RCE| instance. +To use |svn| with read/write support over the |svn| protocol, you have to +configure HTTP |svn| backend. Prerequisites ^^^^^^^^^^^^^ -- Enable lab setting on your |RCE| instance, see :ref:`lab-settings`. -- You need to install the following tools on your local machine: ``Apache`` and - ``mod_dav_svn``. Use the following Ubuntu as an example. +- Enable HTTP support inside labs setting on your |RCE| instance, + see :ref:`lab-settings`. +- You need to install the following tools on the machine that is running an + instance of |RCE|: + ``Apache HTTP Server`` and + ``mod_dav_svn``. + + +Using Ubuntu Distribution as an example you can run: .. code-block:: bash @@ -32,41 +37,80 @@ Configuring Apache Setup It is recommended to run Apache on a port other than 80, due to possible conflicts with other HTTP servers like nginx. To do this, set the ``Listen`` parameter in the ``/etc/apache2/ports.conf`` file, for example - ``Listen 8090`` + ``Listen 8090``. - It is also recommended to run apache as the same user as |RCE|, otherwise - permission issues could occur. To do this edit the ``/etc/apache2/envvars`` + +.. warning:: + + Make sure your Apache instance which runs the mod_dav_svn module is + only accessible by RhodeCode. Otherwise everyone is able to browse + the repositories or run subversion operations (checkout/commit/etc.). + +It is also recommended to run apache as the same user as |RCE|, otherwise +permission issues could occur. To do this edit the ``/etc/apache2/envvars`` .. code-block:: apache - export APACHE_RUN_USER=ubuntu - export APACHE_RUN_GROUP=ubuntu + export APACHE_RUN_USER=rhodecode + export APACHE_RUN_GROUP=rhodecode 1. To configure Apache, create and edit a virtual hosts file, for example - :file:`/etc/apache2/sites-available/default.conf`, or create another - virtual hosts file and add a location section inside the - ```` section. + :file:`/etc/apache2/sites-available/default.conf`. Below is an example + how to use one with auto-generated config ```mod_dav_svn.conf``` + from configured |RCE| instance. .. code-block:: apache - - DAV svn - # Must be explicit path, relative not supported - SVNParentPath /PATH/TO/REPOSITORIES - SVNListParentPath On - Allow from all - Order allow,deny - + + ServerAdmin rhodecode-admin@localhost + DocumentRoot /var/www/html + ErrorLog ${'${APACHE_LOG_DIR}'}/error.log + CustomLog ${'${APACHE_LOG_DIR}'}/access.log combined + Include /home/user/.rccontrol/enterprise-1/mod_dav_svn.conf + -.. note:: - - Once configured, check that you can see the list of repositories on your - |RCE| instance. 2. Go to the :menuselection:`Admin --> Settings --> Labs` page, and enable :guilabel:`Proxy Subversion HTTP requests`, and specify the :guilabel:`Subversion HTTP Server URL`. +3. Open the |RCE| configuration file, + :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` + +4. Add the following configuration option in the ``[app:main]`` + section if you don't have it yet. + + This enable mapping of created |RCE| repo groups into special |svn| paths. + Each time a new repository group will be created the system will update + the template file, and create new mapping. Apache web server needs to be + reloaded to pick up the changes on this file. + It's recommended to add reload into a crontab so the changes can be picked + automatically once someone creates an repository group inside RhodeCode. + + +.. code-block:: ini + + ############################################## + ### Subversion proxy support (mod_dav_svn) ### + ############################################## + ## Enable or disable the config file generation. + svn.proxy.generate_config = true + ## Generate config file with `SVNListParentPath` set to `On`. + svn.proxy.list_parent_path = true + ## Set location and file name of generated config file. + svn.proxy.config_file_path = %(here)s/mod_dav_svn.conf + ## File system path to the directory containing the repositories served by + ## RhodeCode. + svn.proxy.parent_path_root = /path/to/repo_store + ## Used as a prefix to the block in the generated config file. In + ## most cases it should be set to `/`. + svn.proxy.location_root = / + + +This would create a special template file called ```mod_dav_svn.conf```. We +used that file path in the apache config above inside the Include statement. + + Using |svn| ^^^^^^^^^^^ From 9bbf4c4e07c7c28fee264e9e5c5a61f186571f93 Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Tue, 23 Aug 2016 10:11:50 +0000 Subject: [PATCH 07/10] release: Start preparation for 4.3.1 --- .bumpversion.cfg | 2 +- .release.cfg | 15 +++++---------- rhodecode/VERSION | 2 +- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 647990df..cda787cd 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 4.3.0 +current_version = 4.3.1 message = release: Bump version {current_version} to {new_version} [bumpversion:file:rhodecode/VERSION] diff --git a/.release.cfg b/.release.cfg index 3525ade0..267f4ccc 100644 --- a/.release.cfg +++ b/.release.cfg @@ -4,26 +4,21 @@ done = false [task:bump_version] done = true +[task:rc_tools_pinned] + [task:fixes_on_stable] -done = true [task:pip2nix_generated] -done = true [task:changelog_updated] -done = true [task:generate_api_docs] -done = true - -[task:updated_translation] -done = true [release] -state = prepared -version = 4.3.0 +state = in_progress +version = 4.3.1 -[task:rc_tools_pinned] +[task:updated_translation] [task:generate_js_routes] diff --git a/rhodecode/VERSION b/rhodecode/VERSION index 81911389..ecedc98d 100644 --- a/rhodecode/VERSION +++ b/rhodecode/VERSION @@ -1 +1 @@ -4.3.0 \ No newline at end of file +4.3.1 \ No newline at end of file From b89a57a64d5772c11cc5cf9459fb4bd45450c01d Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Tue, 23 Aug 2016 10:15:40 +0000 Subject: [PATCH 08/10] release: updated pip2nix output for 4.3.1 --- .release.cfg | 4 ++++ pkgs/python-packages.nix | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.release.cfg b/.release.cfg index 267f4ccc..3c7cf55a 100644 --- a/.release.cfg +++ b/.release.cfg @@ -5,14 +5,18 @@ done = false done = true [task:rc_tools_pinned] +done = true [task:fixes_on_stable] +done = true [task:pip2nix_generated] [task:changelog_updated] +done = true [task:generate_api_docs] +done = true [release] state = in_progress diff --git a/pkgs/python-packages.nix b/pkgs/python-packages.nix index 1956674c..b00e3722 100644 --- a/pkgs/python-packages.nix +++ b/pkgs/python-packages.nix @@ -1430,7 +1430,7 @@ }; }; rhodecode-enterprise-ce = super.buildPythonPackage { - name = "rhodecode-enterprise-ce-4.3.0"; + name = "rhodecode-enterprise-ce-4.3.1"; buildInputs = with self; [WebTest configobj cssselect flake8 lxml mock pytest pytest-cov pytest-runner]; doCheck = true; propagatedBuildInputs = with self; [Babel Beaker FormEncode Mako Markdown MarkupSafe MySQL-python Paste PasteDeploy PasteScript Pygments Pylons Pyro4 Routes SQLAlchemy Tempita URLObject WebError WebHelpers WebHelpers2 WebOb WebTest Whoosh alembic amqplib anyjson appenlight-client authomatic backport-ipaddress celery channelstream colander decorator deform docutils gevent gunicorn infrae.cache ipython iso8601 kombu msgpack-python packaging psycopg2 py-gfm pycrypto pycurl pyparsing pyramid pyramid-debugtoolbar pyramid-mako pyramid-beaker pysqlite python-dateutil python-ldap python-memcached python-pam recaptcha-client repoze.lru requests simplejson waitress zope.cachedescriptors dogpile.cache dogpile.core psutil py-bcrypt]; From e9d6e36cb217d280a07a95076aaf8c094901b269 Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Tue, 23 Aug 2016 10:15:42 +0000 Subject: [PATCH 09/10] release: Finish preparation for 4.3.1 --- .release.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.release.cfg b/.release.cfg index 3c7cf55a..0c50b5b7 100644 --- a/.release.cfg +++ b/.release.cfg @@ -11,6 +11,7 @@ done = true done = true [task:pip2nix_generated] +done = true [task:changelog_updated] done = true @@ -19,7 +20,7 @@ done = true done = true [release] -state = in_progress +state = prepared version = 4.3.1 [task:updated_translation] From 991a82d132835ba9e131a48af214c54d3af01612 Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Tue, 23 Aug 2016 10:15:42 +0000 Subject: [PATCH 10/10] Added tag v4.3.1 for changeset 0e4dc11b58ca