release: Merge default into stable for release preparation

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 4.11.6
+current_version = 4.12.0
 message = release: Bump version {current_version} to {new_version}
 
 [bumpversion:file:rhodecode/VERSION]

.release.cfg

@@ -5,26 +5,21 @@ done = false
 done = true
 
 [task:rc_tools_pinned]
-done = true
 
 [task:fixes_on_stable]
-done = true
 
 [task:pip2nix_generated]
-done = true
 
 [task:changelog_updated]
-done = true
 
 [task:generate_api_docs]
-done = true
-
-[release]
-state = prepared
-version = 4.11.6
 
 [task:updated_translation]
 
+[release]
+state = in_progress
+version = 4.12.0
+
 [task:generate_js_routes]
 
 [task:updated_trial_license]

MANIFEST.in

@@ -16,9 +16,6 @@ recursive-include configs *
 # translations
 recursive-include rhodecode/i18n *
 
-# hook templates
-recursive-include rhodecode/config/hook_templates *
-
 # non-python core stuff
 recursive-include rhodecode *.cfg
 recursive-include rhodecode *.json

configs/development.ini

@@ -190,9 +190,6 @@ force_https = false
 ## use Strict-Transport-Security headers
 use_htsts = false
 
-## number of commits stats will parse on each iteration
-commit_parse_limit = 25
-
 ## git rev filter option, --all is the default filter, if you need to
 ## hide all refs in changelog switch this to --branches --tags
 git_rev_filter = --branches --tags
@@ -708,12 +705,12 @@ formatter = color_formatter_sql
 
 [formatter_generic]
 class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
-format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
+format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter]
 class = rhodecode.lib.logging_formatter.ColorFormatter
-format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
+format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter_sql]

configs/gunicorn_config.py

@@ -55,12 +55,13 @@ keepalive = 2
 
 # SERVER MECHANICS
 # None == system temp dir
+# worker_tmp_dir is recommended to be set to some tmpfs
 worker_tmp_dir = None
 tmp_upload_dir = None
 
 # Custom log format
 access_log_format = (
-    '%(t)s GNCRN %(p)-8s %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
+    '%(t)s [%(p)-8s] GNCRN %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
 
 # self adjust workers based on CPU count
 # workers = multiprocessing.cpu_count() * 2 + 1
@@ -119,16 +120,16 @@ def child_exit(server, worker):
 
 
 def pre_request(worker, req):
-    return
-    worker.log.debug("[<%-10s>] PRE WORKER: %s %s",
-                     worker.pid, req.method, req.path)
+    worker.start_time = time.time()
+    worker.log.debug(
+        "GNCRN PRE  WORKER [cnt:%s]: %s %s", worker.nr, req.method, req.path)
 
 
 def post_request(worker, req, environ, resp):
-    return
-    worker.log.debug("[<%-10s>] POST WORKER: %s %s resp: %s", worker.pid,
-                     req.method, req.path, resp.status_code)
-
+    total_time = time.time() - worker.start_time
+    worker.log.debug(
+        "GNCRN POST WORKER [cnt:%s]: %s %s resp: %s, Load Time: %.3fs",
+        worker.nr, req.method, req.path, resp.status_code, total_time)
 
 
 class RhodeCodeLogger(Logger):

configs/production.ini

@@ -165,9 +165,6 @@ force_https = false
 ## use Strict-Transport-Security headers
 use_htsts = false
 
-## number of commits stats will parse on each iteration
-commit_parse_limit = 25
-
 ## git rev filter option, --all is the default filter, if you need to
 ## hide all refs in changelog switch this to --branches --tags
 git_rev_filter = --branches --tags
@@ -678,12 +675,12 @@ formatter = generic
 
 [formatter_generic]
 class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
-format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
+format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter]
 class = rhodecode.lib.logging_formatter.ColorFormatter
-format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
+format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter_sql]

docs/admin/sec-sophos-umc.rst

@@ -1,4 +1,4 @@
-.. _sec-your-server:
+.. _sec-sophos-umc:
 
 Securing Your Server via Sophos UTM 9
 -------------------------------------

docs/admin/svn-path-permissions.rst

@@ -0,0 +1,80 @@
+.. _svn-path-permissions:
+
+|svn| Enabling Path Permissions
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Because |RCEE| uses standard svn apache mod_svn we can take advantage of the
+authz configuration to protect paths and branches.
+
+
+Configuring RhodeCode
+=====================
+
+
+1. To configure path based permissions first we need to use a customized
+   mod_dav_svn.conf.
+
+   Open :file:`home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file.
+   And find `svn.proxy.config_template` setting. Now set a new path to read
+   the template from. For example:
+
+    .. code-block:: ini
+
+        svn.proxy.config_template = /home/ubuntu/rhodecode/custom_mod_dav_svn.conf.mako
+
+
+2. Create the file as in example: `/home/ubuntu/rhodecode/custom_mod_dav_svn.conf.mako`
+   You can download one from:
+
+    `<https://code.rhodecode.com/rhodecode-enterprise-ce/files/default/rhodecode/apps/svn_support/templates/mod-dav-svn.conf.mako/>`_
+
+3. Add (if not yet exists) a section `AuthzSVNReposRelativeAccessFile`  in order
+   to read the path auth file.
+
+   Example modified config section enabling reading the authz file relative
+   to repository path. Means located in `/storage_dir/repo_name/conf/authz`
+
+    .. code-block:: text
+
+
+        # snip ...
+
+        # use specific SVN conf/authz file for each repository
+        AuthzSVNReposRelativeAccessFile authz
+
+        Allow from all
+        # snip ...
+
+    .. note::
+
+       The `AuthzSVNReposRelativeAccessFile` should go above the `Allow from all`
+       directive.
+
+
+4. Restart RhodeCode, Go to
+   the :menuselection:`Admin --> Settings --> VCS` page, and
+   click :guilabel:`Generate Apache Config`.
+   This will now generate a new configuration with enabled changes to read
+   the authz file. You can verify if changes were made by checking the generated
+   mod_dav_svn.conf file which is included in your apache configuration.
+
+5. Specify new rules in the repository authz configuration.
+   edit a file in :file:`repo_name/conf/authz`. For example, we specify that
+   only admin is allowed to push to develop branch
+
+    .. code-block:: ini
+
+        [/branches/develop]
+        * = r
+        admin = rw
+
+
+   For more example see:
+   `<https://svn.apache.org/repos/asf/subversion/trunk/subversion/mod_authz_svn/INSTALL/>`_
+
+  Those rules also work for paths, so not only branches but all different
+  paths inside the repository can be specified.
+
+6. Reload Apache. If all is configured correctly it should not be allowed to
+   commit according to specified rules.
+

docs/admin/system-admin.rst

@@ -19,6 +19,7 @@ The following are the most common system administration tasks.
     config-files-overview
     vcs-server
     svn-http
+    svn-path-permissions
     gunicorn-ssl-support
     apache-config
     nginx-config

docs/api/api.rst

@@ -196,6 +196,7 @@ are not required in args.
 .. --- API DEFS MARKER ---
 .. toctree::
 
+   methods/views
    methods/license-methods
    methods/deprecated-methods
    methods/gist-methods

docs/api/methods/repo-methods.rst

@@ -66,7 +66,7 @@ comment_commit
 create_repo 
 -----------
 
-.. py:function:: create_repo(apiuser, repo_name, repo_type, owner=<Optional:<OptionalAttr:apiuser>>, description=<Optional:''>, private=<Optional:False>, clone_uri=<Optional:None>, landing_rev=<Optional:'rev:tip'>, enable_statistics=<Optional:False>, enable_locking=<Optional:False>, enable_downloads=<Optional:False>, copy_permissions=<Optional:False>)
+.. py:function:: create_repo(apiuser, repo_name, repo_type, owner=<Optional:<OptionalAttr:apiuser>>, description=<Optional:''>, private=<Optional:False>, clone_uri=<Optional:None>, push_uri=<Optional:None>, landing_rev=<Optional:'rev:tip'>, enable_statistics=<Optional:False>, enable_locking=<Optional:False>, enable_downloads=<Optional:False>, copy_permissions=<Optional:False>)
 
    Creates a repository.
 
@@ -95,6 +95,8 @@ create_repo
    :type private: bool
    :param clone_uri: set clone_uri
    :type clone_uri: str
+   :param push_uri: set push_uri
+   :type push_uri: str
    :param landing_rev: <rev_type>:<rev>
    :type landing_rev: str
    :param enable_locking:
@@ -789,7 +791,7 @@ maintenance
 pull 
 ----
 
-.. py:function:: pull(apiuser, repoid)
+.. py:function:: pull(apiuser, repoid, remote_uri=<Optional:None>)
 
    Triggers a pull on the given repository from a remote location. You
    can use this to keep remote repositories up-to-date.
@@ -804,6 +806,8 @@ pull
    :type apiuser: AuthUser
    :param repoid: The repository name or repository ID.
    :type repoid: str or int
+   :param remote_uri: Optional remote URI to pass in for pull
+   :type remote_uri: str
 
    Example output:
 
@@ -811,7 +815,7 @@ pull
 
      id : <id_given_in_input>
      result : {
-       "msg": "Pulled from `<repository name>`"
+       "msg": "Pulled from url `<remote_url>` on repo `<repository name>`"
        "repository": "<repository name>"
      }
      error :  null
@@ -823,7 +827,7 @@ pull
      id : <id_given_in_input>
      result : null
      error :  {
-       "Unable to pull changes from `<reponame>`"
+       "Unable to push changes from `<remote_url>`"
      }
 
 
@@ -976,7 +980,7 @@ strip
 update_repo 
 -----------
 
-.. py:function:: update_repo(apiuser, repoid, repo_name=<Optional:None>, owner=<Optional:<OptionalAttr:apiuser>>, description=<Optional:''>, private=<Optional:False>, clone_uri=<Optional:None>, landing_rev=<Optional:'rev:tip'>, fork_of=<Optional:None>, enable_statistics=<Optional:False>, enable_locking=<Optional:False>, enable_downloads=<Optional:False>, fields=<Optional:''>)
+.. py:function:: update_repo(apiuser, repoid, repo_name=<Optional:None>, owner=<Optional:<OptionalAttr:apiuser>>, description=<Optional:''>, private=<Optional:False>, clone_uri=<Optional:None>, push_uri=<Optional:None>, landing_rev=<Optional:'rev:tip'>, fork_of=<Optional:None>, enable_statistics=<Optional:False>, enable_locking=<Optional:False>, enable_downloads=<Optional:False>, fields=<Optional:''>)
 
    Updates a repository with the given information.
 

docs/api/methods/user-group-methods.rst

@@ -51,7 +51,7 @@ add_user_to_user_group
 create_user_group 
 -----------------
 
-.. py:function:: create_user_group(apiuser, group_name, description=<Optional:''>, owner=<Optional:<OptionalAttr:apiuser>>, active=<Optional:True>)
+.. py:function:: create_user_group(apiuser, group_name, description=<Optional:''>, owner=<Optional:<OptionalAttr:apiuser>>, active=<Optional:True>, sync=<Optional:None>)
 
    Creates a new user group.
 
@@ -71,6 +71,9 @@ create_user_group
    :type owner: Optional(str or int)
    :param active: Set this group as active.
    :type active: Optional(``True`` | ``False``)
+   :param sync: Set enabled or disabled the automatically sync from
+       external authentication types like ldap.
+   :type sync: Optional(``True`` | ``False``)
 
    Example output:
 
@@ -368,7 +371,7 @@ revoke_user_permission_from_user_group
 update_user_group 
 -----------------
 
-.. py:function:: update_user_group(apiuser, usergroupid, group_name=<Optional:''>, description=<Optional:''>, owner=<Optional:None>, active=<Optional:True>)
+.. py:function:: update_user_group(apiuser, usergroupid, group_name=<Optional:''>, description=<Optional:''>, owner=<Optional:None>, active=<Optional:True>, sync=<Optional:None>)
 
    Updates the specified `user group` with the details provided.
 
@@ -387,6 +390,9 @@ update_user_group
    :type owner: Optional(str or int)
    :param active: Set the group as active.
    :type active: Optional(``True`` | ``False``)
+   :param sync: Set enabled or disabled the automatically sync from
+       external authentication types like ldap.
+   :type sync: Optional(``True`` | ``False``)
 
    Example output:
 

docs/api/methods/views.rst

@@ -0,0 +1,48 @@
+.. _views-ref:
+
+views
+=====
+
+push (EE only)
+--------------
+
+.. py:function:: push(apiuser, repoid, remote_uri=<Optional:None>)
+
+   Triggers a push on the given repository from a remote location. You
+   can use this to keep remote repositories up-to-date.
+
+   This command can only be run using an |authtoken| with admin
+   rights to the specified repository. For more information,
+   see :ref:`config-token-ref`.
+
+   This command takes the following options:
+
+   :param apiuser: This is filled automatically from the |authtoken|.
+   :type apiuser: AuthUser
+   :param repoid: The repository name or repository ID.
+   :type repoid: str or int
+   :param remote_uri: Optional remote URI to pass in for push
+   :type remote_uri: str
+
+   Example output:
+
+   .. code-block:: bash
+
+     id : <id_given_in_input>
+     result : {
+       "msg": "Pushed to url `<remote_url>` on repo `<repository name>`"
+       "repository": "<repository name>"
+     }
+     error :  null
+
+   Example error output:
+
+   .. code-block:: bash
+
+     id : <id_given_in_input>
+     result : null
+     error :  {
+       "Unable to push changes to `<remote_url>`"
+     }
+
+

docs/auth/auth-ldap-groups.rst

@@ -0,0 +1,112 @@
+.. _config-ldap-groups-ref:
+
+LDAP/AD With User Groups Sync
+-----------------------------
+
+|RCM| supports LDAP (Lightweight Directory Access Protocol) or
+AD (active Directory) authentication.
+All LDAP versions are supported, with the following |RCM| plugins managing each:
+
+* For LDAP/AD with user group sync use ``LDAP + User Groups (egg:rhodecode-enterprise-ee#ldap_group)``
+
+RhodeCode reads all data defined from plugin and creates corresponding
+accounts on local database after receiving data from LDAP. This is done on
+every user log-in including operations like pushing/pulling/checkout.
+In addition group membership is read from LDAP and following operations are done:
+
+- automatic addition of user to |RCM| user group
+- automatic removal of user from any other  |RCM| user groups not specified in LDAP.
+  The removal is done *only* on groups that are marked to be synced from ldap.
+  This setting can be changed in advanced settings on user groups
+- automatic creation of user groups if they aren't yet existing in |RCM|
+- marking user as super-admins if he is a member of any admin group defined in plugin settings
+
+This plugin is available only in EE Edition.
+
+.. important::
+
+   The email used with your |RCE| super-admin account needs to match the email
+   address attached to your admin profile in LDAP. This is because
+   within |RCE| the user email needs to be unique, and multiple users
+   cannot share an email account.
+
+   Likewise, if as an admin you also have a user account, the email address
+   attached to the user account needs to be different.
+
+
+LDAP Configuration Steps
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+To configure |LDAP|, use the following steps:
+
+1. From the |RCM| interface, select
+   :menuselection:`Admin --> Authentication`
+2. Enable the ldap+ groups plugin and select :guilabel:`Save`
+3. Select the :guilabel:`Enabled` check box in the plugin configuration section
+4. Add the required LDAP information and :guilabel:`Save`, for more details,
+   see :ref:`config-ldap-groups-examples`
+
+For a more detailed description of LDAP objects, see :ref:`ldap-gloss-ref`:
+
+.. _config-ldap-groups-examples:
+
+Example LDAP configuration
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. code-block:: bash
+
+        # Auth Cache TTL, Defines the caching for authentication to offload LDAP server.
+        # This means that cache result will be saved for 3600 before contacting LDAP server to verify the user access
+        3600
+        # Host, comma seperated format is optionally possible to specify more than 1 server
+        https://ldap1.server.com/ldap-admin/,https://ldap2.server.com/ldap-admin/
+        # Default LDAP Port, use 689 for LDAPS
+        389
+        # Account, used for SimpleBind if LDAP server requires an authentication
+        e.g admin@server.com
+        # Password used for simple bind
+        ldap-user-password
+        # LDAP connection security
+        LDAPS
+        # Certificate checks level
+        DEMAND
+        # Base DN
+        cn=Rufus Magillacuddy,ou=users,dc=rhodecode,dc=com
+        # User Search Base
+        ou=groups,ou=users
+        # LDAP search filter to narrow the results
+        (objectClass=person)
+        # LDAP search scope
+        SUBTREE
+        # Login attribute
+        sAMAccountName
+        # First Name Attribute to read
+        givenName
+        # Last Name Attribute to read
+        sn
+        # Email Attribute to read email address from
+        mail
+        # group extraction method
+        rfc2307bis
+        # Group search base
+        ou=RC-Groups
+        # Group Name Attribute, field to read the group name from
+        sAMAAccountName
+        # User Member of Attribute, field in which groups are stored
+        memberOf
+        # LDAP Group Search Filter, allows narrowing the results
+
+        # Admin Groups. Comma separated list of groups. If user is member of
+        # any of those he will be marked as super-admin in RhodeCode
+        admins, management
+
+
+Below is example setup that can be used with Active Directory and ldap groups.
+
+.. image:: ../images/ldap-groups-example.png
+   :alt: LDAP/AD setup example
+   :scale: 50 %
+
+.. toctree::
+
+   ldap-active-directory
+   ldap-authentication

docs/auth/auth-ldap.rst

@@ -1,14 +1,17 @@
 .. _config-ldap-ref:
 
-LDAP
-----
+LDAP/AD
+-------
 
 |RCM| supports LDAP (Lightweight Directory Access Protocol) or
 AD (active Directory) authentication.
 All LDAP versions are supported, with the following |RCM| plugins managing each:
 
-* For LDAPv3 use ``LDAP (egg:rhodecode-enterprise-ce#ldap)``
-* For LDAPv3 with user group sync use ``LDAP + User Groups (egg:rhodecode-enterprise-ee#ldap_group)``
+* For LDAP or Active Directory use ``LDAP (egg:rhodecode-enterprise-ce#ldap)``
+
+RhodeCode reads all data defined from plugin and creates corresponding
+accounts on local database after receiving data from LDAP. This is done on
+every user log-in including operations like pushing/pulling/checkout.
 
 
 .. important::
@@ -21,6 +24,7 @@ All LDAP versions are supported, with the following |RCM| plugins managing each:
    Likewise, if as an admin you also have a user account, the email address
    attached to the user account needs to be different.
 
+
 LDAP Configuration Steps
 ^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -28,7 +32,7 @@ To configure |LDAP|, use the following steps:
 
 1. From the |RCM| interface, select
    :menuselection:`Admin --> Authentication`
-2. Enable the required plugin and select :guilabel:`Save`
+2. Enable the ldap plugin and select :guilabel:`Save`
 3. Select the :guilabel:`Enabled` check box in the plugin configuration section
 4. Add the required LDAP information and :guilabel:`Save`, for more details,
    see :ref:`config-ldap-examples`
@@ -41,15 +45,16 @@ Example LDAP configuration
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 .. code-block:: bash
 
-        # Auth Cache TTL
+        # Auth Cache TTL, Defines the caching for authentication to offload LDAP server.
+        # This means that cache result will be saved for 3600 before contacting LDAP server to verify the user access
         3600
-        # Host
+        # Host, comma seperated format is optionally possible to specify more than 1 server
         https://ldap1.server.com/ldap-admin/,https://ldap2.server.com/ldap-admin/
-        # Port
+        # Default LDAP Port, use 689 for LDAPS
         389
-        # Account
-        cn=admin,dc=rhodecode,dc=com
-        # Password
+        # Account, used for SimpleBind if LDAP server requires an authentication
+        e.g admin@server.com
+        # Password used for simple bind
         ldap-user-password
         # LDAP connection security
         LDAPS
@@ -57,32 +62,26 @@ Example LDAP configuration
         DEMAND
         # Base DN
         cn=Rufus Magillacuddy,ou=users,dc=rhodecode,dc=com
-        # User Search Base
-        ou=groups,ou=users
-        # LDAP search filter
+        # LDAP search filter to narrow the results
         (objectClass=person)
         # LDAP search scope
         SUBTREE
         # Login attribute
-        rmagillacuddy
-        # First Name Attribute
-        Rufus
-        # Last Name Attribute
-        Magillacuddy
-        # Email Attribute
-        LDAP-Registered@email.ac
-        # User Member of Attribute
-        Organizational Role
-        # Group search base
-        cn=users,ou=groups,dc=rhodecode,dc=com
-        # LDAP Group Search Filter
-        (objectclass=posixGroup)
-        # Group Name Attribute
-        users
-        # Group Member Of Attribute
-        cn
-        # Admin Groups
-        admin,devops,qa
+        sAMAccountName
+        # First Name Attribute to read
+        givenName
+        # Last Name Attribute to read
+        sn
+        # Email Attribute to read email address from
+        mail
+
+
+Below is example setup that can be used with Active Directory/LDAP server.
+
+.. image:: ../images/ldap-example.png
+   :alt: LDAP/AD setup example
+   :scale: 50 %
+
 
 .. toctree::
 

docs/auth/auth.rst

@@ -3,35 +3,30 @@
 Authentication Options
 ======================
 
-|RCE| provides a built in authentication plugin
-``rhodecode.lib.auth_rhodecode``. This is enabled by default and accessed
-through the administrative interface. Additionally,
-|RCE| provides a Pluggable Authentication System (PAS). This gives the
+|RCE| provides a built in authentication against its own database. This is
+implemented using ``rhodecode.lib.auth_rhodecode`` plugin. This plugin is
+enabled by default.
+Additionally, |RCE| provides a Pluggable Authentication System. This gives the
 administrator greater control over how users authenticate with the system.
 
 .. important::
 
   You can disable the built in |RCM| authentication plugin
   ``rhodecode.lib.auth_rhodecode`` and force all authentication to go
-  through your authentication plugin. However, if you do this,
-  and your external authentication tools fails, you will be unable to
-  access |RCM|.
+  through your authentication plugin of choice e.g LDAP only.
+  However, if you do this, and your external authentication tools fails,
+  you will be unable to access |RCM|.
 
 |RCM| comes with the following user authentication management plugins:
 
-.. only:: latex
-
-    * :ref:`config-ldap-ref`
-    * :ref:`config-pam-ref`
-    * :ref:`config-crowd-ref`
-    * :ref:`config-token-ref`
 
 .. toctree::
 
-    ldap-config-steps
-    crowd-auth
-    pam-auth
-    token-auth
+    auth-ldap
+    auth-ldap-groups
+    auth-crowd
+    auth-pam
+    auth-token
     ssh-connection
 
 

docs/auth/ldap-configuration-example.rst

@@ -1,13 +0,0 @@
-    LDAP Host hostname1,hostname2 # testing here
-    Port port
-    Account: `uid=admin,cn=users,cn=accounts,dc=localdomain,dc=tld`
-    Password: userpassword #Testing this here
-    Connection Security LDAPS
-    Certificate Checks ``NEVER``
-    Base DN cn=users,cn=accounts,dc=localdomain,dc=tld
-    LDAP Search Filter (objectClass=person)
-    LDAP Search Scope SUBTREE
-    Login Attribute uid
-    First Name Attribute givenname
-    Last Name Attribute sn
-    Email Attribute mail

docs/conf.py

@@ -34,7 +34,7 @@ import common
 extensions = [
     'sphinx.ext.intersphinx',
     'sphinx.ext.todo',
-    'sphinx.ext.pngmath'
+    'sphinx.ext.imgmath'
 ]
 
 intersphinx_mapping = {
@@ -104,7 +104,6 @@ exclude_patterns = [
 
     # Other RST files
     'admin/rhodecode-backup.rst',
-    'auth/ldap-configuration-example.rst',
     'issue-trackers/redmine.rst',
     'known-issues/error-msg-guide.rst',
     'tutorials/docs-build.rst',

docs/contributing/dev-setup.rst

@@ -34,6 +34,13 @@ following commands:
 
    Update your channels frequently by running ``nix-channel --update``.
 
+.. note::
+
+   To uninstall nix run the following:
+
+    remove the . "$HOME/.nix-profile/etc/profile.d/nix.sh" line in your ~/.profile or ~/.bash_profile
+    rm -rf $HOME/{.nix-channels,.nix-defexpr,.nix-profile,.config/nixpkgs}
+    sudo rm -rf /nix
 
 Switch nix to the latest STABLE channel
 ---------------------------------------

docs/default.nix

@@ -169,17 +169,6 @@ let
     };
   };
 
-  setuptools = buildPythonPackage {
-    name = "setuptools-36.6.0";
-    buildInputs = [];
-    doCheck = false;
-    propagatedBuildInputs = [];
-    src = fetchurl {
-      url = "https://pypi.python.org/packages/45/29/8814bf414e7cd1031e1a3c8a4169218376e284ea2553cc0822a6ea1c2d78/setuptools-36.6.0.zip";
-      md5 = "74663b15117d9a2cc5295d76011e6fd1";
-    };
-  };
-
   six = buildPythonPackage {
     name = "six-1.11.0";
     buildInputs = [];
@@ -259,6 +248,9 @@ let
 
 
   };
+  # Avoid that setuptools is replaced, this leads to trouble
+  # with buildPythonPackage.
+  setuptools = pkgs.python27Packages.setuptools;
 
 in python.buildEnv.override {
   inherit python;

docs/release-notes/release-notes-4.12.0.rst

@@ -0,0 +1,139 @@
+|RCE| 4.12.0 |RNS|
+------------------
+
+Release Date
+^^^^^^^^^^^^
+
+- 2018-04-24
+
+
+New Features
+^^^^^^^^^^^^
+
+- Svn: added support for RhodeCode integration framework. All integrations like
+  slack, email, Jenkins now also fully work for SVN.
+- Integrations: added new dedicated Jenkins integration with the support of
+  CSRF authentication. Available in EE edition only.
+- Automation: added new bi-directional remote sync. RhodeCode instances can now
+  automatically push or pull from/to remote locations. This feature is powered
+  by the Scheduler of 4.11 release, and it is required to be enabled for this feature to work.
+  Available in EE edition only.
+- Mercurial: path-based permissions. RhodeCode can now use Mercurials narrowhg
+  to implement path-based permissions. All permissions are read from .hg/hgacl.
+  Thanks to the great contribution from Sandu Turcan.
+- VCS: added new diff caches. Available as an option under vcs settings.
+  Diff caches work on pull-request, or individual commits for greater
+  performance and reduced memory usage. This feature increases speed of large
+  pull requests significantly. In addition for pull requests it will allow
+  showing old closed pull requests even if commits from source were removed,
+  further enhancing auditing capabilities.
+- Audit: added few new audit log entries especially around changing permissions.
+- LDAP: added connection pinning and timeout option to ldap plugin. This should
+  prevent problems when connection to LDAP is not stable causing RhodeCode
+  instances to freeze waiting on LDAP connections.
+- User groups: expose public user group profiles. Allows to see members of a user
+  groups by other team members, if they have proper permissions.
+- UI: show pull request page in quick nav menu on my account for quicker access.
+- UI: hidden/outdated comments now have visible markers next to line numbers.
+  This allows access to them without showing all hidden comments.
+
+
+General
+^^^^^^^
+
+- Ssh: show conflicting fingerprint when adding an already existing key.
+  Helps to track why adding a key failed.
+- System info: added ulimit to system info. This is causing lots of problems
+  when we hit any of those limits, that is why it's important to show this.
+- Repository settings: add hidden view to force re-install hooks.
+  Available under /{repo_name}/settings/advanced/hooks
+- Integrations: Webhook now handles response errors and show response for
+  easier debugging.
+- Cli: speed up CLI execution start by skipping auth plugin search/registry.
+- SVN: added an example in the docs on how to enable path-based permissions.
+- LDAP: enable connection recycling on LDAP plugin.
+- Auth plugins: use a nicer visual display of auth plugins that would
+  highlight that order of enabled plugins does matter.
+- Events: expose shadow repo build url.
+- Events: expose pull request title and uid in event data.
+- API: enable setting sync flag for user groups on create/edit.
+- API: update pull method with a possible specification of the url
+- Logging: improved consistency of auth plugins logs.
+- Logging: improved log for ssl required
+- Dependencies: bumped mercurial to 4.4 series
+- Dependencies: bumped zope.cachedescriptors==4.3.1
+- Dependencies: bumped zope.deprecation==4.3.0
+- Dependencies: bumped zope.event==4.3.0
+- Dependencies: bumped zope.interface==4.4.3
+- Dependencies: bumped graphviz 0.8.2
+- Dependencies: bumped to ipaddress 0.1.19
+- Dependencies: bumped pyexpect to 4.3.1
+- Dependencies: bumped ws4py to 0.4.3
+- Dependencies: bumped bleach to 2.1.2
+- Dependencies: bumped html5lib 1.0.1
+- Dependencies: bumped greenlet to 0.4.13
+- Dependencies: bumped markdown to 2.6.11
+- Dependencies: bumped psutil to 5.4.3
+- Dependencies: bumped beaker to 1.9.1
+- Dependencies: bumped alembic to 0.6.8 release.
+- Dependencies: bumped supervisor to 3.3.4
+- Dependencies: bumped pyexpect to 4.4.0 and scandir to 1.7
+- Dependencies: bumped appenlight client to 0.6.25
+- Dependencies: don't require full mysql lib for the db driver.
+  Reduces installation package size by around 100MB.
+
+
+Security
+^^^^^^^^
+
+- My account: changing email in my account now requires providing user
+  access password. This is a case for only RhodeCode built-in accounts.
+  Prevents adding recovery email by unauthorized users who gain
+  access to logged in session of user.
+- Logging: fix leaking of tokens to logging.
+- General: serialize the repo name in repo checks to prevent potential
+  html injections by providing a malformed url.
+
+
+Performance
+^^^^^^^^^^^
+
+- Diffs: don't use recurred diffset attachment in diffs. This makes
+this structure much harder to garbage collect. Reduces memory usage.
+- Diff cache: added caching for better performance of large pull requests.
+
+
+Fixes
+^^^^^
+
+- Age helper: fix issues with proper timezone detection for certain timezones.
+  Fixes wrong age display in few cases.
+- API: added audit logs for user group related calls that were
+  accidentally missing.
+- Diffs: fix and improve line selections and anchor links.
+- Pull requests: fixed cases with default expected refs are closed or unavailable.
+  For Mercurial with closed default branch a compare across forks could fail.
+- Core: properly report 502 errors for gevent and gunicorn.
+  Gevent wtih Gunicorn doesn't raise normal pycurl errors.
+- Auth plugins: fixed problem with cache of settings in multi-worker mode.
+  The previous implementation had a bug that cached the settings in each class,
+  caused not refreshing the update of settings in multi-worker mode.
+  Only restart of RhodeCode loaded new settings.
+- Audit logs: properly handle query syntax in the search field.
+- Repositories: better handling of missing requirements errors for repositories.
+- API: fixed problems with repository fork/create using celery backend.
+- VCS settings: added missing flash message on validation errors to prevent
+  missing out some field input validation problems.
+
+
+Upgrade notes
+^^^^^^^^^^^^^
+
+- This release adds support for SVN hook. This required lots of changes on how we
+handle SVN protocol. We did thoughtful tests for SVN compatibility.
+Please be advised to check the behaviour of SVN repositories during this update.
+
+- Diff caches are turned off by default for backward compatibility. We however recommend
+turning them on either individually for bigger repositories or globally for every repository.
+This setting can be found in admin > settings > vcs, or repository > settings > vcs
+

docs/release-notes/release-notes.rst

@@ -9,6 +9,7 @@ Release Notes
 .. toctree::
    :maxdepth: 1
 
+   release-notes-4.12.0.rst
    release-notes-4.11.6.rst
    release-notes-4.11.5.rst
    release-notes-4.11.4.rst

docs/tutorials/deploy-from-host.rst

@@ -88,7 +88,7 @@ Install |RCC|
 
 The |RCC| installer is now on your server, and you can read the full
 instructions here
-:ref:`Install RhodeCode Control <control:rcc-install-ref>`,
+:ref:`Install RhodeCode Control <control:rcc-linux-ref>` ,
 but below is the example shortcut.
 
 .. code-block:: bash

pkgs/patch-beaker-lock-func-debug.diff

@@ -0,0 +1,20 @@
+diff -rup Beaker-1.9.1-orig/beaker/container.py Beaker-1.9.1/beaker/container.py
+--- Beaker-1.9.1-orig/beaker/container.py	2018-04-10 10:23:04.000000000 +0200
++++ Beaker-1.9.1/beaker/container.py	2018-04-10 10:23:34.000000000 +0200
+@@ -353,13 +353,13 @@ class Value(object):
+                 debug("get_value returning old value while new one is created")
+                 return value
+             else:
+-                debug("lock_creatfunc (didnt wait)")
++                debug("lock_creatfunc `%s` (didnt wait)", self.createfunc.__name__)
+                 has_createlock = True
+
+         if not has_createlock:
+-            debug("lock_createfunc (waiting)")
++            debug("lock_createfunc `%s` (waiting)", self.createfunc.__name__)
+             creation_lock.acquire()
+-            debug("lock_createfunc (waited)")
++            debug("lock_createfunc `%s` (waited)", self.createfunc.__name__)
+
+         try:
+             # see if someone created the value already

pkgs/python-packages-overrides.nix

@@ -25,6 +25,12 @@ self: super: {
     };
   });
 
+  Beaker = super.Beaker.override (attrs: {
+    patches = [
+      ./patch-beaker-lock-func-debug.diff
+    ];
+  });
+
   future = super.future.override (attrs: {
     meta = {
       license = [ pkgs.lib.licenses.mit ];
@@ -82,7 +88,7 @@ self: super: {
       pkgs.openssl
     ];
     propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
-      pkgs.mysql.lib
+      pkgs.libmysql
       pkgs.zlib
     ];
   });

pkgs/python-packages.nix

@@ -16,13 +16,13 @@
     };
   };
   Beaker = super.buildPythonPackage {
-    name = "Beaker-1.9.0";
+    name = "Beaker-1.9.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [funcsigs];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/93/b2/12de6937b06e9615dbb3cb3a1c9af17f133f435bdef59f4ad42032b6eb49/Beaker-1.9.0.tar.gz";
-      md5 = "38b3fcdfa24faf97c6cf66991eb54e9c";
+      url = "https://pypi.python.org/packages/ca/14/a626188d0d0c7b55dd7cf1902046c2743bd392a7078bb53073e13280eb1e/Beaker-1.9.1.tar.gz";
+      md5 = "46fda0a164e2b0d24ccbda51a2310301";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
@@ -94,13 +94,13 @@
     };
   };
   Markdown = super.buildPythonPackage {
-    name = "Markdown-2.6.9";
+    name = "Markdown-2.6.11";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/29/82/dfe242bcfd9eec0e7bf93a80a8f8d8515a95b980c44f5c0b45606397a423/Markdown-2.6.9.tar.gz";
-      md5 = "56547d362a9abcf30955b8950b08b5e3";
+      url = "https://pypi.python.org/packages/b3/73/fc5c850f44af5889192dff783b7b0d8f3fe8d30b65c8e3f78f8f0265fecf/Markdown-2.6.11.tar.gz";
+      md5 = "a67c1b2914f7d74eeede2ebe0fdae470";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
@@ -315,13 +315,13 @@
     };
   };
   alembic = super.buildPythonPackage {
-    name = "alembic-0.9.6";
+    name = "alembic-0.9.8";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [SQLAlchemy Mako python-editor python-dateutil];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/bf/b3/b28ea715824f8455635ece3c12f59d5d205f98cc378858e414e3aa6ebdbc/alembic-0.9.6.tar.gz";
-      md5 = "fcb096bccc87c8770bd07a04606cb989";
+      url = "https://pypi.python.org/packages/a1/95/2252783859df9ec76b9a25d968c2827ed75a43ba34c6e8d38f87a5c0fb26/alembic-0.9.8.tar.gz";
+      md5 = "5cfef58641c9a94d4a5d547e951a7dda";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
@@ -341,13 +341,13 @@
     };
   };
   appenlight-client = super.buildPythonPackage {
-    name = "appenlight-client-0.6.22";
+    name = "appenlight-client-0.6.25";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [WebOb requests six];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/73/37/0a64460fa9670b17c061adc433bc8be5079cba21e8b3a92d824adccb12bc/appenlight_client-0.6.22.tar.gz";
-      md5 = "641afc114a9a3b3af4f75b11c70968ee";
+      url = "https://pypi.python.org/packages/fa/44/2911ef85ea4f4fe65058fd22959d8dad598fab6a3c84e5bcb569d15c8783/appenlight_client-0.6.25.tar.gz";
+      md5 = "76dd2f9d42659fae8f290982078dc880";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
@@ -406,13 +406,13 @@
     };
   };
   bleach = super.buildPythonPackage {
-    name = "bleach-2.1.1";
+    name = "bleach-2.1.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [six html5lib];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/d4/3f/d517089af35b01bb9bc4eac5ea04bae342b37a5e9abbb27b7c3ce0eae070/bleach-2.1.1.tar.gz";
-      md5 = "7c5dfb1d66ea979b5a465afb12c82ec4";
+      url = "https://pypi.python.org/packages/b3/5f/0da670d30d3ffbc57cc97fa82947f81bbe3eab8d441e2d42e661f215baf2/bleach-2.1.2.tar.gz";
+      md5 = "d0b14ae43a437ee0c650e04c6063eedd";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
@@ -710,8 +710,8 @@
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/5e/1a/0aa2c8195a204a9f51284018562dea77e25511f02fe924fac202fc012172/functools32-3.2.3-2.zip";
-      md5 = "d55232eb132ec779e6893c902a0bc5ad";
+      url = "https://pypi.python.org/packages/c5/60/6ac26ad05857c601308d8fb9e87fa36d0ebf889423f47c3502ef034365db/functools32-3.2.3-2.tar.gz";
+      md5 = "09f24ffd9af9f6cd0f63cb9f4e23d4b2";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl ];
@@ -783,26 +783,26 @@
     };
   };
   graphviz = super.buildPythonPackage {
-    name = "graphviz-0.8.1";
+    name = "graphviz-0.8.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/a9/a6/ee6721349489a2da6eedd3dba124f2b5ac15ee1e0a7bd4d3cfdc4fff0327/graphviz-0.8.1.zip";
-      md5 = "88d8efa88c02a735b3659fe0feaf0b96";
+      url = "https://pypi.python.org/packages/fa/d1/63b62dee9e55368f60b5ea445e6afb361bb47e692fc27553f3672e16efb8/graphviz-0.8.2.zip";
+      md5 = "50866e780f43e1cb0d073c70424fcaff";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   greenlet = super.buildPythonPackage {
-    name = "greenlet-0.4.12";
+    name = "greenlet-0.4.13";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/be/76/82af375d98724054b7e273b5d9369346937324f9bcc20980b45b068ef0b0/greenlet-0.4.12.tar.gz";
-      md5 = "e8637647d58a26c4a1f51ca393e53c00";
+      url = "https://pypi.python.org/packages/13/de/ba92335e9e76040ca7274224942282a80d54f85e342a5e33c5277c7f87eb/greenlet-0.4.13.tar.gz";
+      md5 = "6e0b9dd5385f81d478451ec8ed1d62b3";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
@@ -822,13 +822,13 @@
     };
   };
   html5lib = super.buildPythonPackage {
-    name = "html5lib-1.0b10";
+    name = "html5lib-1.0.1";
     buildInputs = with self; [];
     doCheck = false;
-    propagatedBuildInputs = with self; [six webencodings setuptools];
+    propagatedBuildInputs = with self; [six webencodings];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/97/16/982214624095c1420c75f3bd295d9e658794aafb95fc075823de107e0ae4/html5lib-1.0b10.tar.gz";
-      md5 = "5ada1243b7a863624b2f35245b2186e9";
+      url = "https://pypi.python.org/packages/85/3e/cf449cf1b5004e87510b9368e7a5f1acd8831c2d6691edd3c62a0823f98f/html5lib-1.0.1.tar.gz";
+      md5 = "942a0688d6bdf20d087c9805c40182ad";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
@@ -874,13 +874,13 @@
     };
   };
   ipaddress = super.buildPythonPackage {
-    name = "ipaddress-1.0.18";
+    name = "ipaddress-1.0.19";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/4e/13/774faf38b445d0b3a844b65747175b2e0500164b7c28d78e34987a5bfe06/ipaddress-1.0.18.tar.gz";
-      md5 = "310c2dfd64eb6f0df44aa8c59f2334a7";
+      url = "https://pypi.python.org/packages/f0/ba/860a4a3e283456d6b7e2ab39ce5cf11a3490ee1a363652ac50abf9f0f5df/ipaddress-1.0.19.tar.gz";
+      md5 = "d0687efaf93a32476d81e90ba0609c57";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl ];
@@ -1048,8 +1048,8 @@
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/15/45/30273ee91feb60dabb8fbb2da7868520525f02cf910279b3047182feed80/mock-1.0.1.zip";
-      md5 = "869f08d003c289a97c1a6610faf5e913";
+      url = "https://pypi.python.org/packages/a2/52/7edcd94f0afb721a2d559a5b9aae8af4f8f2c79bc63fdbe8a8a6c9b23bbe/mock-1.0.1.tar.gz";
+      md5 = "c3971991738caa55ec7c356bbc154ee2";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
@@ -1160,13 +1160,13 @@
     };
   };
   pexpect = super.buildPythonPackage {
-    name = "pexpect-4.3.0";
+    name = "pexpect-4.4.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [ptyprocess];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/f8/44/5466c30e49762bb92e442bbdf4472d6904608d211258eb3198a11f0309a4/pexpect-4.3.0.tar.gz";
-      md5 = "047a486dcd26134b74f2e67046bb61a0";
+      url = "https://pypi.python.org/packages/fa/c3/60c0cbf96f242d0b47a82e9ca634dcd6dcb043832cf05e17540812e1c707/pexpect-4.4.0.tar.gz";
+      md5 = "e9b07f0765df8245ac72201d757baaef";
     };
     meta = {
       license = [ pkgs.lib.licenses.isc { fullName = "ISC License (ISCL)"; } ];
@@ -1225,13 +1225,13 @@
     };
   };
   psutil = super.buildPythonPackage {
-    name = "psutil-5.4.0";
+    name = "psutil-5.4.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/8d/96/1fc6468be91521192861966c40bd73fdf8b065eae6d82dd0f870b9825a65/psutil-5.4.0.tar.gz";
-      md5 = "01af6219b1e8fcfd53603023967713bf";
+      url = "https://pypi.python.org/packages/e2/e1/600326635f97fee89bf8426fef14c5c29f4849c79f68fd79f433d8c1bd96/psutil-5.4.3.tar.gz";
+      md5 = "3b291833dbea631db9d271aa602a169a";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
@@ -1360,8 +1360,8 @@
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/2e/26/e8fb5b4256a5f5036be7ce115ef8db8d06bc537becfbdc46c6af008314ee/pyparsing-1.5.7.zip";
-      md5 = "b86854857a368d6ccb4d5b6e76d0637f";
+      url = "https://pypi.python.org/packages/6f/2c/47457771c02a8ff0f302b695e094ec309e30452232bd79198ee94fda689f/pyparsing-1.5.7.tar.gz";
+      md5 = "9be0fcdcc595199c646ab317c1d9a709";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
@@ -1602,13 +1602,13 @@
     };
   };
   pytz = super.buildPythonPackage {
-    name = "pytz-2017.3";
+    name = "pytz-2018.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/60/88/d3152c234da4b2a1f7a989f89609ea488225eaea015bc16fbde2b3fdfefa/pytz-2017.3.zip";
-      md5 = "7006b56c0d68a162d9fe57d4249c3171";
+      url = "https://pypi.python.org/packages/1b/50/4cdc62fc0753595fc16c8f722a89740f487c6e5670c644eb8983946777be/pytz-2018.3.tar.gz";
+      md5 = "abb07c09c79f78d7c04f222a550c99ef";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
@@ -1680,10 +1680,10 @@
     };
   };
   rhodecode-enterprise-ce = super.buildPythonPackage {
-    name = "rhodecode-enterprise-ce-4.11.6";
+    name = "rhodecode-enterprise-ce-4.12.0";
     buildInputs = with self; [pytest py pytest-cov pytest-sugar pytest-runner pytest-catchlog pytest-profiling gprof2dot pytest-timeout mock WebTest cov-core coverage configobj];
     doCheck = true;
-    propagatedBuildInputs = with self; [setuptools-scm amqp authomatic Babel Beaker celery Chameleon channelstream click colander configobj cssselect decorator deform docutils dogpile.cache dogpile.core ecdsa FormEncode future futures gnureadline infrae.cache iso8601 itsdangerous Jinja2 billiard kombu lxml Mako Markdown MarkupSafe msgpack-python MySQL-python objgraph packaging Paste PasteDeploy PasteScript pathlib2 peppercorn psutil psycopg2 py-bcrypt pycrypto pycurl pyflakes pygments-markdown-lexer Pygments pyparsing pyramid-beaker pyramid-debugtoolbar pyramid-jinja2 pyramid-mako pyramid pysqlite python-dateutil python-ldap python-memcached python-pam pytz pyzmq py-gfm recaptcha-client redis repoze.lru requests Routes setproctitle simplejson six SQLAlchemy sshpubkeys subprocess32 supervisor Tempita translationstring trollius urllib3 URLObject venusian WebError WebHelpers2 WebHelpers WebOb Whoosh wsgiref zope.cachedescriptors zope.deprecation zope.event zope.interface nbconvert bleach nbformat jupyter-client alembic invoke bumpversion transifex-client gevent greenlet gunicorn waitress uWSGI ipdb ipython CProfileV bottle rhodecode-tools appenlight-client pytest py pytest-cov pytest-sugar pytest-runner pytest-catchlog pytest-profiling gprof2dot pytest-timeout mock WebTest cov-core coverage];
+    propagatedBuildInputs = with self; [setuptools-scm amqp authomatic Babel Beaker celery Chameleon channelstream click colander configobj cssselect decorator deform docutils dogpile.cache dogpile.core ecdsa FormEncode future futures gnureadline infrae.cache iso8601 itsdangerous Jinja2 billiard kombu lxml Mako Markdown MarkupSafe msgpack-python MySQL-python objgraph packaging Paste PasteDeploy PasteScript pathlib2 peppercorn psutil psycopg2 py-bcrypt pycrypto pycurl pyflakes pygments-markdown-lexer Pygments pyparsing pyramid-beaker pyramid-debugtoolbar pyramid-jinja2 pyramid-mako pyramid pysqlite python-dateutil python-ldap python-memcached python-pam pytz tzlocal pyzmq py-gfm recaptcha-client redis repoze.lru requests Routes setproctitle simplejson six SQLAlchemy sshpubkeys subprocess32 supervisor Tempita translationstring trollius urllib3 URLObject venusian WebError WebHelpers2 WebHelpers WebOb Whoosh wsgiref zope.cachedescriptors zope.deprecation zope.event zope.interface nbconvert bleach nbformat jupyter-client alembic invoke bumpversion transifex-client gevent greenlet gunicorn waitress uWSGI ipdb ipython CProfileV bottle rhodecode-tools appenlight-client pytest py pytest-cov pytest-sugar pytest-runner pytest-catchlog pytest-profiling gprof2dot pytest-timeout mock WebTest cov-core coverage];
     src = ./.;
     meta = {
       license = [ { fullName = "Affero GNU General Public License v3 or later (AGPLv3+)"; } { fullName = "AGPLv3, and Commercial License"; } ];
@@ -1703,13 +1703,13 @@
     };
   };
   scandir = super.buildPythonPackage {
-    name = "scandir-1.6";
+    name = "scandir-1.7";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/77/3f/916f524f50ee65e3f465a280d2851bd63685250fddb3020c212b3977664d/scandir-1.6.tar.gz";
-      md5 = "0180ddb97c96cbb2d4f25d2ae11c64ac";
+      url = "https://pypi.python.org/packages/13/bb/e541b74230bbf7a20a3949a2ee6631be299378a784f5445aa5d0047c192b/scandir-1.7.tar.gz";
+      md5 = "037e5f24d1a0e78b17faca72dea9555f";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal { fullName = "New BSD License"; } ];
@@ -1820,13 +1820,13 @@
     };
   };
   supervisor = super.buildPythonPackage {
-    name = "supervisor-3.3.3";
+    name = "supervisor-3.3.4";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [meld3];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/31/7e/788fc6566211e77c395ea272058eb71299c65cc5e55b6214d479c6c2ec9a/supervisor-3.3.3.tar.gz";
-      md5 = "0fe86dfec4e5c5d98324d24c4cf944bd";
+      url = "https://pypi.python.org/packages/44/60/698e54b4a4a9b956b2d709b4b7b676119c833d811d53ee2500f1b5e96dc3/supervisor-3.3.4.tar.gz";
+      md5 = "f1814d71d820ddfa8c86d46a72314cec";
     };
     meta = {
       license = [ { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
@@ -1910,6 +1910,19 @@
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
+  tzlocal = super.buildPythonPackage {
+    name = "tzlocal-1.5.1";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [pytz];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/cb/89/e3687d3ed99bc882793f82634e9824e62499fdfdc4b1ae39e211c5b05017/tzlocal-1.5.1.tar.gz";
+      md5 = "4553be891efa0812c4adfb0c6e818eec";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.mit ];
+    };
+  };
   uWSGI = super.buildPythonPackage {
     name = "uWSGI-2.0.15";
     buildInputs = with self; [];
@@ -2002,13 +2015,13 @@
     };
   };
   ws4py = super.buildPythonPackage {
-    name = "ws4py-0.4.2";
+    name = "ws4py-0.4.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/b8/98/a90f1d96ffcb15dfc220af524ce23e0a5881258dafa197673357ce1683dd/ws4py-0.4.2.tar.gz";
-      md5 = "f0603ae376707a58d205bd87a67758a2";
+      url = "https://pypi.python.org/packages/fa/a1/33c43a4304ac3b4dc81deb93cbd329de9297dd034d75c47cce64fda806bc/ws4py-0.4.3.tar.gz";
+      md5 = "d5834cf7d3965bb0da31bbb02bd8513a";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
@@ -2028,52 +2041,52 @@
     };
   };
   zope.cachedescriptors = super.buildPythonPackage {
-    name = "zope.cachedescriptors-4.0.0";
+    name = "zope.cachedescriptors-4.3.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [setuptools];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/40/33/694b6644c37f28553f4b9f20b3c3a20fb709a22574dff20b5bdffb09ecd5/zope.cachedescriptors-4.0.0.tar.gz";
-      md5 = "8d308de8c936792c8e758058fcb7d0f0";
+      url = "https://pypi.python.org/packages/2f/89/ebe1890cc6d3291ebc935558fa764d5fffe571018dbbee200e9db78762cb/zope.cachedescriptors-4.3.1.tar.gz";
+      md5 = "42f3693f43bc93f3b1eb86940f58acf3";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpt21 ];
     };
   };
   zope.deprecation = super.buildPythonPackage {
-    name = "zope.deprecation-4.1.2";
+    name = "zope.deprecation-4.3.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [setuptools];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/c1/d3/3919492d5e57d8dd01b36f30b34fc8404a30577392b1eb817c303499ad20/zope.deprecation-4.1.2.tar.gz";
-      md5 = "e9a663ded58f4f9f7881beb56cae2782";
+      url = "https://pypi.python.org/packages/a1/18/2dc5e6bfe64fdc3b79411b67464c55bb0b43b127051a20f7f492ab767758/zope.deprecation-4.3.0.tar.gz";
+      md5 = "2166b2cb7e0e96a21104e6f8f9b696bb";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpt21 ];
     };
   };
   zope.event = super.buildPythonPackage {
-    name = "zope.event-4.0.3";
+    name = "zope.event-4.3.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [setuptools];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/c1/29/91ba884d7d6d96691df592e9e9c2bfa57a47040ec1ff47eff18c85137152/zope.event-4.0.3.tar.gz";
-      md5 = "9a3780916332b18b8b85f522bcc3e249";
+      url = "https://pypi.python.org/packages/9e/d0/54ba59f19a0635f6591b74be259cf6fbf67e73f4edda27b5cd0cf4d26efa/zope.event-4.3.0.tar.gz";
+      md5 = "8ca737960741c6fd112972f3313303bd";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpt21 ];
     };
   };
   zope.interface = super.buildPythonPackage {
-    name = "zope.interface-4.1.3";
+    name = "zope.interface-4.4.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [setuptools];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/9d/81/2509ca3c6f59080123c1a8a97125eb48414022618cec0e64eb1313727bfe/zope.interface-4.1.3.tar.gz";
-      md5 = "9ae3d24c0c7415deb249dd1a132f0f79";
+      url = "https://pypi.python.org/packages/bd/d2/25349ed41f9dcff7b3baf87bd88a4c82396cf6e02f1f42bb68657a3132af/zope.interface-4.4.3.tar.gz";
+      md5 = "8700a4f527c1203b34b10c2b4e7a6912";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpt21 ];

requirements.txt

@@ -5,7 +5,7 @@ setuptools-scm==1.15.6
 amqp==2.2.2
 authomatic==0.1.0.post1
 Babel==1.3
-Beaker==1.9.0
+Beaker==1.9.1
 celery==4.1.0
 Chameleon==2.24
 channelstream==0.5.2
@@ -31,7 +31,7 @@ billiard==3.5.0.3
 kombu==4.1.0
 lxml==3.7.3
 Mako==1.0.7
-Markdown==2.6.9
+Markdown==2.6.11
 MarkupSafe==1.0.0
 msgpack-python==0.4.8
 MySQL-python==1.2.5
@@ -42,7 +42,7 @@ PasteDeploy==1.5.2
 PasteScript==2.0.2
 pathlib2==2.3.0
 peppercorn==0.5
-psutil==5.4.0
+psutil==5.4.3
 psycopg2==2.7.3.2
 py-bcrypt==0.4
 pycrypto==2.6.1
@@ -61,7 +61,8 @@ python-dateutil
 python-ldap==2.4.45
 python-memcached==1.58
 python-pam==1.8.2
-pytz==2017.3
+pytz==2018.3
+tzlocal==1.5.1
 pyzmq==14.6.0
 py-gfm==0.1.3
 recaptcha-client==1.0.6
@@ -75,7 +76,7 @@ six==1.11.0
 SQLAlchemy==1.1.15
 sshpubkeys==2.2.0
 subprocess32==3.2.7
-supervisor==3.3.3
+supervisor==3.3.4
 Tempita==0.5.2
 translationstring==1.3
 trollius==1.0.4
@@ -88,29 +89,29 @@ WebHelpers==1.3
 WebOb==1.7.4
 Whoosh==2.7.4
 wsgiref==0.1.2
-zope.cachedescriptors==4.0.0
-zope.deprecation==4.1.2
-zope.event==4.0.3
-zope.interface==4.1.3
+zope.cachedescriptors==4.3.1
+zope.deprecation==4.3.0
+zope.event==4.3.0
+zope.interface==4.4.3
 
 
 # IPYTHON RENDERING
 # entrypoints backport, pypi version doesn't support egg installs
 https://code.rhodecode.com/upstream/entrypoints/archive/96e6d645684e1af3d7df5b5272f3fe85a546b233.tar.gz?md5=7db37771aea9ac9fefe093e5d6987313#egg=entrypoints==0.2.2.rhodecode-upstream1
 nbconvert==5.3.1
-bleach==2.1.1
+bleach==2.1.2
 nbformat==4.4.0
 jupyter_client==5.0.0
 
 ## cli tools
-alembic==0.9.6
+alembic==0.9.8
 invoke==0.13.0
 bumpversion==0.5.3
 transifex-client==0.12.5
 
 ## http servers
 gevent==1.2.2
-greenlet==0.4.12
+greenlet==0.4.13
 gunicorn==19.7.1
 waitress==1.1.0
 uWSGI==2.0.15
@@ -125,7 +126,7 @@ bottle==0.12.13
 https://code.rhodecode.com/rhodecode-tools-ce/archive/v0.14.1.tar.gz?md5=0b9c2caad160b68889f8172ea54af7b2#egg=rhodecode-tools==0.14.1
 
 ## appenlight
-appenlight-client==0.6.22
+appenlight-client==0.6.25
 
 ## test related requirements
 -r requirements_test.txt

rhodecode/VERSION

@@ -1 +1 @@
-4.11.6
+4.12.0

rhodecode/__init__.py

@@ -51,7 +51,7 @@ PYRAMID_SETTINGS = {}
 EXTENSIONS = {}
 
 __version__ = ('.'.join((str(each) for each in VERSION[:3])))
-__dbversion__ = 85  # defines current db version for migrations
+__dbversion__ = 86  # defines current db version for migrations
 __platform__ = platform.system()
 __license__ = 'AGPLv3, and Commercial License'
 __author__ = 'RhodeCode GmbH'

rhodecode/api/tests/test_get_repo_changeset.py

@@ -42,7 +42,8 @@ class TestGetRepoChangeset(object):
         if details == 'full':
             assert result['refs']['bookmarks'] == getattr(
                 commit, 'bookmarks', [])
-            assert result['refs']['branches'] == [commit.branch]
+            branches = [commit.branch] if commit.branch else []
+            assert result['refs']['branches'] == branches
             assert result['refs']['tags'] == commit.tags
 
     @pytest.mark.parametrize("details", ['basic', 'extended', 'full'])

rhodecode/api/tests/test_pull.py

@@ -33,12 +33,14 @@ class TestPull(object):
     def test_api_pull(self, backend):
         r = backend.create_repo()
         repo_name = r.repo_name
-        r.clone_uri = os.path.join(TESTS_TMP_PATH, backend.repo_name)
+        clone_uri = os.path.join(TESTS_TMP_PATH, backend.repo_name)
+        r.clone_uri = clone_uri
 
         id_, params = build_data(self.apikey, 'pull', repoid=repo_name,)
         response = api_call(self.app, params)
-
-        expected = {'msg': 'Pulled from `%s`' % (repo_name,),
+        msg = 'Pulled from url `%s` on repo `%s`' % (
+            clone_uri, repo_name)
+        expected = {'msg': msg,
                     'repository': repo_name}
         assert_ok(id_, expected, given=response.body)
 
@@ -47,5 +49,5 @@ class TestPull(object):
             self.apikey, 'pull', repoid=backend.repo_name)
         response = api_call(self.app, params)
 
-        expected = 'Unable to pull changes from `%s`' % (backend.repo_name,)
+        expected = 'Unable to pull changes from `None`'
         assert_error(id_, expected, given=response.body)

rhodecode/api/tests/test_update_repo.py

@@ -56,6 +56,9 @@ class TestApiUpdateRepo(object):
         ({'clone_uri': ''},
          {'clone_uri': ''}),
 
+        ({'push_uri': ''},
+         {'push_uri': ''}),
+
         ({'landing_rev': 'rev:tip'},
          {'landing_rev': ['rev', 'tip']}),
 

rhodecode/api/tests/test_update_user_group.py

@@ -36,7 +36,9 @@ class TestUpdateUserGroup(object):
         # ('owner', {'owner': TEST_USER_REGULAR_LOGIN}),
         ('owner_email', {'owner_email': TEST_USER_ADMIN_EMAIL}),
         ('active', {'active': False}),
-        ('active', {'active': True})
+        ('active', {'active': True}),
+        ('sync', {'sync': False}),
+        ('sync', {'sync': True})
     ])
     def test_api_update_user_group(self, changing_attr, updates, user_util):
         user_group = user_util.create_user_group()
@@ -49,6 +51,12 @@ class TestUpdateUserGroup(object):
             **updates)
         response = api_call(self.app, params)
 
+        # special case for sync
+        if changing_attr == 'sync' and updates['sync'] is False:
+            expected_api_data['sync'] = None
+        elif changing_attr == 'sync' and updates['sync'] is True:
+            expected_api_data['sync'] = 'manual_api'
+
         expected = {
             'msg': 'updated user group ID:%s %s' % (
                 user_group.users_group_id, user_group.users_group_name),
@@ -63,7 +71,9 @@ class TestUpdateUserGroup(object):
         # ('owner', {'owner': TEST_USER_REGULAR_LOGIN}),
         ('owner_email', {'owner_email': TEST_USER_ADMIN_EMAIL}),
         ('active', {'active': False}),
-        ('active', {'active': True})
+        ('active', {'active': True}),
+        ('sync', {'sync': False}),
+        ('sync', {'sync': True})
     ])
     def test_api_update_user_group_regular_user(
             self, changing_attr, updates, user_util):
@@ -72,7 +82,6 @@ class TestUpdateUserGroup(object):
         expected_api_data = user_group.get_api_data()
         expected_api_data.update(updates)
 
-
         # grant permission to this user
         user = UserModel().get_by_username(self.TEST_USER_LOGIN)
 
@@ -82,6 +91,12 @@ class TestUpdateUserGroup(object):
             self.apikey_regular, 'update_user_group',
             usergroupid=group_name, **updates)
         response = api_call(self.app, params)
+        # special case for sync
+        if changing_attr == 'sync' and updates['sync'] is False:
+            expected_api_data['sync'] = None
+        elif changing_attr == 'sync' and updates['sync'] is True:
+            expected_api_data['sync'] = 'manual_api'
+
         expected = {
             'msg': 'updated user group ID:%s %s' % (
                 user_group.users_group_id, user_group.users_group_name),

rhodecode/api/views/repo_api.py

@@ -563,6 +563,7 @@ def create_repo(
         description=Optional(''),
         private=Optional(False),
         clone_uri=Optional(None),
+        push_uri=Optional(None),
         landing_rev=Optional('rev:tip'),
         enable_statistics=Optional(False),
         enable_locking=Optional(False),
@@ -596,6 +597,8 @@ def create_repo(
     :type private: bool
     :param clone_uri: set clone_uri
     :type clone_uri: str
+    :param push_uri: set push_uri
+    :type push_uri: str
     :param landing_rev: <rev_type>:<rev>
     :type landing_rev: str
     :param enable_locking:
@@ -639,6 +642,7 @@ def create_repo(
     description = Optional.extract(description)
     copy_permissions = Optional.extract(copy_permissions)
     clone_uri = Optional.extract(clone_uri)
+    push_uri = Optional.extract(push_uri)
     landing_commit_ref = Optional.extract(landing_rev)
 
     defs = SettingsModel().get_default_repo_settings(strip_prefix=True)
@@ -667,6 +671,7 @@ def create_repo(
             repo_description=description,
             repo_landing_commit_ref=landing_commit_ref,
             repo_clone_uri=clone_uri,
+            repo_push_uri=push_uri,
             repo_private=private,
             repo_copy_permissions=copy_permissions,
             repo_enable_statistics=enable_statistics,
@@ -685,6 +690,7 @@ def create_repo(
             'repo_description': schema_data['repo_description'],
             'repo_private': schema_data['repo_private'],
             'clone_uri': schema_data['repo_clone_uri'],
+            'push_uri': schema_data['repo_push_uri'],
             'repo_landing_rev': schema_data['repo_landing_commit_ref'],
             'enable_statistics': schema_data['repo_enable_statistics'],
             'enable_locking': schema_data['repo_enable_locking'],
@@ -692,7 +698,7 @@ def create_repo(
             'repo_copy_permissions': schema_data['repo_copy_permissions'],
         }
 
-        task = RepoModel().create(form_data=data, cur_user=owner)
+        task = RepoModel().create(form_data=data, cur_user=owner.user_id)
         task_id = get_task_id(task)
         # no commit, it's done in RepoModel, or async via celery
         return {
@@ -800,7 +806,8 @@ def remove_field_from_repo(request, apiuser, repoid, key):
 def update_repo(
         request, apiuser, repoid, repo_name=Optional(None),
         owner=Optional(OAttr('apiuser')), description=Optional(''),
-        private=Optional(False), clone_uri=Optional(None),
+        private=Optional(False),
+        clone_uri=Optional(None), push_uri=Optional(None),
         landing_rev=Optional('rev:tip'), fork_of=Optional(None),
         enable_statistics=Optional(False),
         enable_locking=Optional(False),
@@ -877,6 +884,9 @@ def update_repo(
         clone_uri=clone_uri
         if not isinstance(clone_uri, Optional) else repo.clone_uri,
 
+        push_uri=push_uri
+        if not isinstance(push_uri, Optional) else repo.push_uri,
+
         repo_landing_rev=landing_rev
         if not isinstance(landing_rev, Optional) else repo._landing_revision,
 
@@ -910,6 +920,7 @@ def update_repo(
             repo_owner=updates['user'],
             repo_description=updates['repo_description'],
             repo_clone_uri=updates['clone_uri'],
+            repo_push_uri=updates['push_uri'],
             repo_fork_of=updates['fork_id'],
             repo_private=updates['repo_private'],
             repo_landing_commit_ref=updates['repo_landing_rev'],
@@ -928,6 +939,7 @@ def update_repo(
         repo_description=schema_data['repo_description'],
         repo_private=schema_data['repo_private'],
         clone_uri=schema_data['repo_clone_uri'],
+        push_uri=schema_data['repo_push_uri'],
         repo_landing_rev=schema_data['repo_landing_commit_ref'],
         repo_enable_statistics=schema_data['repo_enable_statistics'],
         repo_enable_locking=schema_data['repo_enable_locking'],
@@ -1084,7 +1096,7 @@ def fork_repo(request, apiuser, repoid, fork_name,
             'landing_rev': schema_data['repo_landing_commit_ref'],
         }
 
-        task = RepoModel().create_fork(data, cur_user=owner)
+        task = RepoModel().create_fork(data, cur_user=owner.user_id)
         # no commit, it's done in RepoModel, or async via celery
         task_id = get_task_id(task)
 
@@ -1749,7 +1761,7 @@ def revoke_user_group_permission(request, apiuser, repoid, usergroupid):
 
 
 @jsonrpc_method()
-def pull(request, apiuser, repoid):
+def pull(request, apiuser, repoid, remote_uri=Optional(None)):
     """
     Triggers a pull on the given repository from a remote location. You
     can use this to keep remote repositories up-to-date.
@@ -1764,6 +1776,8 @@ def pull(request, apiuser, repoid):
     :type apiuser: AuthUser
     :param repoid: The repository name or repository ID.
     :type repoid: str or int
+    :param remote_uri: Optional remote URI to pass in for pull
+    :type remote_uri: str
 
     Example output:
 
@@ -1771,7 +1785,7 @@ def pull(request, apiuser, repoid):
 
       id : <id_given_in_input>
       result : {
-        "msg": "Pulled from `<repository name>`"
+        "msg": "Pulled from url `<remote_url>` on repo `<repository name>`"
         "repository": "<repository name>"
       }
       error :  null
@@ -1783,27 +1797,31 @@ def pull(request, apiuser, repoid):
       id : <id_given_in_input>
       result : null
       error :  {
-        "Unable to pull changes from `<reponame>`"
+        "Unable to push changes from `<remote_url>`"
       }
 
     """
 
     repo = get_repo_or_error(repoid)
+    remote_uri = Optional.extract(remote_uri)
+    remote_uri_display = remote_uri or repo.clone_uri_hidden
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         validate_repo_permissions(apiuser, repoid, repo, _perms)
 
     try:
-        ScmModel().pull_changes(repo.repo_name, apiuser.username)
+        ScmModel().pull_changes(
+            repo.repo_name, apiuser.username, remote_uri=remote_uri)
         return {
-            'msg': 'Pulled from `%s`' % repo.repo_name,
+            'msg': 'Pulled from url `%s` on repo `%s`' % (
+                remote_uri_display, repo.repo_name),
             'repository': repo.repo_name
         }
     except Exception:
         log.exception("Exception occurred while trying to "
                       "pull changes from remote location")
         raise JSONRPCError(
-            'Unable to pull changes from `%s`' % repo.repo_name
+            'Unable to pull changes from `%s`' % remote_uri_display
         )
 
 

rhodecode/api/views/user_group_api.py

@@ -168,7 +168,8 @@ def get_user_groups(request, apiuser):
 @jsonrpc_method()
 def create_user_group(
         request, apiuser, group_name, description=Optional(''),
-        owner=Optional(OAttr('apiuser')), active=Optional(True)):
+        owner=Optional(OAttr('apiuser')), active=Optional(True),
+        sync=Optional(None)):
     """
     Creates a new user group.
 
@@ -188,6 +189,9 @@ def create_user_group(
     :type owner: Optional(str or int)
     :param active: Set this group as active.
     :type active: Optional(``True`` | ``False``)
+    :param sync: Set enabled or disabled the automatically sync from
+        external authentication types like ldap.
+    :type sync: Optional(``True`` | ``False``)
 
     Example output:
 
@@ -227,6 +231,15 @@ def create_user_group(
     owner = get_user_or_error(owner)
     active = Optional.extract(active)
     description = Optional.extract(description)
+    sync = Optional.extract(sync)
+
+    # set the sync option based on group_data
+    group_data = None
+    if sync:
+        group_data = {
+            'extern_type': 'manual_api',
+            'extern_type_set_by': apiuser.username
+        }
 
     schema = user_group_schema.UserGroupSchema().bind(
         # user caller
@@ -246,7 +259,7 @@ def create_user_group(
             name=schema_data['user_group_name'],
             description=schema_data['user_group_description'],
             owner=owner,
-            active=schema_data['user_group_active'])
+            active=schema_data['user_group_active'], group_data=group_data)
         Session().flush()
         creation_data = user_group.get_api_data()
         audit_logger.store_api(
@@ -265,7 +278,7 @@ def create_user_group(
 @jsonrpc_method()
 def update_user_group(request, apiuser, usergroupid, group_name=Optional(''),
                       description=Optional(''), owner=Optional(None),
-                      active=Optional(True)):
+                      active=Optional(True), sync=Optional(None)):
     """
     Updates the specified `user group` with the details provided.
 
@@ -284,6 +297,9 @@ def update_user_group(request, apiuser, usergroupid, group_name=Optional(''),
     :type owner: Optional(str or int)
     :param active: Set the group as active.
     :type active: Optional(``True`` | ``False``)
+    :param sync: Set enabled or disabled the automatically sync from
+        external authentication types like ldap.
+    :type sync: Optional(``True`` | ``False``)
 
     Example output:
 
@@ -329,8 +345,21 @@ def update_user_group(request, apiuser, usergroupid, group_name=Optional(''),
     store_update(updates, description, 'user_group_description')
     store_update(updates, owner, 'user')
     store_update(updates, active, 'users_group_active')
+
+    sync = Optional.extract(sync)
+    group_data = None
+    if sync is True:
+        group_data = {
+            'extern_type': 'manual_api',
+            'extern_type_set_by': apiuser.username
+        }
+    if sync is False:
+        group_data = user_group.group_data
+        if group_data and "extern_type" in group_data:
+            del group_data["extern_type"]
+
     try:
-        UserGroupModel().update(user_group, updates)
+        UserGroupModel().update(user_group, updates, group_data=group_data)
         audit_logger.store_api(
             'user_group.edit', action_data={'old_data': old_data},
             user=apiuser)
@@ -609,8 +638,18 @@ def grant_user_permission_to_user_group(
     perm = get_perm_or_error(perm, prefix='usergroup.')
 
     try:
-        UserGroupModel().grant_user_permission(
+        changes = UserGroupModel().grant_user_permission(
             user_group=user_group, user=user, perm=perm)
+
+        action_data = {
+            'added': changes['added'],
+            'updated': changes['updated'],
+            'deleted': changes['deleted'],
+        }
+        audit_logger.store_api(
+            'user_group.edit.permissions', action_data=action_data,
+            user=apiuser)
+
         Session().commit()
         return {
             'msg':
@@ -669,8 +708,17 @@ def revoke_user_permission_from_user_group(
     user = get_user_or_error(userid)
 
     try:
-        UserGroupModel().revoke_user_permission(
+        changes = UserGroupModel().revoke_user_permission(
             user_group=user_group, user=user)
+        action_data = {
+            'added': changes['added'],
+            'updated': changes['updated'],
+            'deleted': changes['deleted'],
+        }
+        audit_logger.store_api(
+            'user_group.edit.permissions', action_data=action_data,
+            user=apiuser)
+
         Session().commit()
         return {
             'msg': 'Revoked perm for user: `%s` in user group: `%s`' % (
@@ -735,11 +783,20 @@ def grant_user_group_permission_to_user_group(
                 'user group `%s` does not exist' % (sourceusergroupid,))
 
     try:
-        UserGroupModel().grant_user_group_permission(
+        changes = UserGroupModel().grant_user_group_permission(
             target_user_group=target_user_group,
             user_group=user_group, perm=perm)
-        Session().commit()
 
+        action_data = {
+            'added': changes['added'],
+            'updated': changes['updated'],
+            'deleted': changes['deleted'],
+        }
+        audit_logger.store_api(
+            'user_group.edit.permissions', action_data=action_data,
+            user=apiuser)
+
+        Session().commit()
         return {
             'msg': 'Granted perm: `%s` for user group: `%s` '
                    'in user group: `%s`' % (
@@ -806,8 +863,17 @@ def revoke_user_group_permission_from_user_group(
                 'user group `%s` does not exist' % (sourceusergroupid,))
 
     try:
-        UserGroupModel().revoke_user_group_permission(
+        changes = UserGroupModel().revoke_user_group_permission(
             target_user_group=target_user_group, user_group=user_group)
+        action_data = {
+            'added': changes['added'],
+            'updated': changes['updated'],
+            'deleted': changes['deleted'],
+        }
+        audit_logger.store_api(
+            'user_group.edit.permissions', action_data=action_data,
+            user=apiuser)
+
         Session().commit()
 
         return {

rhodecode/apps/_base/__init__.py

@@ -22,9 +22,9 @@ import time
 import logging
 import operator
 
-from pyramid.httpexceptions import HTTPFound
+from pyramid.httpexceptions import HTTPFound, HTTPForbidden
 
-from rhodecode.lib import helpers as h
+from rhodecode.lib import helpers as h, diffs
 from rhodecode.lib.utils2 import StrictAttributeDict, safe_int, datetime_to_time
 from rhodecode.lib.vcs.exceptions import RepositoryRequirementError
 from rhodecode.model import repo
@@ -33,6 +33,7 @@ from rhodecode.model import user_group
 from rhodecode.model import user
 from rhodecode.model.db import User
 from rhodecode.model.scm import ScmModel
+from rhodecode.model.settings import VcsSettingsModel
 
 log = logging.getLogger(__name__)
 
@@ -204,28 +205,47 @@ class RepoAppView(BaseAppView):
         c.rhodecode_db_repo = self.db_repo
         c.repo_name = self.db_repo_name
         c.repository_pull_requests = self.db_repo_pull_requests
+        self.path_filter = PathFilter(None)
 
-        c.repository_requirements_missing = False
+        c.repository_requirements_missing = {}
         try:
             self.rhodecode_vcs_repo = self.db_repo.scm_instance()
+            if self.rhodecode_vcs_repo:
+                path_perms = self.rhodecode_vcs_repo.get_path_permissions(
+                    c.auth_user.username)
+                self.path_filter = PathFilter(path_perms)
         except RepositoryRequirementError as e:
-            c.repository_requirements_missing = True
+            c.repository_requirements_missing = {'error': str(e)}
             self._handle_missing_requirements(e)
             self.rhodecode_vcs_repo = None
 
-        if (not c.repository_requirements_missing
-            and self.rhodecode_vcs_repo is None):
+        c.path_filter = self.path_filter  # used by atom_feed_entry.mako
+
+        if self.rhodecode_vcs_repo is None:
             # unable to fetch this repo as vcs instance, report back to user
             h.flash(_(
                 "The repository `%(repo_name)s` cannot be loaded in filesystem. "
                 "Please check if it exist, or is not damaged.") %
                     {'repo_name': c.repo_name},
                     category='error', ignore_duplicate=True)
-            raise HTTPFound(h.route_path('home'))
+            if c.repository_requirements_missing:
+                route = self.request.matched_route.name
+                if route.startswith(('edit_repo', 'repo_summary')):
+                    # allow summary and edit repo on missing requirements
+                    return c
+
+                raise HTTPFound(
+                    h.route_path('repo_summary', repo_name=self.db_repo_name))
+
+            else:  # redirect if we don't show missing requirements
+                raise HTTPFound(h.route_path('home'))
 
         return c
 
-    def _get_f_path(self, matchdict, default=None):
+    def _get_f_path_unchecked(self, matchdict, default=None):
+        """
+        Should only be used by redirects, everything else should call _get_f_path
+        """
         f_path = matchdict.get('f_path')
         if f_path:
             # fix for multiple initial slashes that causes errors for GIT
@@ -233,6 +253,63 @@ class RepoAppView(BaseAppView):
 
         return default
 
+    def _get_f_path(self, matchdict, default=None):
+        f_path_match = self._get_f_path_unchecked(matchdict, default)
+        return self.path_filter.assert_path_permissions(f_path_match)
+
+    def _get_general_setting(self, target_repo, settings_key, default=False):
+        settings_model = VcsSettingsModel(repo=target_repo)
+        settings = settings_model.get_general_settings()
+        return settings.get(settings_key, default)
+
+
+class PathFilter(object):
+
+    # Expects and instance of BasePathPermissionChecker or None
+    def __init__(self, permission_checker):
+        self.permission_checker = permission_checker
+
+    def assert_path_permissions(self, path):
+        if path and self.permission_checker and not self.permission_checker.has_access(path):
+            raise HTTPForbidden()
+        return path
+
+    def filter_patchset(self, patchset):
+        if not self.permission_checker or not patchset:
+            return patchset, False
+        had_filtered = False
+        filtered_patchset = []
+        for patch in patchset:
+            filename = patch.get('filename', None)
+            if not filename or self.permission_checker.has_access(filename):
+                filtered_patchset.append(patch)
+            else:
+                had_filtered = True
+        if had_filtered:
+            if isinstance(patchset, diffs.LimitedDiffContainer):
+                filtered_patchset = diffs.LimitedDiffContainer(patchset.diff_limit, patchset.cur_diff_size, filtered_patchset)
+            return filtered_patchset, True
+        else:
+            return patchset, False
+
+    def render_patchset_filtered(self, diffset, patchset, source_ref=None, target_ref=None):
+        filtered_patchset, has_hidden_changes = self.filter_patchset(patchset)
+        result = diffset.render_patchset(filtered_patchset, source_ref=source_ref, target_ref=target_ref)
+        result.has_hidden_changes = has_hidden_changes
+        return result
+
+    def get_raw_patch(self, diff_processor):
+        if self.permission_checker is None:
+            return diff_processor.as_raw()
+        elif self.permission_checker.has_full_access:
+            return diff_processor.as_raw()
+        else:
+            return '# Repository has user-specific filters, raw patch generation is disabled.'
+
+    @property
+    def is_enabled(self):
+        return self.permission_checker is not None
+
 
 class RepoGroupAppView(BaseAppView):
     def __init__(self, context, request):

rhodecode/apps/admin/__init__.py

@@ -169,6 +169,11 @@ def admin_routes(config):
         name='admin_settings_labs_update',
         pattern='/settings/labs/update')
 
+    # Automation EE feature
+    config.add_route(
+        'admin_settings_automation',
+        pattern=ADMIN_PREFIX + '/settings/automation')
+
     # global permissions
 
     config.add_route(

rhodecode/apps/admin/navigation.py

@@ -95,7 +95,8 @@ class NavigationRegistry(object):
                  'admin_settings_sessions'),
         NavEntry('open_source', _('Open Source Licenses'),
                  'admin_settings_open_source'),
-
+        NavEntry('automation', _('Automation'),
+                 'admin_settings_automation')
     ]
 
     _labs_entry = NavEntry('labs', _('Labs'),

rhodecode/apps/admin/tests/test_admin_auth_settings.py

@@ -86,6 +86,7 @@ class TestAuthSettingsView(object):
 
             'host': 'dc.example.com',
             'port': '999',
+            'timeout': 3600,
             'tls_kind': 'PLAIN',
             'tls_reqcert': 'NEVER',
 

rhodecode/apps/admin/tests/test_admin_users_ssh_keys.py

@@ -67,6 +67,7 @@ class TestAdminUsersSshKeysView(TestController):
                    'I4fG8+hBHzpeFxUGvSGNtXPUbwaAY8j/oHYrTpMgkj6pUEFsiKfC5zPq' \
                    'PFR5HyKTCHW0nFUJnZsbyFT5hMiF/hZkJc9A0ZbdSvJwCRQ/g3bmdL ' \
                    'your_email@example.com'
+    FINGERPRINT = 'MD5:01:4f:ad:29:22:6e:01:37:c9:d2:52:26:52:b0:2d:93'
 
     def test_ssh_keys_default_user(self):
         self.log_user()
@@ -111,9 +112,11 @@ class TestAdminUsersSshKeysView(TestController):
             route_path('edit_user_ssh_keys_add', user_id=user_id),
             {'description': desc, 'key_data': key_data,
              'csrf_token': self.csrf_token})
+
+        err = 'Such key with fingerprint `{}` already exists, ' \
+              'please use a different one'.format(self.FINGERPRINT)
         assert_session_flash(response, 'An error occurred during ssh key '
-                                       'saving: Such key already exists, '
-                                       'please use a different one')
+                                       'saving: {}'.format(err))
 
     def test_add_ssh_key(self, user_util):
         self.log_user()

rhodecode/apps/admin/views/process_management.py

@@ -28,7 +28,7 @@ from rhodecode.apps._base import BaseAppView
 from rhodecode.apps.admin.navigation import navigation_list
 from rhodecode.lib.auth import (
     LoginRequired, HasPermissionAllDecorator, CSRFRequired)
-from rhodecode.lib.utils2 import safe_int
+from rhodecode.lib.utils2 import safe_int, StrictAttributeDict
 
 log = logging.getLogger(__name__)
 
@@ -36,9 +36,41 @@ log = logging.getLogger(__name__)
 class AdminProcessManagementView(BaseAppView):
     def load_default_context(self):
         c = self._get_local_tmpl_context()
-
         return c
 
+    def _format_proc(self, proc, with_children=False):
+        try:
+            mem = proc.memory_info()
+            proc_formatted = StrictAttributeDict({
+                'pid': proc.pid,
+                'name': proc.name(),
+                'mem_rss': mem.rss,
+                'mem_vms': mem.vms,
+                'cpu_percent': proc.cpu_percent(),
+                'create_time': proc.create_time(),
+                'cmd': ' '.join(proc.cmdline()),
+            })
+
+            if with_children:
+                proc_formatted.update({
+                    'children': [self._format_proc(x)
+                                 for x in proc.children(recursive=True)]
+                })
+        except Exception:
+            log.exception('Failed to load proc')
+            proc_formatted = None
+        return proc_formatted
+
+    def get_processes(self):
+        proc_list = []
+        for p in psutil.process_iter():
+            if 'gunicorn' in p.name():
+                proc = self._format_proc(p, with_children=True)
+                if proc:
+                    proc_list.append(proc)
+
+        return proc_list
+
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
@@ -50,8 +82,7 @@ class AdminProcessManagementView(BaseAppView):
 
         c.active = 'process_management'
         c.navlist = navigation_list(self.request)
-        c.gunicorn_processes = (
-            p for p in psutil.process_iter() if 'gunicorn' in p.name())
+        c.gunicorn_processes = self.get_processes()
         return self._get_template_context(c)
 
     @LoginRequired()
@@ -62,8 +93,7 @@ class AdminProcessManagementView(BaseAppView):
     def process_management_data(self):
         _ = self.request.translate
         c = self.load_default_context()
-        c.gunicorn_processes = (
-            p for p in psutil.process_iter() if 'gunicorn' in p.name())
+        c.gunicorn_processes = self.get_processes()
         return self._get_template_context(c)
 
     @LoginRequired()
@@ -75,9 +105,10 @@ class AdminProcessManagementView(BaseAppView):
     def process_management_signal(self):
         pids = self.request.json.get('pids', [])
         result = []
+
         def on_terminate(proc):
             msg = "process `PID:{}` terminated with exit code {}".format(
-                proc.pid, proc.returncode)
+                proc.pid, proc.returncode or 0)
             result.append(msg)
 
         procs = []
@@ -91,15 +122,22 @@ class AdminProcessManagementView(BaseAppView):
 
                 children = proc.children(recursive=True)
                 if children:
-                    print('Wont kill Master Process')
+                    log.warning('Wont kill Master Process')
                 else:
                     procs.append(proc)
 
         for p in procs:
-            p.terminate()
+            try:
+                p.terminate()
+            except psutil.AccessDenied as e:
+                log.warning('Access denied: {}'.format(e))
+
         gone, alive = psutil.wait_procs(procs, timeout=10, callback=on_terminate)
         for p in alive:
-            p.kill()
+            try:
+                p.kill()
+            except psutil.AccessDenied as e:
+                log.warning('Access denied: {}'.format(e))
 
         return {'result': result}
 

rhodecode/apps/admin/views/settings.py

@@ -314,6 +314,9 @@ class AdminSettingsView(BaseAppView):
         try:
             form_result = application_form.to_python(dict(self.request.POST))
         except formencode.Invalid as errors:
+            h.flash(
+                _("Some form inputs contain invalid data."),
+                category='error')
             data = render('rhodecode:templates/admin/settings/settings.mako',
                           self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
@@ -386,6 +389,9 @@ class AdminSettingsView(BaseAppView):
         try:
             form_result = application_form.to_python(dict(self.request.POST))
         except formencode.Invalid as errors:
+            h.flash(
+                _("Some form inputs contain invalid data."),
+                category='error')
             data = render('rhodecode:templates/admin/settings/settings.mako',
                           self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
@@ -666,6 +672,17 @@ class AdminSettingsView(BaseAppView):
 
         return self._get_template_context(c)
 
+    @LoginRequired()
+    @HasPermissionAllDecorator('hg.admin')
+    @view_config(
+        route_name='admin_settings_automation', request_method='GET',
+        renderer='rhodecode:templates/admin/settings/settings.mako')
+    def settings_automation(self):
+        c = self.load_default_context()
+        c.active = 'automation'
+
+        return self._get_template_context(c)
+
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
@@ -705,7 +722,7 @@ class AdminSettingsView(BaseAppView):
             form_result = application_form.to_python(dict(self.request.POST))
         except formencode.Invalid as errors:
             h.flash(
-                _('Some form inputs contain invalid data.'),
+                _("Some form inputs contain invalid data."),
                 category='error')
             data = render('rhodecode:templates/admin/settings/settings.mako',
                           self._get_template_context(c), self.request)

rhodecode/apps/admin/views/system_info.py

@@ -123,6 +123,9 @@ class AdminSystemInfoSettingsView(BaseAppView):
             (_('Uptime'), val('uptime')['text'], state('uptime')),
             ('', '', ''),  # spacer
 
+            # ulimit
+            (_('Ulimit'), val('ulimit')['text'], state('ulimit')),
+
             # Repo storage
             (_('Storage location'), val('storage')['path'], state('storage')),
             (_('Storage info'), val('storage')['text'], state('storage')),

rhodecode/apps/admin/views/user_groups.py

@@ -88,8 +88,8 @@ class AdminUserGroupsView(BaseAppView, DataGridAppView):
         _render = self.request.get_partial_renderer(
             'rhodecode:templates/data_table/_dt_elements.mako')
 
-        def user_group_name(user_group_id, user_group_name):
-            return _render("user_group_name", user_group_id, user_group_name)
+        def user_group_name(user_group_name):
+            return _render("user_group_name", user_group_name)
 
         def user_group_actions(user_group_id, user_group_name):
             return _render("user_group_actions", user_group_id, user_group_name)
@@ -153,15 +153,14 @@ class AdminUserGroupsView(BaseAppView, DataGridAppView):
         user_groups_data = []
         for user_gr in auth_user_group_list:
             user_groups_data.append({
-                "users_group_name": user_group_name(
-                    user_gr.users_group_id, h.escape(user_gr.users_group_name)),
+                "users_group_name": user_group_name(user_gr.users_group_name),
                 "name_raw": h.escape(user_gr.users_group_name),
                 "description": h.escape(user_gr.user_group_description),
                 "members": user_gr.member_count,
                 # NOTE(marcink): because of advanced query we
                 # need to load it like that
-                "sync": UserGroup._load_group_data(
-                    user_gr.group_data).get('extern_type'),
+                "sync": UserGroup._load_sync(
+                    UserGroup._load_group_data(user_gr.group_data)),
                 "active": h.bool2icon(user_gr.users_group_active),
                 "owner": user_profile(user_gr.User.username),
                 "action": user_group_actions(

rhodecode/apps/admin/views/users.py

@@ -817,6 +817,7 @@ class UsersView(UserAppView):
         key_data = self.request.POST.get('key_data')
         description = self.request.POST.get('description')
 
+        fingerprint = 'unknown'
         try:
             if not key_data:
                 raise ValueError('Please add a valid public key')
@@ -841,8 +842,9 @@ class UsersView(UserAppView):
 
         except IntegrityError:
             log.exception("Exception during ssh key saving")
-            h.flash(_('An error occurred during ssh key saving: {}').format(
-                'Such key already exists, please use a different one'),
+            err = 'Such key with fingerprint `{}` already exists, ' \
+                  'please use a different one'.format(fingerprint)
+            h.flash(_('An error occurred during ssh key saving: {}').format(err),
                     category='error')
         except Exception as e:
             log.exception("Exception during ssh key saving")

rhodecode/apps/my_account/tests/test_my_account_edit.py

@@ -109,7 +109,7 @@ class TestMyAccountEdit(TestController):
             # ('extern_name', {'extern_name': None}),
             ('active', {'active': False}),
             ('active', {'active': True}),
-            ('email', {'email': 'some@email.com'}),
+            ('email', {'email': u'some@email.com'}),
         ])
     def test_my_account_update(self, name, attrs, user_util):
         usr = user_util.create_user(password='qweqwe')
@@ -120,13 +120,17 @@ class TestMyAccountEdit(TestController):
 
         params.update({'password_confirmation': ''})
         params.update({'new_password': ''})
-        params.update({'extern_type': 'rhodecode'})
-        params.update({'extern_name': 'rhodecode'})
+        params.update({'extern_type': u'rhodecode'})
+        params.update({'extern_name': u'rhodecode'})
         params.update({'csrf_token': self.csrf_token})
 
         params.update(attrs)
         # my account page cannot set language param yet, only for admins
         del params['language']
+        if name == 'email':
+            uem = user_util.create_additional_user_email(usr, attrs['email'])
+            email_before = User.get(user_id).email
+
         response = self.app.post(route_path('my_account_update'), params)
 
         assert_session_flash(
@@ -146,7 +150,7 @@ class TestMyAccountEdit(TestController):
         params['language'] = updated_params['language']
 
         if name == 'email':
-            params['emails'] = [attrs['email']]
+            params['emails'] = [attrs['email'], email_before]
         if name == 'extern_type':
             # cannot update this via form, expected value is original one
             params['extern_type'] = "rhodecode"
@@ -162,10 +166,10 @@ class TestMyAccountEdit(TestController):
 
         assert params == updated_params
 
-    def test_my_account_update_err_email_exists(self):
+    def test_my_account_update_err_email_not_exists_in_emails(self):
         self.log_user()
 
-        new_email = 'test_regular@mail.com'  # already existing email
+        new_email = 'test_regular@mail.com'  # not in emails
         params = {
             'username': 'test_admin',
             'new_password': 'test12',
@@ -179,7 +183,7 @@ class TestMyAccountEdit(TestController):
         response = self.app.post(route_path('my_account_update'),
                                  params=params)
 
-        response.mustcontain('This e-mail address is already taken')
+        response.mustcontain('"test_regular@mail.com" is not one of test_admin@mail.com')
 
     def test_my_account_update_bad_email_address(self):
         self.log_user('test_regular2', 'test12')
@@ -197,7 +201,4 @@ class TestMyAccountEdit(TestController):
         response = self.app.post(route_path('my_account_update'),
                                  params=params)
 
-        response.mustcontain('An email address must contain a single @')
-        msg = u'Username "%(username)s" already exists'
-        msg = h.html_escape(msg % {'username': 'test_admin'})
-        response.mustcontain(u"%s" % msg)
+        response.mustcontain('"newmail.pl" is not one of test_regular2@mail.com')

rhodecode/apps/my_account/tests/test_my_account_emails.py

@@ -24,7 +24,7 @@ from rhodecode.apps._base import ADMIN_PREFIX
 from rhodecode.model.db import User, UserEmailMap
 from rhodecode.tests import (
     TestController, TEST_USER_ADMIN_LOGIN, TEST_USER_REGULAR_EMAIL,
-    assert_session_flash)
+    assert_session_flash, TEST_USER_REGULAR_PASS)
 from rhodecode.tests.fixture import Fixture
 
 fixture = Fixture()
@@ -47,30 +47,14 @@ class TestMyAccountEmails(TestController):
         response = self.app.get(route_path('my_account_emails'))
         response.mustcontain('No additional emails specified')
 
-    def test_my_account_my_emails_add_existing_email(self):
-        self.log_user()
-        response = self.app.get(route_path('my_account_emails'))
-        response.mustcontain('No additional emails specified')
-        response = self.app.post(route_path('my_account_emails_add'),
-                                 {'new_email': TEST_USER_REGULAR_EMAIL,
-                                  'csrf_token': self.csrf_token})
-        assert_session_flash(response, 'This e-mail address is already taken')
-
-    def test_my_account_my_emails_add_mising_email_in_form(self):
-        self.log_user()
-        response = self.app.get(route_path('my_account_emails'))
-        response.mustcontain('No additional emails specified')
-        response = self.app.post(route_path('my_account_emails_add'),
-                                 {'csrf_token': self.csrf_token})
-        assert_session_flash(response, 'Please enter an email address')
-
     def test_my_account_my_emails_add_remove(self):
         self.log_user()
         response = self.app.get(route_path('my_account_emails'))
         response.mustcontain('No additional emails specified')
 
         response = self.app.post(route_path('my_account_emails_add'),
-                                 {'new_email': 'foo@barz.com',
+                                 {'email': 'foo@barz.com',
+                                  'current_password': TEST_USER_REGULAR_PASS,
                                   'csrf_token': self.csrf_token})
 
         response = self.app.get(route_path('my_account_emails'))

rhodecode/apps/my_account/tests/test_my_account_ssh_keys.py

@@ -66,6 +66,7 @@ class TestMyAccountSshKeysView(TestController):
                    'I4fG8+hBHzpeFxUGvSGNtXPUbwaAY8j/oHYrTpMgkj6pUEFsiKfC5zPq' \
                    'PFR5HyKTCHW0nFUJnZsbyFT5hMiF/hZkJc9A0ZbdSvJwCRQ/g3bmdL ' \
                    'your_email@example.com'
+    FINGERPRINT = 'MD5:01:4f:ad:29:22:6e:01:37:c9:d2:52:26:52:b0:2d:93'
 
     def test_add_ssh_key_error(self, user_util):
         user = user_util.create_user(password='qweqwe')
@@ -100,9 +101,11 @@ class TestMyAccountSshKeysView(TestController):
             route_path('my_account_ssh_keys_add'),
             {'description': desc, 'key_data': key_data,
              'csrf_token': self.csrf_token})
+
+        err = 'Such key with fingerprint `{}` already exists, ' \
+              'please use a different one'.format(self.FINGERPRINT)
         assert_session_flash(response, 'An error occurred during ssh key '
-                                       'saving: Such key already exists, '
-                                       'please use a different one')
+                                       'saving: {}'.format(err))
 
     def test_add_ssh_key(self, user_util):
         user = user_util.create_user(password='qweqwe')

rhodecode/apps/my_account/views/my_account.py

@@ -232,40 +232,59 @@ class MyAccountView(BaseAppView, DataGridAppView):
 
         c.user_email_map = UserEmailMap.query()\
             .filter(UserEmailMap.user == c.user).all()
+
+        schema = user_schema.AddEmailSchema().bind(
+            username=c.user.username, user_emails=c.user.emails)
+
+        form = forms.RcForm(schema,
+                            action=h.route_path('my_account_emails_add'),
+                            buttons=(forms.buttons.save, forms.buttons.reset))
+
+        c.form = form
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
     @view_config(
-        route_name='my_account_emails_add', request_method='POST')
+        route_name='my_account_emails_add', request_method='POST',
+        renderer='rhodecode:templates/admin/my_account/my_account.mako')
     def my_account_emails_add(self):
         _ = self.request.translate
         c = self.load_default_context()
+        c.active = 'emails'
 
-        email = self.request.POST.get('new_email')
+        schema = user_schema.AddEmailSchema().bind(
+            username=c.user.username, user_emails=c.user.emails)
 
+        form = forms.RcForm(
+            schema, action=h.route_path('my_account_emails_add'),
+            buttons=(forms.buttons.save, forms.buttons.reset))
+
+        controls = self.request.POST.items()
         try:
-            form = UserExtraEmailForm(self.request.translate)()
-            data = form.to_python({'email': email})
-            email = data['email']
-
-            UserModel().add_extra_email(c.user.user_id, email)
+            valid_data = form.validate(controls)
+            UserModel().add_extra_email(c.user.user_id, valid_data['email'])
             audit_logger.store_web(
                 'user.edit.email.add', action_data={
-                    'data': {'email': email, 'user': 'self'}},
+                    'data': {'email': valid_data['email'], 'user': 'self'}},
                 user=self._rhodecode_user,)
-
             Session().commit()
-            h.flash(_("Added new email address `%s` for user account") % email,
-                    category='success')
         except formencode.Invalid as error:
             h.flash(h.escape(error.error_dict['email']), category='error')
+        except forms.ValidationFailure as e:
+            c.user_email_map = UserEmailMap.query() \
+                .filter(UserEmailMap.user == c.user).all()
+            c.form = e
+            return self._get_template_context(c)
         except Exception:
-            log.exception("Exception in my_account_emails")
-            h.flash(_('An error occurred during email saving'),
+            log.exception("Exception adding email")
+            h.flash(_('Error occurred during adding email'),
                     category='error')
-        return HTTPFound(h.route_path('my_account_emails'))
+        else:
+            h.flash(_("Successfully added email"), category='success')
+
+        raise HTTPFound(self.request.route_path('my_account_emails'))
 
     @LoginRequired()
     @NotAnonymous()
@@ -414,22 +433,23 @@ class MyAccountView(BaseAppView, DataGridAppView):
     def my_account_edit(self):
         c = self.load_default_context()
         c.active = 'profile_edit'
-
-        c.perm_user = c.auth_user
         c.extern_type = c.user.extern_type
         c.extern_name = c.user.extern_name
 
-        defaults = c.user.get_dict()
+        schema = user_schema.UserProfileSchema().bind(
+            username=c.user.username, user_emails=c.user.emails)
+        appstruct = {
+            'username': c.user.username,
+            'email': c.user.email,
+            'firstname': c.user.firstname,
+            'lastname': c.user.lastname,
+        }
+        c.form = forms.RcForm(
+            schema, appstruct=appstruct,
+            action=h.route_path('my_account_update'),
+            buttons=(forms.buttons.save, forms.buttons.reset))
 
-        data = render('rhodecode:templates/admin/my_account/my_account.mako',
-                      self._get_template_context(c), self.request)
-        html = formencode.htmlfill.render(
-            data,
-            defaults=defaults,
-            encoding="UTF-8",
-            force_defaults=False
-        )
-        return Response(html)
+        return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
@@ -442,55 +462,40 @@ class MyAccountView(BaseAppView, DataGridAppView):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'profile_edit'
-
         c.perm_user = c.auth_user
         c.extern_type = c.user.extern_type
         c.extern_name = c.user.extern_name
 
-        _form = UserForm(self.request.translate, edit=True,
-                         old_data={'user_id': self._rhodecode_user.user_id,
-                                   'email': self._rhodecode_user.email})()
-        form_result = {}
+        schema = user_schema.UserProfileSchema().bind(
+            username=c.user.username, user_emails=c.user.emails)
+        form = forms.RcForm(
+            schema, buttons=(forms.buttons.save, forms.buttons.reset))
+
+        controls = self.request.POST.items()
         try:
-            post_data = dict(self.request.POST)
-            post_data['new_password'] = ''
-            post_data['password_confirmation'] = ''
-            form_result = _form.to_python(post_data)
-            # skip updating those attrs for my account
+            valid_data = form.validate(controls)
             skip_attrs = ['admin', 'active', 'extern_type', 'extern_name',
                           'new_password', 'password_confirmation']
-            # TODO: plugin should define if username can be updated
             if c.extern_type != "rhodecode":
                 # forbid updating username for external accounts
                 skip_attrs.append('username')
-
+            old_email = c.user.email
             UserModel().update_user(
-                self._rhodecode_user.user_id, skip_attrs=skip_attrs,
-                **form_result)
-            h.flash(_('Your account was updated successfully'),
-                    category='success')
+                     self._rhodecode_user.user_id, skip_attrs=skip_attrs,
+                     **valid_data)
+            if old_email != valid_data['email']:
+                old = UserEmailMap.query() \
+                    .filter(UserEmailMap.user == c.user).filter(UserEmailMap.email == valid_data['email']).first()
+                old.email = old_email
+            h.flash(_('Your account was updated successfully'), category='success')
             Session().commit()
-
-        except formencode.Invalid as errors:
-            data = render(
-                'rhodecode:templates/admin/my_account/my_account.mako',
-                self._get_template_context(c), self.request)
-
-            html = formencode.htmlfill.render(
-                data,
-                defaults=errors.value,
-                errors=errors.error_dict or {},
-                prefix_error=False,
-                encoding="UTF-8",
-                force_defaults=False)
-            return Response(html)
-
+        except forms.ValidationFailure as e:
+            c.form = e
+            return self._get_template_context(c)
         except Exception:
             log.exception("Exception updating user")
-            h.flash(_('Error occurred during update of user %s')
-                    % form_result.get('username'), category='error')
-            raise HTTPFound(h.route_path('my_account_profile'))
-
+            h.flash(_('Error occurred during update of user'),
+                    category='error')
         raise HTTPFound(h.route_path('my_account_profile'))
 
     def _get_pull_requests_list(self, statuses):

rhodecode/apps/my_account/views/my_account_ssh_keys.py

@@ -89,7 +89,7 @@ class MyAccountSshKeysView(BaseAppView, DataGridAppView):
         user_data = c.user.get_api_data()
         key_data = self.request.POST.get('key_data')
         description = self.request.POST.get('description')
-
+        fingerprint = 'unknown'
         try:
             if not key_data:
                 raise ValueError('Please add a valid public key')
@@ -114,8 +114,9 @@ class MyAccountSshKeysView(BaseAppView, DataGridAppView):
 
         except IntegrityError:
             log.exception("Exception during ssh key saving")
-            h.flash(_('An error occurred during ssh key saving: {}').format(
-                'Such key already exists, please use a different one'),
+            err = 'Such key with fingerprint `{}` already exists, ' \
+                  'please use a different one'.format(fingerprint)
+            h.flash(_('An error occurred during ssh key saving: {}').format(err),
                     category='error')
         except Exception as e:
             log.exception("Exception during ssh key saving")

rhodecode/apps/repository/__init__.py

@@ -331,6 +331,10 @@ def includeme(config):
         name='edit_repo_advanced_fork',
         pattern='/{repo_name:.*?[^/]}/settings/advanced/fork', repo_route=True)
 
+    config.add_route(
+        name='edit_repo_advanced_hooks',
+        pattern='/{repo_name:.*?[^/]}/settings/advanced/hooks', repo_route=True)
+
     # Caches
     config.add_route(
         name='edit_repo_caches',
@@ -373,6 +377,9 @@ def includeme(config):
     config.add_route(
         name='edit_repo_remote_pull',
         pattern='/{repo_name:.*?[^/]}/settings/remote/pull', repo_route=True)
+    config.add_route(
+        name='edit_repo_remote_push',
+        pattern='/{repo_name:.*?[^/]}/settings/remote/push', repo_route=True)
 
     # Statistics
     config.add_route(
@@ -418,6 +425,11 @@ def includeme(config):
         name='repo_default_reviewers_data',
         pattern='/{repo_name:.*?[^/]}/settings/review/default-reviewers', repo_route=True)
 
+    # Repo Automation (EE feature)
+    config.add_route(
+        name='repo_automation',
+        pattern='/{repo_name:.*?[^/]}/settings/automation', repo_route=True)
+
     # Strip
     config.add_route(
         name='edit_repo_strip',

rhodecode/apps/repository/tests/test_repo_summary.py

@@ -234,7 +234,8 @@ class TestSummaryView(object):
             Repository, 'scm_instance', side_effect=RepositoryRequirementError)
 
         with scm_patcher:
-            response = self.app.get(route_path('repo_summary', repo_name=repo_name))
+            response = self.app.get(
+                route_path('repo_summary', repo_name=repo_name))
         assert_response = AssertResponse(response)
         assert_response.element_contains(
             '.main .alert-warning strong', 'Missing requirements')

rhodecode/apps/repository/views/repo_automation.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2016-2018 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import logging
+
+from pyramid.view import view_config
+
+from rhodecode.apps._base import RepoAppView
+from rhodecode.apps.repository.utils import get_default_reviewers_data
+from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
+
+log = logging.getLogger(__name__)
+
+
+class RepoAutomationView(RepoAppView):
+    def load_default_context(self):
+        c = self._get_local_tmpl_context()
+        return c
+
+    @LoginRequired()
+    @HasRepoPermissionAnyDecorator('repository.admin')
+    @view_config(
+        route_name='repo_automation', request_method='GET',
+        renderer='rhodecode:templates/admin/repos/repo_edit.mako')
+    def repo_automation(self):
+        c = self.load_default_context()
+        c.active = 'automation'
+
+        return self._get_template_context(c)

rhodecode/apps/repository/views/repo_caches.py

@@ -18,6 +18,7 @@
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
+import os
 import logging
 
 from pyramid.httpexceptions import HTTPFound
@@ -27,6 +28,7 @@ from rhodecode.apps._base import RepoAppView
 from rhodecode.lib.auth import (
     LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired)
 from rhodecode.lib import helpers as h
+from rhodecode.lib import system_info
 from rhodecode.model.meta import Session
 from rhodecode.model.scm import ScmModel
 
@@ -36,8 +38,6 @@ log = logging.getLogger(__name__)
 class RepoCachesView(RepoAppView):
     def load_default_context(self):
         c = self._get_local_tmpl_context()
-
-
         return c
 
     @LoginRequired()
@@ -48,6 +48,11 @@ class RepoCachesView(RepoAppView):
     def repo_caches(self):
         c = self.load_default_context()
         c.active = 'caches'
+        cached_diffs_dir = c.rhodecode_db_repo.cached_diffs_dir
+        c.cached_diff_count = len(c.rhodecode_db_repo.cached_diffs())
+        c.cached_diff_size = 0
+        if os.path.isdir(cached_diffs_dir):
+            c.cached_diff_size = system_info.get_storage_size(cached_diffs_dir)
 
         return self._get_template_context(c)
 

rhodecode/apps/repository/views/repo_commits.py

@@ -34,17 +34,18 @@ from rhodecode.lib.auth import (
     LoginRequired, HasRepoPermissionAnyDecorator, NotAnonymous, CSRFRequired)
 
 from rhodecode.lib.compat import OrderedDict
+from rhodecode.lib.diffs import cache_diff, load_cached_diff, diff_cache_exist
 from rhodecode.lib.exceptions import StatusChangeOnClosedPullRequestError
 import rhodecode.lib.helpers as h
-from rhodecode.lib.utils2 import safe_unicode
+from rhodecode.lib.utils2 import safe_unicode, str2bool
 from rhodecode.lib.vcs.backends.base import EmptyCommit
 from rhodecode.lib.vcs.exceptions import (
-    RepositoryError, CommitDoesNotExistError, NodeDoesNotExistError)
+    RepositoryError, CommitDoesNotExistError)
 from rhodecode.model.db import ChangesetComment, ChangesetStatus
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.comment import CommentsModel
 from rhodecode.model.meta import Session
-
+from rhodecode.model.settings import VcsSettingsModel
 
 log = logging.getLogger(__name__)
 
@@ -152,6 +153,12 @@ class RepoCommitsView(RepoAppView):
 
         return c
 
+    def _is_diff_cache_enabled(self, target_repo):
+        caching_enabled = self._get_general_setting(
+            target_repo, 'rhodecode_diff_cache')
+        log.debug('Diff caching enabled: %s', caching_enabled)
+        return caching_enabled
+
     def _commit(self, commit_id_range, method):
         _ = self.request.translate
         c = self.load_default_context()
@@ -240,45 +247,65 @@ class RepoCommitsView(RepoAppView):
             commit2 = commit
             commit1 = commit.parents[0] if commit.parents else EmptyCommit()
 
-            _diff = self.rhodecode_vcs_repo.get_diff(
-                commit1, commit2,
-                ignore_whitespace=ign_whitespace_lcl, context=context_lcl)
-            diff_processor = diffs.DiffProcessor(
-                _diff, format='newdiff', diff_limit=diff_limit,
-                file_limit=file_limit, show_full_diff=c.fulldiff)
-
-            commit_changes = OrderedDict()
             if method == 'show':
-                _parsed = diff_processor.prepare()
-                c.limited_diff = isinstance(_parsed, diffs.LimitedDiffContainer)
-
-                _parsed = diff_processor.prepare()
-
-                def _node_getter(commit):
-                    def get_node(fname):
-                        try:
-                            return commit.get_node(fname)
-                        except NodeDoesNotExistError:
-                            return None
-                    return get_node
-
                 inline_comments = CommentsModel().get_inline_comments(
                     self.db_repo.repo_id, revision=commit.raw_id)
                 c.inline_cnt = CommentsModel().get_inline_comments_count(
                     inline_comments)
+                c.inline_comments = inline_comments
 
-                diffset = codeblocks.DiffSet(
-                    repo_name=self.db_repo_name,
-                    source_node_getter=_node_getter(commit1),
-                    target_node_getter=_node_getter(commit2),
-                    comments=inline_comments)
-                diffset = diffset.render_patchset(
-                    _parsed, commit1.raw_id, commit2.raw_id)
+                cache_path = self.rhodecode_vcs_repo.get_create_shadow_cache_pr_path(
+                    self.db_repo)
+                cache_file_path = diff_cache_exist(
+                    cache_path, 'diff', commit.raw_id,
+                    ign_whitespace_lcl, context_lcl, c.fulldiff)
 
+                caching_enabled = self._is_diff_cache_enabled(self.db_repo)
+                force_recache = str2bool(self.request.GET.get('force_recache'))
+
+                cached_diff = None
+                if caching_enabled:
+                    cached_diff = load_cached_diff(cache_file_path)
+
+                has_proper_diff_cache = cached_diff and cached_diff.get('diff')
+                if not force_recache and has_proper_diff_cache:
+                    diffset = cached_diff['diff']
+                else:
+                    vcs_diff = self.rhodecode_vcs_repo.get_diff(
+                        commit1, commit2,
+                        ignore_whitespace=ign_whitespace_lcl,
+                        context=context_lcl)
+
+                    diff_processor = diffs.DiffProcessor(
+                        vcs_diff, format='newdiff', diff_limit=diff_limit,
+                        file_limit=file_limit, show_full_diff=c.fulldiff)
+
+                    _parsed = diff_processor.prepare()
+
+                    diffset = codeblocks.DiffSet(
+                        repo_name=self.db_repo_name,
+                        source_node_getter=codeblocks.diffset_node_getter(commit1),
+                        target_node_getter=codeblocks.diffset_node_getter(commit2))
+
+                    diffset = self.path_filter.render_patchset_filtered(
+                        diffset, _parsed, commit1.raw_id, commit2.raw_id)
+
+                    # save cached diff
+                    if caching_enabled:
+                        cache_diff(cache_file_path, diffset, None)
+
+                c.limited_diff = diffset.limited_diff
                 c.changes[commit.raw_id] = diffset
             else:
+                # TODO(marcink): no cache usage here...
+                _diff = self.rhodecode_vcs_repo.get_diff(
+                    commit1, commit2,
+                    ignore_whitespace=ign_whitespace_lcl, context=context_lcl)
+                diff_processor = diffs.DiffProcessor(
+                    _diff, format='newdiff', diff_limit=diff_limit,
+                    file_limit=file_limit, show_full_diff=c.fulldiff)
                 # downloads/raw we only need RAW diff nothing else
-                diff = diff_processor.as_raw()
+                diff = self.path_filter.get_raw_patch(diff_processor)
                 c.changes[commit.raw_id] = [None, None, None, None, diff, None, None]
 
         # sort comments by how they were generated

rhodecode/apps/repository/views/repo_compare.py

@@ -295,22 +295,13 @@ class RepoCompareView(RepoAppView):
             file_limit=file_limit, show_full_diff=c.fulldiff)
         _parsed = diff_processor.prepare()
 
-        def _node_getter(commit):
-            """ Returns a function that returns a node for a commit or None """
-            def get_node(fname):
-                try:
-                    return commit.get_node(fname)
-                except NodeDoesNotExistError:
-                    return None
-            return get_node
-
         diffset = codeblocks.DiffSet(
             repo_name=source_repo.repo_name,
-            source_node_getter=_node_getter(source_commit),
-            target_node_getter=_node_getter(target_commit),
+            source_node_getter=codeblocks.diffset_node_getter(source_commit),
+            target_node_getter=codeblocks.diffset_node_getter(target_commit),
         )
-        c.diffset = diffset.render_patchset(
-            _parsed, source_ref, target_ref)
+        c.diffset = self.path_filter.render_patchset_filtered(
+            diffset, _parsed, source_ref, target_ref)
 
         c.preview_mode = merge
         c.source_commit = source_commit

rhodecode/apps/repository/views/repo_feed.py

@@ -90,13 +90,15 @@ class RepoFeedView(RepoAppView):
         _renderer = self.request.get_partial_renderer(
             'rhodecode:templates/feed/atom_feed_entry.mako')
         diff_processor, parsed_diff, limited_diff = self._changes(commit)
+        filtered_parsed_diff, has_hidden_changes = self.path_filter.filter_patchset(parsed_diff)
         return _renderer(
             'body',
             commit=commit,
-            parsed_diff=parsed_diff,
+            parsed_diff=filtered_parsed_diff,
             limited_diff=limited_diff,
             feed_include_diff=self.feed_include_diff,
             diff_processor=diff_processor,
+            has_hidden_changes=has_hidden_changes
         )
 
     def _set_timezone(self, date, tzinfo=pytz.utc):
@@ -122,8 +124,7 @@ class RepoFeedView(RepoAppView):
         """
         self.load_default_context()
 
-        @cache_region('long_term')
-        def _generate_feed(cache_key):
+        def _generate_feed():
             feed = Atom1Feed(
                 title=self.title % self.db_repo_name,
                 link=h.route_url('repo_summary', repo_name=self.db_repo_name),
@@ -146,12 +147,18 @@ class RepoFeedView(RepoAppView):
 
             return feed.mime_type, feed.writeString('utf-8')
 
-        invalidator_context = CacheKey.repo_context_cache(
-            _generate_feed, self.db_repo_name, CacheKey.CACHE_TYPE_ATOM)
+        @cache_region('long_term')
+        def _generate_feed_and_cache(cache_key):
+            return _generate_feed()
 
-        with invalidator_context as context:
-            context.invalidate()
-            mime_type, feed = context.compute()
+        if self.path_filter.is_enabled:
+            invalidator_context = CacheKey.repo_context_cache(
+                _generate_feed_and_cache, self.db_repo_name, CacheKey.CACHE_TYPE_ATOM)
+            with invalidator_context as context:
+                context.invalidate()
+                mime_type, feed = context.compute()
+        else:
+            mime_type, feed = _generate_feed()
 
         response = Response(feed)
         response.content_type = mime_type
@@ -169,8 +176,7 @@ class RepoFeedView(RepoAppView):
         """
         self.load_default_context()
 
-        @cache_region('long_term')
-        def _generate_feed(cache_key):
+        def _generate_feed():
             feed = Rss201rev2Feed(
                 title=self.title % self.db_repo_name,
                 link=h.route_url('repo_summary', repo_name=self.db_repo_name),
@@ -193,12 +199,19 @@ class RepoFeedView(RepoAppView):
 
             return feed.mime_type, feed.writeString('utf-8')
 
-        invalidator_context = CacheKey.repo_context_cache(
-            _generate_feed, self.db_repo_name, CacheKey.CACHE_TYPE_RSS)
+        @cache_region('long_term')
+        def _generate_feed_and_cache(cache_key):
+            return _generate_feed()
 
-        with invalidator_context as context:
-            context.invalidate()
-            mime_type, feed = context.compute()
+        if self.path_filter.is_enabled:
+            invalidator_context = CacheKey.repo_context_cache(
+                _generate_feed_and_cache, self.db_repo_name, CacheKey.CACHE_TYPE_RSS)
+
+            with invalidator_context as context:
+                context.invalidate()
+                mime_type, feed = context.compute()
+        else:
+            mime_type, feed = _generate_feed()
 
         response = Response(feed)
         response.content_type = mime_type

rhodecode/apps/repository/views/repo_files.py

@@ -426,7 +426,7 @@ class RepoFilesView(RepoAppView):
                                       context=line_context)
             diff = diffs.DiffProcessor(_diff, format='gitdiff')
 
-            response = Response(diff.as_raw())
+            response = Response(self.path_filter.get_raw_patch(diff))
             response.content_type = 'text/plain'
             response.content_disposition = (
                 'attachment; filename=%s_%s_vs_%s.diff' % (f_path, diff1, diff2)
@@ -442,7 +442,7 @@ class RepoFilesView(RepoAppView):
                                       context=line_context)
             diff = diffs.DiffProcessor(_diff, format='gitdiff')
 
-            response = Response(diff.as_raw())
+            response = Response(self.path_filter.get_raw_patch(diff))
             response.content_type = 'text/plain'
             charset = self._get_default_encoding(c)
             if charset:
@@ -462,7 +462,7 @@ class RepoFilesView(RepoAppView):
         """
         Kept only to make OLD links work
         """
-        f_path = self._get_f_path(self.request.matchdict)
+        f_path = self._get_f_path_unchecked(self.request.matchdict)
         diff1 = self.request.GET.get('diff1', '')
         diff2 = self.request.GET.get('diff2', '')
 

rhodecode/apps/repository/views/repo_pull_requests.py

@@ -34,6 +34,7 @@ from rhodecode.apps._base import RepoAppView, DataGridAppView
 
 from rhodecode.lib import helpers as h, diffs, codeblocks, channelstream
 from rhodecode.lib.base import vcs_operation_context
+from rhodecode.lib.diffs import load_cached_diff, cache_diff, diff_cache_exist
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.auth import (
     LoginRequired, HasRepoPermissionAny, HasRepoPermissionAnyDecorator,
@@ -41,7 +42,7 @@ from rhodecode.lib.auth import (
 from rhodecode.lib.utils2 import str2bool, safe_str, safe_unicode
 from rhodecode.lib.vcs.backends.base import EmptyCommit, UpdateFailureReason
 from rhodecode.lib.vcs.exceptions import (CommitDoesNotExistError,
-    RepositoryRequirementError, NodeDoesNotExistError, EmptyRepositoryError)
+    RepositoryRequirementError, EmptyRepositoryError)
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.comment import CommentsModel
 from rhodecode.model.db import (func, or_, PullRequest, PullRequestVersion,
@@ -201,10 +202,16 @@ class RepoPullRequestsView(RepoAppView, DataGridAppView):
 
         return data
 
+    def _is_diff_cache_enabled(self, target_repo):
+        caching_enabled = self._get_general_setting(
+            target_repo, 'rhodecode_diff_cache')
+        log.debug('Diff caching enabled: %s', caching_enabled)
+        return caching_enabled
+
     def _get_diffset(self, source_repo_name, source_repo,
                      source_ref_id, target_ref_id,
-                     target_commit, source_commit, diff_limit, fulldiff,
-                     file_limit, display_inline_comments):
+                     target_commit, source_commit, diff_limit, file_limit,
+                     fulldiff):
 
         vcs_diff = PullRequestModel().get_diff(
             source_repo, source_ref_id, target_ref_id)
@@ -215,24 +222,14 @@ class RepoPullRequestsView(RepoAppView, DataGridAppView):
 
         _parsed = diff_processor.prepare()
 
-        def _node_getter(commit):
-            def get_node(fname):
-                try:
-                    return commit.get_node(fname)
-                except NodeDoesNotExistError:
-                    return None
-
-            return get_node
-
         diffset = codeblocks.DiffSet(
             repo_name=self.db_repo_name,
             source_repo_name=source_repo_name,
-            source_node_getter=_node_getter(target_commit),
-            target_node_getter=_node_getter(source_commit),
-            comments=display_inline_comments
+            source_node_getter=codeblocks.diffset_node_getter(target_commit),
+            target_node_getter=codeblocks.diffset_node_getter(source_commit),
         )
-        diffset = diffset.render_patchset(
-            _parsed, target_commit.raw_id, source_commit.raw_id)
+        diffset = self.path_filter.render_patchset_filtered(
+            diffset, _parsed, target_commit.raw_id, source_commit.raw_id)
 
         return diffset
 
@@ -443,42 +440,54 @@ class RepoPullRequestsView(RepoAppView, DataGridAppView):
             commits_source_repo = source_scm
 
         c.commits_source_repo = commits_source_repo
-        commit_cache = {}
-        try:
-            pre_load = ["author", "branch", "date", "message"]
-            show_revs = pull_request_at_ver.revisions
-            for rev in show_revs:
-                comm = commits_source_repo.get_commit(
-                    commit_id=rev, pre_load=pre_load)
-                c.commit_ranges.append(comm)
-                commit_cache[comm.raw_id] = comm
-
-            # Order here matters, we first need to get target, and then
-            # the source
-            target_commit = commits_source_repo.get_commit(
-                commit_id=safe_str(target_ref_id))
-
-            source_commit = commits_source_repo.get_commit(
-                commit_id=safe_str(source_ref_id))
-
-        except CommitDoesNotExistError:
-            log.warning(
-                'Failed to get commit from `{}` repo'.format(
-                    commits_source_repo), exc_info=True)
-        except RepositoryRequirementError:
-            log.warning(
-                'Failed to get all required data from repo', exc_info=True)
-            c.missing_requirements = True
-
         c.ancestor = None  # set it to None, to hide it from PR view
 
-        try:
-            ancestor_id = source_scm.get_common_ancestor(
-                source_commit.raw_id, target_commit.raw_id, target_scm)
-            c.ancestor_commit = source_scm.get_commit(ancestor_id)
-        except Exception:
-            c.ancestor_commit = None
+        # empty version means latest, so we keep this to prevent
+        # double caching
+        version_normalized = version or 'latest'
+        from_version_normalized = from_version or 'latest'
 
+        cache_path = self.rhodecode_vcs_repo.get_create_shadow_cache_pr_path(
+            target_repo)
+        cache_file_path = diff_cache_exist(
+            cache_path, 'pull_request', pull_request_id, version_normalized,
+            from_version_normalized, source_ref_id, target_ref_id, c.fulldiff)
+
+        caching_enabled = self._is_diff_cache_enabled(c.target_repo)
+        force_recache = str2bool(self.request.GET.get('force_recache'))
+
+        cached_diff = None
+        if caching_enabled:
+            cached_diff = load_cached_diff(cache_file_path)
+
+        has_proper_commit_cache = (
+                cached_diff and cached_diff.get('commits')
+                and len(cached_diff.get('commits', [])) == 5
+                and cached_diff.get('commits')[0]
+                and cached_diff.get('commits')[3])
+        if not force_recache and has_proper_commit_cache:
+            diff_commit_cache = \
+                (ancestor_commit, commit_cache, missing_requirements,
+                 source_commit, target_commit) = cached_diff['commits']
+        else:
+            diff_commit_cache = \
+                (ancestor_commit, commit_cache, missing_requirements,
+                 source_commit, target_commit) = self.get_commits(
+                    commits_source_repo,
+                    pull_request_at_ver,
+                    source_commit,
+                    source_ref_id,
+                    source_scm,
+                    target_commit,
+                    target_ref_id,
+                    target_scm)
+
+        # register our commit range
+        for comm in commit_cache.values():
+            c.commit_ranges.append(comm)
+
+        c.missing_requirements = missing_requirements
+        c.ancestor_commit = ancestor_commit
         c.statuses = source_repo.statuses(
             [x.raw_id for x in c.commit_ranges])
 
@@ -500,12 +509,23 @@ class RepoPullRequestsView(RepoAppView, DataGridAppView):
 
             c.missing_commits = True
         else:
+            c.inline_comments = display_inline_comments
 
-            c.diffset = self._get_diffset(
-                c.source_repo.repo_name, commits_source_repo,
-                source_ref_id, target_ref_id,
-                target_commit, source_commit,
-                diff_limit, c.fulldiff, file_limit, display_inline_comments)
+            has_proper_diff_cache = cached_diff and cached_diff.get('commits')
+            if not force_recache and has_proper_diff_cache:
+                c.diffset = cached_diff['diff']
+                (ancestor_commit, commit_cache, missing_requirements,
+                 source_commit, target_commit) = cached_diff['commits']
+            else:
+                c.diffset = self._get_diffset(
+                    c.source_repo.repo_name, commits_source_repo,
+                    source_ref_id, target_ref_id,
+                    target_commit, source_commit,
+                    diff_limit, file_limit, c.fulldiff)
+
+                # save cached diff
+                if caching_enabled:
+                    cache_diff(cache_file_path, c.diffset, diff_commit_cache)
 
             c.limited_diff = c.diffset.limited_diff
 
@@ -568,7 +588,6 @@ class RepoPullRequestsView(RepoAppView, DataGridAppView):
             if self._rhodecode_user.user_id in allowed_reviewers:
                 for co in general_comments:
                     if co.author.user_id == self._rhodecode_user.user_id:
-                        # each comment has a status change
                         status = co.status_change
                         if status:
                             _ver_pr = status[0].comment.pull_request_version_id
@@ -576,6 +595,43 @@ class RepoPullRequestsView(RepoAppView, DataGridAppView):
 
         return self._get_template_context(c)
 
+    def get_commits(
+            self, commits_source_repo, pull_request_at_ver, source_commit,
+            source_ref_id, source_scm, target_commit, target_ref_id, target_scm):
+        commit_cache = collections.OrderedDict()
+        missing_requirements = False
+        try:
+            pre_load = ["author", "branch", "date", "message"]
+            show_revs = pull_request_at_ver.revisions
+            for rev in show_revs:
+                comm = commits_source_repo.get_commit(
+                    commit_id=rev, pre_load=pre_load)
+                commit_cache[comm.raw_id] = comm
+
+            # Order here matters, we first need to get target, and then
+            # the source
+            target_commit = commits_source_repo.get_commit(
+                commit_id=safe_str(target_ref_id))
+
+            source_commit = commits_source_repo.get_commit(
+                commit_id=safe_str(source_ref_id))
+        except CommitDoesNotExistError:
+            log.warning(
+                'Failed to get commit from `{}` repo'.format(
+                    commits_source_repo), exc_info=True)
+        except RepositoryRequirementError:
+            log.warning(
+                'Failed to get all required data from repo', exc_info=True)
+            missing_requirements = True
+        ancestor_commit = None
+        try:
+            ancestor_id = source_scm.get_common_ancestor(
+                source_commit.raw_id, target_commit.raw_id, target_scm)
+            ancestor_commit = source_scm.get_commit(ancestor_id)
+        except Exception:
+            ancestor_commit = None
+        return ancestor_commit, commit_cache, missing_requirements, source_commit, target_commit
+
     def assure_not_empty_repo(self):
         _ = self.request.translate
 

rhodecode/apps/repository/views/repo_settings.py

@@ -138,15 +138,19 @@ class RepoSettingsView(RepoAppView):
             repo_description=schema_data['repo_description'],
             repo_private=schema_data['repo_private'],
             clone_uri=schema_data['repo_clone_uri'],
+            push_uri=schema_data['repo_push_uri'],
             repo_landing_rev=schema_data['repo_landing_commit_ref'],
             repo_enable_statistics=schema_data['repo_enable_statistics'],
             repo_enable_locking=schema_data['repo_enable_locking'],
             repo_enable_downloads=schema_data['repo_enable_downloads'],
         )
-        # detect if CLONE URI changed, if we get OLD means we keep old values
+        # detect if SYNC URI changed, if we get OLD means we keep old values
         if schema_data['repo_clone_uri_change'] == 'OLD':
             validated_updates['clone_uri'] = self.db_repo.clone_uri
 
+        if schema_data['repo_push_uri_change'] == 'OLD':
+            validated_updates['push_uri'] = self.db_repo.push_uri
+
         # use the new full name for redirect
         new_repo_name = schema_data['repo_group']['repo_name_with_group']
 

rhodecode/apps/repository/views/repo_settings_advanced.py

@@ -43,8 +43,6 @@ class RepoSettingsView(RepoAppView):
 
     def load_default_context(self):
         c = self._get_local_tmpl_context()
-
-
         return c
 
     @LoginRequired()
@@ -231,3 +229,19 @@ class RepoSettingsView(RepoAppView):
 
         raise HTTPFound(
             h.route_path('edit_repo_advanced', repo_name=self.db_repo_name))
+
+    @LoginRequired()
+    @HasRepoPermissionAnyDecorator('repository.admin')
+    @view_config(
+        route_name='edit_repo_advanced_hooks', request_method='GET',
+        renderer='rhodecode:templates/admin/repos/repo_edit.mako')
+    def edit_advanced_install_hooks(self):
+        """
+        Install Hooks for repository
+        """
+        _ = self.request.translate
+        self.load_default_context()
+        self.rhodecode_vcs_repo.install_hooks(force=True)
+        h.flash(_('installed hooks repository'), category='success')
+        raise HTTPFound(
+            h.route_path('edit_repo_advanced', repo_name=self.db_repo_name))

rhodecode/apps/repository/views/repo_settings_remote.py

@@ -35,8 +35,6 @@ log = logging.getLogger(__name__)
 class RepoSettingsRemoteView(RepoAppView):
     def load_default_context(self):
         c = self._get_local_tmpl_context()
-
-
         return c
 
     @LoginRequired()

rhodecode/apps/ssh_support/lib/backends/svn.py

@@ -117,6 +117,7 @@ class SubversionTunnelWrapper(object):
                 message=self._svn_string(message)))
         self.remove_configs()
         self.process.kill()
+        return 1
 
     def interrupt(self, signum, frame):
         self.fail("Exited by timeout")
@@ -171,7 +172,7 @@ class SubversionTunnelWrapper(object):
 
         first_response = self.get_first_client_response()
         if not first_response:
-            self.fail("Repository name cannot be extracted")
+            return self.fail("Repository name cannot be extracted")
 
         url_parts = urlparse.urlparse(first_response['url'])
         self.server.repo_name = url_parts.path.strip('/')

rhodecode/apps/ssh_support/lib/ssh_wrapper.py

@@ -68,7 +68,7 @@ def main(ini_path, mode, user, user_id, key_id, shell, debug):
             'of this script.')
     connection_info = os.environ.get('SSH_CONNECTION', '')
 
-    with bootstrap(ini_path) as env:
+    with bootstrap(ini_path, env={'RC_CMD_SSH_WRAPPER': '1'}) as env:
         try:
             ssh_wrapper = SshWrapper(
                 command, connection_info, mode,

rhodecode/apps/ssh_support/tests/test_server_svn.py

@@ -103,7 +103,7 @@ class TestSubversionServer(object):
                         return_value=0):
                     with mock.patch.object(
                             SubversionTunnelWrapper, 'command',
-                            return_value='date'):
+                            return_value=['date']):
 
                         exit_code = server.run()
         # SVN has this differently configured, and we get in our mock env
@@ -115,7 +115,7 @@ class TestSubversionServer(object):
         from rhodecode.apps.ssh_support.lib.backends.svn import SubversionTunnelWrapper
         with mock.patch.object(
                 SubversionTunnelWrapper, 'command',
-                return_value='date'):
+                return_value=['date']):
             with mock.patch.object(
                     SubversionTunnelWrapper, 'get_first_client_response',
                     return_value=None):

rhodecode/apps/svn_support/templates/mod-dav-svn.conf.mako

@@ -64,6 +64,9 @@ RequestHeader edit Destination ^https: http: early
     SVNParentPath "${parent_path_root|n}"
     SVNListParentPath ${"On" if svn_list_parent_path else "Off"|n}
 
+    # use specific SVN conf/authz file for each repository
+    #AuthzSVNReposRelativeAccessFile authz
+
     Allow from all
     Order allow,deny
 </Location>
@@ -82,6 +85,9 @@ RequestHeader edit Destination ^https: http: early
     SVNParentPath "${parent_path|n}"
     SVNListParentPath ${"On" if svn_list_parent_path else "Off"|n}
 
+    # use specific SVN conf/authz file for each repository
+    #AuthzSVNReposRelativeAccessFile authz
+
     Allow from all
     Order allow,deny
 </Location>

rhodecode/apps/user_group_profile/__init__.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2016-2018 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+
+def includeme(config):
+    config.add_route(
+        name='user_group_profile',
+        pattern='/_profile_user_group/{user_group_name}')
+    # Scan module for configuration decorators.
+    config.scan('.views', ignore='.tests')

rhodecode/apps/user_group_profile/tests/__init__.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2016-2018 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/

rhodecode/apps/user_group_profile/tests/test_user_group.py

@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2018 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+from rhodecode.model.user_group import UserGroupModel
+from rhodecode.tests import (
+    TestController, TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS)
+from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.utils import AssertResponse
+
+fixture = Fixture()
+
+
+def route_path(name, **kwargs):
+    return '/_profile_user_group/{user_group_name}'.format(**kwargs)
+
+
+class TestUsersController(TestController):
+
+    def test_user_group_profile(self, user_util):
+        self.log_user(TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS)
+        user, usergroup = user_util.create_user_with_group()
+
+        response = self.app.get(route_path('profile_user_group', user_group_name=usergroup.users_group_name))
+        response.mustcontain(usergroup.users_group_name)
+        response.mustcontain(user.username)
+
+    def test_user_can_check_own_group(self, user_util):
+        user = user_util.create_user(
+            TEST_USER_REGULAR_LOGIN, password=TEST_USER_REGULAR_PASS, email='testme@rhodecode.org')
+        self.log_user(TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS)
+        usergroup = user_util.create_user_group(owner=user)
+        response = self.app.get(route_path('profile_user_group', user_group_name=usergroup.users_group_name))
+        response.mustcontain(usergroup.users_group_name)
+        response.mustcontain(user.username)
+
+    def test_user_can_not_check_other_group(self, user_util):
+        self.log_user(TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS)
+        user_group = user_util.create_user_group()
+        UserGroupModel().grant_user_permission(user_group, self._get_logged_user(), 'usergroup.none')
+        response = self.app.get(route_path('profile_user_group', user_group_name=user_group.users_group_name), status=404)
+        assert response.status_code == 404
+
+    def test_another_user_can_check_if_he_is_in_group(self, user_util):
+        self.log_user(TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS)
+        user = user_util.create_user(
+            'test-my-user', password='qweqwe', email='testme@rhodecode.org')
+        user_group = user_util.create_user_group()
+        UserGroupModel().add_user_to_group(user_group, user)
+        UserGroupModel().grant_user_permission(user_group, self._get_logged_user(), 'usergroup.read')
+        response = self.app.get(route_path('profile_user_group', user_group_name=user_group.users_group_name))
+        response.mustcontain(user_group.users_group_name)
+        response.mustcontain(user.username)
+
+    def test_with_anonymous_user(self, user_util):
+        user = user_util.create_user(
+            'test-my-user', password='qweqwe', email='testme@rhodecode.org')
+        user_group = user_util.create_user_group()
+        UserGroupModel().add_user_to_group(user_group, user)
+        response = self.app.get(route_path('profile_user_group', user_group_name=user_group.users_group_name), status=302)
+        assert response.status_code == 302

rhodecode/apps/user_group_profile/views.py

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2016-2018 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import logging
+
+from pyramid.httpexceptions import HTTPNotFound
+from pyramid.view import view_config
+
+from rhodecode.apps._base import BaseAppView
+from rhodecode.lib.auth import HasUserGroupPermissionAnyDecorator, LoginRequired, NotAnonymous
+from rhodecode.model.db import UserGroup, User
+
+
+log = logging.getLogger(__name__)
+
+
+class UserGroupProfileView(BaseAppView):
+
+    @LoginRequired()
+    @NotAnonymous()
+    @HasUserGroupPermissionAnyDecorator('usergroup.read', 'usergroup.write', 'usergroup.admin',)
+    @view_config(
+        route_name='user_group_profile', request_method='GET',
+        renderer='rhodecode:templates/user_group/user_group.mako')
+    def user_group_profile(self):
+        c = self._get_local_tmpl_context()
+        c.active = 'profile'
+        self.db_user_group_name = self.request.matchdict.get('user_group_name')
+        c.user_group = UserGroup().get_by_group_name(self.db_user_group_name)
+        if not c.user_group:
+            raise HTTPNotFound()
+        group_members_obj = sorted((x.user for x in c.user_group.members),
+                                   key=lambda u: u.username.lower())
+        c.group_members = group_members_obj
+        c.anonymous = self._rhodecode_user.username == User.DEFAULT_USER
+        return self._get_template_context(c)

rhodecode/authentication/__init__.py

@@ -18,6 +18,7 @@
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
+import os
 import logging
 import importlib
 
@@ -71,9 +72,12 @@ def _discover_legacy_plugins(config, prefix=legacy_plugin_prefix):
     setting in database which are using the specified prefix. Normally 'py:' is
     used for the legacy plugins.
     """
-    auth_plugins = SettingsModel().get_setting_by_name('auth_plugins')
-    enabled_plugins = auth_plugins.app_settings_value
-    legacy_plugins = [id_ for id_ in enabled_plugins if id_.startswith(prefix)]
+    try:
+        auth_plugins = SettingsModel().get_setting_by_name('auth_plugins')
+        enabled_plugins = auth_plugins.app_settings_value
+        legacy_plugins = [id_ for id_ in enabled_plugins if id_.startswith(prefix)]
+    except Exception:
+        legacy_plugins = []
 
     for plugin_id in legacy_plugins:
         log.debug('Legacy plugin discovered: "%s"', plugin_id)
@@ -117,6 +121,11 @@ def includeme(config):
                     route_name='auth_home',
                     context=AuthnRootResource)
 
+    for key in ['RC_CMD_SETUP_RC', 'RC_CMD_UPGRADE_DB', 'RC_CMD_SSH_WRAPPER']:
+        if os.environ.get(key):
+            # skip this heavy step below on certain CLI commands
+            return
+
     # Auto discover authentication plugins and include their configuration.
     _discover_plugins(config)
     _discover_legacy_plugins(config)

rhodecode/authentication/base.py

@@ -171,11 +171,6 @@ class RhodeCodeAuthPluginBase(object):
             db_type = '{}.encrypted'.format(db_type)
         return db_type
 
-    @LazyProperty
-    def plugin_settings(self):
-        settings = SettingsModel().get_all_settings()
-        return settings
-
     def is_enabled(self):
         """
         Returns true if this plugin is enabled. An enabled plugin can be
@@ -185,12 +180,13 @@ class RhodeCodeAuthPluginBase(object):
         auth_plugins = SettingsModel().get_auth_plugins()
         return self.get_id() in auth_plugins
 
-    def is_active(self):
+    def is_active(self, plugin_cached_settings=None):
         """
         Returns true if the plugin is activated. An activated plugin is
         consulted during authentication, assumed it is also enabled.
         """
-        return self.get_setting_by_name('enabled')
+        return self.get_setting_by_name(
+            'enabled', plugin_cached_settings=plugin_cached_settings)
 
     def get_id(self):
         """
@@ -210,13 +206,24 @@ class RhodeCodeAuthPluginBase(object):
         """
         return AuthnPluginSettingsSchemaBase()
 
-    def get_setting_by_name(self, name, default=None, cache=True):
+    def get_settings(self):
+        """
+        Returns the plugin settings as dictionary.
+        """
+        settings = {}
+        raw_settings = SettingsModel().get_all_settings()
+        for node in self.get_settings_schema():
+            settings[node.name] = self.get_setting_by_name(
+                node.name, plugin_cached_settings=raw_settings)
+        return settings
+
+    def get_setting_by_name(self, name, default=None, plugin_cached_settings=None):
         """
         Returns a plugin setting by name.
         """
         full_name = 'rhodecode_{}'.format(self._get_setting_full_name(name))
-        if cache:
-            plugin_settings = self.plugin_settings
+        if plugin_cached_settings:
+            plugin_settings = plugin_cached_settings
         else:
             plugin_settings = SettingsModel().get_all_settings()
 
@@ -235,15 +242,6 @@ class RhodeCodeAuthPluginBase(object):
             full_name, value, type_)
         return db_setting.app_settings_value
 
-    def get_settings(self):
-        """
-        Returns the plugin settings as dictionary.
-        """
-        settings = {}
-        for node in self.get_settings_schema():
-            settings[node.name] = self.get_setting_by_name(node.name)
-        return settings
-
     def log_safe_settings(self, settings):
         """
         returns a log safe representation of settings, without any secrets
@@ -625,12 +623,13 @@ def authenticate(username, password, environ=None, auth_type=None,
                       'headers plugin, skipping...', plugin.get_id())
             continue
 
+        log.debug('Trying authentication using ** %s **', plugin.get_id())
+
         # load plugin settings from RhodeCode database
         plugin_settings = plugin.get_settings()
         plugin_sanitized_settings = plugin.log_safe_settings(plugin_settings)
-        log.debug('Plugin settings:%s', plugin_sanitized_settings)
+        log.debug('Plugin `%s` settings:%s', plugin.get_id(), plugin_sanitized_settings)
 
-        log.debug('Trying authentication using ** %s **', plugin.get_id())
         # use plugin's method of user extraction.
         user = plugin.get_user(username, environ=environ,
                                settings=plugin_settings)
@@ -684,7 +683,8 @@ def authenticate(username, password, environ=None, auth_type=None,
                 environ=environ or {})
 
         if plugin_cache_active:
-            log.debug('Trying to fetch cached auth by `...%s`', _password_hash[:6])
+            log.debug('Trying to fetch cached auth by pwd hash `...%s`',
+                      _password_hash[:6])
             plugin_user = cache_manager.get(
                 _password_hash, createfunc=auth_func)
         else:

rhodecode/authentication/plugins/auth_crowd.py

@@ -281,5 +281,5 @@ class RhodeCodeAuthPlugin(RhodeCodeExternalAuthPlugin):
             if group in user_attrs["groups"]:
                 user_attrs["admin"] = True
         log.debug("Final crowd user object: \n%s" % (formatted_json(user_attrs)))
-        log.info('user %s authenticated correctly' % user_attrs['username'])
+        log.info('user `%s` authenticated correctly' % user_attrs['username'])
         return user_attrs

rhodecode/authentication/plugins/auth_jasig_cas.py

@@ -163,5 +163,5 @@ class RhodeCodeAuthPlugin(RhodeCodeExternalAuthPlugin):
             'extern_type': extern_type,
         }
 
-        log.info('user %s authenticated correctly' % user_attrs['username'])
+        log.info('user `%s` authenticated correctly' % user_attrs['username'])
         return user_attrs

rhodecode/authentication/plugins/auth_ldap.py

@@ -22,10 +22,12 @@
 RhodeCode authentication plugin for LDAP
 """
 
-
+import socket
+import re
 import colander
 import logging
 import traceback
+import string
 
 from rhodecode.translation import _
 from rhodecode.authentication.base import (
@@ -50,6 +52,9 @@ except ImportError:
     ldap = Missing
 
 
+class LdapError(Exception):
+    pass
+
 def plugin_factory(plugin_id, *args, **kwds):
     """
     Factory function that is called during plugin discovery.
@@ -86,6 +91,16 @@ class LdapSettingsSchema(AuthnPluginSettingsSchemaBase):
         title=_('Port'),
         validator=colander.Range(min=0, max=65536),
         widget='int')
+
+    timeout = colander.SchemaNode(
+        colander.Int(),
+        default=60 * 5,
+        description=_('Timeout for LDAP connection'),
+        preparer=strip_whitespace,
+        title=_('Connection timeout'),
+        validator=colander.Range(min=1),
+        widget='int')
+
     dn_user = colander.SchemaNode(
         colander.String(),
         default='',
@@ -187,19 +202,47 @@ class LdapSettingsSchema(AuthnPluginSettingsSchemaBase):
 class AuthLdap(object):
 
     def _build_servers(self):
+        def host_resolver(host, port):
+            """
+            Main work for this function is to prevent ldap connection issues,
+            and detect them early using a "greenified" sockets
+            """
+            host = host.strip()
+
+            log.info('Resolving LDAP host %s', host)
+            try:
+                ip = socket.gethostbyname(host)
+                log.info('Got LDAP server %s ip %s', host, ip)
+            except Exception:
+                raise LdapConnectionError(
+                    'Failed to resolve host: `{}`'.format(host))
+
+            log.info('Checking LDAP IP access %s', ip)
+            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            try:
+                s.connect((ip, int(port)))
+                s.shutdown(socket.SHUT_RD)
+            except Exception:
+                raise LdapConnectionError(
+                    'Failed to connect to host: `{}:{}`'.format(host, port))
+
+            return '{}:{}'.format(host, port)
+
+        port = self.LDAP_SERVER_PORT
         return ', '.join(
-            ["{}://{}:{}".format(
-                self.ldap_server_type, host.strip(), self.LDAP_SERVER_PORT)
+            ["{}://{}".format(
+                self.ldap_server_type, host_resolver(host, port))
              for host in self.SERVER_ADDRESSES])
 
     def __init__(self, server, base_dn, port=389, bind_dn='', bind_pass='',
                  tls_kind='PLAIN', tls_reqcert='DEMAND', ldap_version=3,
                  search_scope='SUBTREE', attr_login='uid',
-                 ldap_filter=''):
+                 ldap_filter='', timeout=None):
         if ldap == Missing:
             raise LdapImportError("Missing or incompatible ldap library")
 
         self.debug = False
+        self.timeout = timeout or 60 * 5
         self.ldap_version = ldap_version
         self.ldap_server_type = 'ldap'
 
@@ -226,34 +269,40 @@ class AuthLdap(object):
         self.BASE_DN = safe_str(base_dn)
         self.LDAP_FILTER = safe_str(ldap_filter)
 
-    def _get_ldap_server(self):
+    def _get_ldap_conn(self):
+        log.debug('initializing LDAP connection to:%s', self.LDAP_SERVER)
+
         if self.debug:
             ldap.set_option(ldap.OPT_DEBUG_LEVEL, 255)
-        if hasattr(ldap, 'OPT_X_TLS_CACERTDIR'):
-            ldap.set_option(ldap.OPT_X_TLS_CACERTDIR,
-                            '/etc/openldap/cacerts')
-        ldap.set_option(ldap.OPT_REFERRALS, ldap.OPT_OFF)
-        ldap.set_option(ldap.OPT_RESTART, ldap.OPT_ON)
-        ldap.set_option(ldap.OPT_NETWORK_TIMEOUT, 60 * 10)
-        ldap.set_option(ldap.OPT_TIMEOUT, 60 * 10)
 
+        if hasattr(ldap, 'OPT_X_TLS_CACERTDIR'):
+            ldap.set_option(ldap.OPT_X_TLS_CACERTDIR, '/etc/openldap/cacerts')
         if self.TLS_KIND != 'PLAIN':
             ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, self.TLS_REQCERT)
-        server = ldap.initialize(self.LDAP_SERVER)
+
+        ldap.set_option(ldap.OPT_REFERRALS, ldap.OPT_OFF)
+        ldap.set_option(ldap.OPT_RESTART, ldap.OPT_ON)
+
+        # init connection now
+        ldap_conn = ldap.initialize(self.LDAP_SERVER)
+        ldap_conn.set_option(ldap.OPT_NETWORK_TIMEOUT, self.timeout)
+        ldap_conn.set_option(ldap.OPT_TIMEOUT, self.timeout)
+        ldap_conn.timeout = self.timeout
+
         if self.ldap_version == 2:
-            server.protocol = ldap.VERSION2
+            ldap_conn.protocol = ldap.VERSION2
         else:
-            server.protocol = ldap.VERSION3
+            ldap_conn.protocol = ldap.VERSION3
 
         if self.TLS_KIND == 'START_TLS':
-            server.start_tls_s()
+            ldap_conn.start_tls_s()
 
         if self.LDAP_BIND_DN and self.LDAP_BIND_PASS:
             log.debug('Trying simple_bind with password and given login DN: %s',
                       self.LDAP_BIND_DN)
-            server.simple_bind_s(self.LDAP_BIND_DN, self.LDAP_BIND_PASS)
+            ldap_conn.simple_bind_s(self.LDAP_BIND_DN, self.LDAP_BIND_PASS)
 
-        return server
+        return ldap_conn
 
     def get_uid(self, username):
         uid = username
@@ -295,13 +344,14 @@ class AuthLdap(object):
         if "," in username:
             raise LdapUsernameError(
                 "invalid character `,` in username: `{}`".format(username))
+        ldap_conn = None
         try:
-            server = self._get_ldap_server()
+            ldap_conn = self._get_ldap_conn()
             filter_ = '(&%s(%s=%s))' % (
                 self.LDAP_FILTER, self.attr_login, username)
             log.debug("Authenticating %r filter %s at %s", self.BASE_DN,
                       filter_, self.LDAP_SERVER)
-            lobjects = server.search_ext_s(
+            lobjects = ldap_conn.search_ext_s(
                 self.BASE_DN, self.SEARCH_SCOPE, filter_)
 
             if not lobjects:
@@ -315,7 +365,7 @@ class AuthLdap(object):
                     continue
 
                 user_attrs = self.fetch_attrs_from_simple_bind(
-                    server, dn, username, password)
+                    ldap_conn, dn, username, password)
                 if user_attrs:
                     break
 
@@ -333,6 +383,15 @@ class AuthLdap(object):
             raise LdapConnectionError(
                 "LDAP can't access authentication "
                 "server, org_exc:%s" % org_exc)
+        finally:
+            if ldap_conn:
+                log.debug('ldap: connection release')
+                try:
+                    ldap_conn.unbind_s()
+                except Exception:
+                    # for any reason this can raise exception we must catch it
+                    # to not crush the server
+                    pass
 
         return dn, user_attrs
 
@@ -429,6 +488,7 @@ class RhodeCodeAuthPlugin(RhodeCodeExternalAuthPlugin):
             'attr_login': settings.get('attr_login'),
             'ldap_version': 3,
             'ldap_filter': settings.get('filter'),
+            'timeout': settings.get('timeout')
         }
 
         ldap_attrs = self.try_dynamic_binding(username, password, ldap_args)
@@ -469,7 +529,7 @@ class RhodeCodeAuthPlugin(RhodeCodeExternalAuthPlugin):
                 'extern_type': extern_type,
             }
             log.debug('ldap user: %s', user_attrs)
-            log.info('user %s authenticated correctly', user_attrs['username'])
+            log.info('user `%s` authenticated correctly', user_attrs['username'])
 
             return user_attrs
 
@@ -479,3 +539,4 @@ class RhodeCodeAuthPlugin(RhodeCodeExternalAuthPlugin):
         except (Exception,):
             log.exception("Other exception")
             return None
+

rhodecode/authentication/plugins/auth_pam.py

@@ -157,5 +157,5 @@ class RhodeCodeAuthPlugin(RhodeCodeExternalAuthPlugin):
             pass
 
         log.debug("pamuser: %s", user_attrs)
-        log.info('user %s authenticated correctly' % user_attrs['username'])
+        log.info('user `%s` authenticated correctly' % user_attrs['username'])
         return user_attrs

rhodecode/authentication/plugins/auth_rhodecode.py

@@ -127,14 +127,14 @@ class RhodeCodeAuthPlugin(RhodeCodeAuthPluginBase):
 
             if userobj.username == User.DEFAULT_USER and userobj.active:
                 log.info(
-                    'user %s authenticated correctly as anonymous user', userobj)
+                    'user `%s` authenticated correctly as anonymous user', userobj.username)
                 return user_attrs
 
             elif userobj.username == username and password_match:
-                log.info('user %s authenticated correctly', userobj)
+                log.info('user `%s` authenticated correctly', userobj.username)
                 return user_attrs
-            log.info("user %s had a bad password when "
-                     "authenticating on this plugin", userobj)
+            log.warn("user `%s` used a wrong password when "
+                     "authenticating on this plugin", userobj.username)
             return None
         else:
             log.warning(

rhodecode/authentication/plugins/auth_token.py

@@ -137,7 +137,7 @@ class RhodeCodeAuthPlugin(RhodeCodeAuthPluginBase):
                     'user `%s` successfully authenticated via %s',
                     user_attrs['username'], self.name)
                 return user_attrs
-            log.error(
+            log.warn(
                 'user `%s` failed to authenticate via %s, reason: bad or '
                 'inactive token.', username, self.name)
         else:

rhodecode/authentication/registry.py

@@ -70,9 +70,11 @@ class AuthenticationPluginRegistry(object):
         # Add all enabled and active plugins to the list. We iterate over the
         # auth_plugins setting from DB because it also represents the ordering.
         enabled_plugins = SettingsModel().get_auth_plugins()
+        raw_settings = SettingsModel().get_all_settings()
         for plugin_id in enabled_plugins:
             plugin = self.get_plugin(plugin_id)
-            if plugin is not None and plugin.is_active():
+            if plugin is not None and plugin.is_active(
+                    plugin_cached_settings=raw_settings):
                 plugins.append(plugin)
 
         # Add the fallback plugin from ini file.

rhodecode/authentication/views.py

@@ -63,7 +63,7 @@ class AuthnPluginViewBase(BaseAppView):
         for node in schema:
             if node.name not in defaults:
                 defaults[node.name] = self.plugin.get_setting_by_name(
-                    node.name, node.default, cache=False)
+                    node.name, node.default)
 
         template_context = {
             'defaults': defaults,
@@ -145,7 +145,7 @@ class AuthSettingsView(BaseAppView):
 
         # Create form default values and fill the form.
         form_defaults = {
-            'auth_plugins': ','.join(enabled_plugins)
+            'auth_plugins': ',\n'.join(enabled_plugins)
         }
         form_defaults.update(defaults)
         html = formencode.htmlfill.render(

rhodecode/config/hook_templates/git_post_receive.py.tmpl

@@ -1,46 +0,0 @@
-#!_ENV_
-import os
-import sys
-
-try:
-    from vcsserver import hooks
-except ImportError:
-    if os.environ.get('RC_DEBUG_GIT_HOOK'):
-        import traceback
-        print traceback.format_exc()
-    hooks = None
-
-
-RC_HOOK_VER = '_TMPL_'
-
-
-def main():
-    if hooks is None:
-        # exit with success if we cannot import vcsserver.hooks !!
-        # this allows simply push to this repo even without rhodecode
-        sys.exit(0)
-
-    if os.environ.get('RC_SKIP_HOOKS'):
-        sys.exit(0)
-
-    repo_path = os.getcwd()
-    push_data = sys.stdin.readlines()
-    os.environ['RC_HOOK_VER'] = RC_HOOK_VER
-    # os.environ is modified here by a subprocess call that
-    # runs git and later git executes this hook.
-    # Environ gets some additional info from rhodecode system
-    # like IP or username from basic-auth
-    try:
-        result = hooks.git_post_receive(repo_path, push_data, os.environ)
-        sys.exit(result)
-    except Exception as error:
-        # TODO: johbo: Improve handling of this special case
-        if not getattr(error, '_vcs_kind', None) == 'repo_locked':
-            raise
-        print 'ERROR:', error
-        sys.exit(1)
-    sys.exit(0)
-
-
-if __name__ == '__main__':
-    main()

rhodecode/config/hook_templates/git_pre_receive.py.tmpl

@@ -1,46 +0,0 @@
-#!_ENV_
-import os
-import sys
-
-try:
-    from vcsserver import hooks
-except ImportError:
-    if os.environ.get('RC_DEBUG_GIT_HOOK'):
-        import traceback
-        print traceback.format_exc()
-    hooks = None
-
-
-RC_HOOK_VER = '_TMPL_'
-
-
-def main():
-    if hooks is None:
-        # exit with success if we cannot import vcsserver.hooks !!
-        # this allows simply push to this repo even without rhodecode
-        sys.exit(0)
-
-    if os.environ.get('RC_SKIP_HOOKS'):
-        sys.exit(0)
-
-    repo_path = os.getcwd()
-    push_data = sys.stdin.readlines()
-    os.environ['RC_HOOK_VER'] = RC_HOOK_VER
-    # os.environ is modified here by a subprocess call that
-    # runs git and later git executes this hook.
-    # Environ gets some additional info from rhodecode system
-    # like IP or username from basic-auth
-    try:
-        result = hooks.git_pre_receive(repo_path, push_data, os.environ)
-        sys.exit(result)
-    except Exception as error:
-        # TODO: johbo: Improve handling of this special case
-        if not getattr(error, '_vcs_kind', None) == 'repo_locked':
-            raise
-        print 'ERROR:', error
-        sys.exit(1)
-    sys.exit(0)
-
-
-if __name__ == '__main__':
-    main()

rhodecode/config/hook_templates/svn_post_commit_hook.py.tmpl

@@ -1,14 +0,0 @@
-#!_ENV_
-
-import sys
-
-
-RC_HOOK_VER = '_TMPL_'
-
-
-def main():
-    sys.exit(0)
-
-
-if __name__ == '__main__':
-    main()

rhodecode/config/hook_templates/svn_pre_commit_hook.py.tmpl

@@ -1,19 +0,0 @@
-#!_ENV_
-
-import os
-import sys
-
-
-RC_HOOK_VER = '_TMPL_'
-
-
-def main():
-    if os.environ.get('SSH_READ_ONLY') == '1':
-        sys.stderr.write('Only read-only access is allowed')
-        sys.exit(1)
-
-    sys.exit(0)
-
-
-if __name__ == '__main__':
-    main()

rhodecode/config/middleware.py

@@ -226,6 +226,7 @@ def includeme(config):
     config.include('rhodecode.apps.user_group')
     config.include('rhodecode.apps.search')
     config.include('rhodecode.apps.user_profile')
+    config.include('rhodecode.apps.user_group_profile')
     config.include('rhodecode.apps.my_account')
     config.include('rhodecode.apps.svn_support')
     config.include('rhodecode.apps.ssh_support')

rhodecode/events/pullrequest.py

@@ -37,6 +37,7 @@ class PullRequestEvent(RepoEvent):
         self.pullrequest = pullrequest
 
     def as_dict(self):
+        from rhodecode.lib.utils2 import md5_safe
         from rhodecode.model.pull_request import PullRequestModel
         data = super(PullRequestEvent, self).as_dict()
 
@@ -46,6 +47,9 @@ class PullRequestEvent(RepoEvent):
             repos=[self.pullrequest.source_repo]
         )
         issues = _issues_as_dict(commits)
+        # calculate hashes of all commits for unique identifier of commits
+        # inside that pull request
+        commits_hash = md5_safe(':'.join(x.get('raw_id', '') for x in commits))
 
         data.update({
             'pullrequest': {
@@ -56,7 +60,10 @@ class PullRequestEvent(RepoEvent):
                     self.pullrequest, request=self.request),
                 'permalink_url': PullRequestModel().get_url(
                     self.pullrequest, request=self.request, permalink=True),
+                'shadow_url': PullRequestModel().get_shadow_clone_url(
+                    self.pullrequest, request=self.request),
                 'status': self.pullrequest.calculated_review_status(),
+                'commits_uid': commits_hash,
                 'commits': commits,
             }
         })

rhodecode/events/repo.py

@@ -132,7 +132,8 @@ def _commits_as_dict(event, commit_ids, repos):
 
     missing_commits = set(commit_ids) - set(c['raw_id'] for c in commits)
     if missing_commits:
-        log.error('missing commits: %s' % ', '.join(missing_commits))
+        log.error('Inconsistent repository state. '
+                  'Missing commits: %s' % ', '.join(missing_commits))
 
     return commits
 

rhodecode/integrations/__init__.py

@@ -45,6 +45,8 @@ integration_type_registry.register_integration_type(
     base.EEIntegration('Jira Issues integration', 'jira'))
 integration_type_registry.register_integration_type(
     base.EEIntegration('Redmine Tracker integration', 'redmine'))
+integration_type_registry.register_integration_type(
+    base.EEIntegration('Jenkins CI integration', 'jenkins'))
 
 
 def integrations_event_handler(event):

rhodecode/integrations/types/base.py

@@ -19,74 +19,86 @@
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import colander
+import string
+import collections
+import logging
+
+from mako import exceptions
+
 from rhodecode.translation import _
 
 
+log = logging.getLogger(__name__)
+
+
 class IntegrationTypeBase(object):
     """ Base class for IntegrationType plugins """
     is_dummy = False
     description = ''
-    icon = '''
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg
-   xmlns:dc="http://purl.org/dc/elements/1.1/"
-   xmlns:cc="http://creativecommons.org/ns#"
-   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
-   xmlns:svg="http://www.w3.org/2000/svg"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
-   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
-   viewBox="0 -256 1792 1792"
-   id="svg3025"
-   version="1.1"
-   inkscape:version="0.48.3.1 r9886"
-   width="100%"
-   height="100%"
-   sodipodi:docname="cog_font_awesome.svg">
-  <metadata
-     id="metadata3035">
-    <rdf:RDF>
-      <cc:Work
-         rdf:about="">
-        <dc:format>image/svg+xml</dc:format>
-        <dc:type
-           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
-      </cc:Work>
-    </rdf:RDF>
-  </metadata>
-  <defs
-     id="defs3033" />
-  <sodipodi:namedview
-     pagecolor="#ffffff"
-     bordercolor="#666666"
-     borderopacity="1"
-     objecttolerance="10"
-     gridtolerance="10"
-     guidetolerance="10"
-     inkscape:pageopacity="0"
-     inkscape:pageshadow="2"
-     inkscape:window-width="640"
-     inkscape:window-height="480"
-     id="namedview3031"
-     showgrid="false"
-     inkscape:zoom="0.13169643"
-     inkscape:cx="896"
-     inkscape:cy="896"
-     inkscape:window-x="0"
-     inkscape:window-y="25"
-     inkscape:window-maximized="0"
-     inkscape:current-layer="svg3025" />
-  <g
-     transform="matrix(1,0,0,-1,121.49153,1285.4237)"
-     id="g3027">
-    <path
-       d="m 1024,640 q 0,106 -75,181 -75,75 -181,75 -106,0 -181,-75 -75,-75 -75,-181 0,-106 75,-181 75,-75 181,-75 106,0 181,75 75,75 75,181 z m 512,109 V 527 q 0,-12 -8,-23 -8,-11 -20,-13 l -185,-28 q -19,-54 -39,-91 35,-50 107,-138 10,-12 10,-25 0,-13 -9,-23 -27,-37 -99,-108 -72,-71 -94,-71 -12,0 -26,9 l -138,108 q -44,-23 -91,-38 -16,-136 -29,-186 -7,-28 -36,-28 H 657 q -14,0 -24.5,8.5 Q 622,-111 621,-98 L 593,86 q -49,16 -90,37 L 362,16 Q 352,7 337,7 323,7 312,18 186,132 147,186 q -7,10 -7,23 0,12 8,23 15,21 51,66.5 36,45.5 54,70.5 -27,50 -41,99 L 29,495 Q 16,497 8,507.5 0,518 0,531 v 222 q 0,12 8,23 8,11 19,13 l 186,28 q 14,46 39,92 -40,57 -107,138 -10,12 -10,24 0,10 9,23 26,36 98.5,107.5 72.5,71.5 94.5,71.5 13,0 26,-10 l 138,-107 q 44,23 91,38 16,136 29,186 7,28 36,28 h 222 q 14,0 24.5,-8.5 Q 914,1391 915,1378 l 28,-184 q 49,-16 90,-37 l 142,107 q 9,9 24,9 13,0 25,-10 129,-119 165,-170 7,-8 7,-22 0,-12 -8,-23 -15,-21 -51,-66.5 -36,-45.5 -54,-70.5 26,-50 41,-98 l 183,-28 q 13,-2 21,-12.5 8,-10.5 8,-23.5 z"
-       id="path3029"
-       inkscape:connector-curvature="0"
-       style="fill:currentColor" />
-  </g>
-</svg>
-'''
+
+    @classmethod
+    def icon(cls):
+        return '''
+        <?xml version="1.0" encoding="UTF-8" standalone="no"?>
+        <svg
+           xmlns:dc="http://purl.org/dc/elements/1.1/"
+           xmlns:cc="http://creativecommons.org/ns#"
+           xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+           xmlns:svg="http://www.w3.org/2000/svg"
+           xmlns="http://www.w3.org/2000/svg"
+           xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+           xmlns:inkscape="http://setwww.inkscape.org/namespaces/inkscape"
+           viewBox="0 -256 1792 1792"
+           id="svg3025"
+           version="1.1"
+           inkscape:version="0.48.3.1 r9886"
+           width="100%"
+           height="100%"
+           sodipodi:docname="cog_font_awesome.svg">
+          <metadata
+             id="metadata3035">
+            <rdf:RDF>
+              <cc:Work
+                 rdf:about="">
+                <dc:format>image/svg+xml</dc:format>
+                <dc:type
+                   rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+              </cc:Work>
+            </rdf:RDF>
+          </metadata>
+          <defs
+             id="defs3033" />
+          <sodipodi:namedview
+             pagecolor="#ffffff"
+             bordercolor="#666666"
+             borderopacity="1"
+             objecttolerance="10"
+             gridtolerance="10"
+             guidetolerance="10"
+             inkscape:pageopacity="0"
+             inkscape:pageshadow="2"
+             inkscape:window-width="640"
+             inkscape:window-height="480"
+             id="namedview3031"
+             showgrid="false"
+             inkscape:zoom="0.13169643"
+             inkscape:cx="896"
+             inkscape:cy="896"
+             inkscape:window-x="0"
+             inkscape:window-y="25"
+             inkscape:window-maximized="0"
+             inkscape:current-layer="svg3025" />
+          <g
+             transform="matrix(1,0,0,-1,121.49153,1285.4237)"
+             id="g3027">
+            <path
+               d="m 1024,640 q 0,106 -75,181 -75,75 -181,75 -106,0 -181,-75 -75,-75 -75,-181 0,-106 75,-181 75,-75 181,-75 106,0 181,75 75,75 75,181 z m 512,109 V 527 q 0,-12 -8,-23 -8,-11 -20,-13 l -185,-28 q -19,-54 -39,-91 35,-50 107,-138 10,-12 10,-25 0,-13 -9,-23 -27,-37 -99,-108 -72,-71 -94,-71 -12,0 -26,9 l -138,108 q -44,-23 -91,-38 -16,-136 -29,-186 -7,-28 -36,-28 H 657 q -14,0 -24.5,8.5 Q 622,-111 621,-98 L 593,86 q -49,16 -90,37 L 362,16 Q 352,7 337,7 323,7 312,18 186,132 147,186 q -7,10 -7,23 0,12 8,23 15,21 51,66.5 36,45.5 54,70.5 -27,50 -41,99 L 29,495 Q 16,497 8,507.5 0,518 0,531 v 222 q 0,12 8,23 8,11 19,13 l 186,28 q 14,46 39,92 -40,57 -107,138 -10,12 -10,24 0,10 9,23 26,36 98.5,107.5 72.5,71.5 94.5,71.5 13,0 26,-10 l 138,-107 q 44,23 91,38 16,136 29,186 7,28 36,28 h 222 q 14,0 24.5,-8.5 Q 914,1391 915,1378 l 28,-184 q 49,-16 90,-37 l 142,107 q 9,9 24,9 13,0 25,-10 129,-119 165,-170 7,-8 7,-22 0,-12 -8,-23 -15,-21 -51,-66.5 -36,-45.5 -54,-70.5 26,-50 41,-98 l 183,-28 q 13,-2 21,-12.5 8,-10.5 8,-23.5 z"
+               id="path3029"
+               inkscape:connector-curvature="0"
+               style="fill:currentColor" />
+          </g>
+        </svg>
+        '''
 
     def __init__(self, settings):
         """
@@ -108,3 +120,192 @@ class EEIntegration(IntegrationTypeBase):
     def __init__(self, name, key, settings=None):
         self.display_name = name
         self.key = key
+        super(EEIntegration, self).__init__(settings)
+
+
+# Helpers #
+WEBHOOK_URL_VARS = [
+    ('event_name', 'Unique name of the event type, e.g pullrequest-update'),
+    ('repo_name', 'Full name of the repository'),
+    ('repo_type', 'VCS type of repository'),
+    ('repo_id', 'Unique id of repository'),
+    ('repo_url', 'Repository url'),
+    # extra repo fields
+    ('extra:<extra_key_name>', 'Extra repo variables, read from its settings.'),
+
+    # special attrs below that we handle, using multi-call
+    ('branch', 'Name of each brach submitted, if any.'),
+    ('commit_id', 'Id of each commit submitted, if any.'),
+
+    # pr events vars
+    ('pull_request_id', 'Unique ID of the pull request.'),
+    ('pull_request_title', 'Title of the pull request.'),
+    ('pull_request_url', 'Pull request url.'),
+    ('pull_request_shadow_url', 'Pull request shadow repo clone url.'),
+    ('pull_request_commits_uid', 'Calculated UID of all commits inside the PR. '
+                                 'Changes after PR update'),
+
+    # user who triggers the call
+    ('username', 'User who triggered the call.'),
+    ('user_id', 'User id who triggered the call.'),
+]
+
+# common vars for url template used for CI plugins. Shared with webhook
+CI_URL_VARS = WEBHOOK_URL_VARS
+
+
+class CommitParsingDataHandler(object):
+
+    def aggregate_branch_data(self, branches, commits):
+        branch_data = collections.OrderedDict()
+        for obj in branches:
+            branch_data[obj['name']] = obj
+
+        branches_commits = collections.OrderedDict()
+        for commit in commits:
+            if commit.get('git_ref_change'):
+                # special case for GIT that allows creating tags,
+                # deleting branches without associated commit
+                continue
+            commit_branch = commit['branch']
+
+            if commit_branch not in branches_commits:
+                _branch = branch_data[commit_branch] \
+                    if commit_branch else commit_branch
+                branch_commits = {'branch': _branch,
+                                  'commits': []}
+                branches_commits[commit_branch] = branch_commits
+
+            branch_commits = branches_commits[commit_branch]
+            branch_commits['commits'].append(commit)
+        return branches_commits
+
+
+class WebhookDataHandler(CommitParsingDataHandler):
+    name = 'webhook'
+
+    def __init__(self, template_url, headers):
+        self.template_url = template_url
+        self.headers = headers
+
+    def get_base_parsed_template(self, data):
+        """
+        initially parses the passed in template with some common variables
+        available on ALL calls
+        """
+        # note: make sure to update the `WEBHOOK_URL_VARS` if this changes
+        common_vars = {
+            'repo_name': data['repo']['repo_name'],
+            'repo_type': data['repo']['repo_type'],
+            'repo_id': data['repo']['repo_id'],
+            'repo_url': data['repo']['url'],
+            'username': data['actor']['username'],
+            'user_id': data['actor']['user_id'],
+            'event_name': data['name']
+        }
+
+        extra_vars = {}
+        for extra_key, extra_val in data['repo']['extra_fields'].items():
+            extra_vars['extra__{}'.format(extra_key)] = extra_val
+        common_vars.update(extra_vars)
+
+        template_url = self.template_url.replace('${extra:', '${extra__')
+        return string.Template(template_url).safe_substitute(**common_vars)
+
+    def repo_push_event_handler(self, event, data):
+        url = self.get_base_parsed_template(data)
+        url_cals = []
+
+        branches_commits = self.aggregate_branch_data(
+            data['push']['branches'], data['push']['commits'])
+        if '${branch}' in url:
+            # call it multiple times, for each branch if used in variables
+            for branch, commit_ids in branches_commits.items():
+                branch_url = string.Template(url).safe_substitute(branch=branch)
+                # call further down for each commit if used
+                if '${commit_id}' in branch_url:
+                    for commit_data in commit_ids['commits']:
+                        commit_id = commit_data['raw_id']
+                        commit_url = string.Template(branch_url).safe_substitute(
+                            commit_id=commit_id)
+                        # register per-commit call
+                        log.debug(
+                            'register %s call(%s) to url %s',
+                            self.name, event, commit_url)
+                        url_cals.append(
+                            (commit_url, self.headers, data))
+
+                else:
+                    # register per-branch call
+                    log.debug(
+                        'register %s call(%s) to url %s',
+                        self.name, event, branch_url)
+                    url_cals.append(
+                        (branch_url, self.headers, data))
+
+        else:
+            log.debug(
+                'register %s call(%s) to url %s', self.name, event, url)
+            url_cals.append((url, self.headers, data))
+
+        return url_cals
+
+    def repo_create_event_handler(self, event, data):
+        url = self.get_base_parsed_template(data)
+        log.debug(
+            'register %s call(%s) to url %s', self.name, event, url)
+        return [(url, self.headers, data)]
+
+    def pull_request_event_handler(self, event, data):
+        url = self.get_base_parsed_template(data)
+        log.debug(
+            'register %s call(%s) to url %s', self.name, event, url)
+        url = string.Template(url).safe_substitute(
+            pull_request_id=data['pullrequest']['pull_request_id'],
+            pull_request_title=data['pullrequest']['title'],
+            pull_request_url=data['pullrequest']['url'],
+            pull_request_shadow_url=data['pullrequest']['shadow_url'],
+            pull_request_commits_uid=data['pullrequest']['commits_uid'],
+        )
+        return [(url, self.headers, data)]
+
+    def __call__(self, event, data):
+        from rhodecode import events
+
+        if isinstance(event, events.RepoPushEvent):
+            return self.repo_push_event_handler(event, data)
+        elif isinstance(event, events.RepoCreateEvent):
+            return self.repo_create_event_handler(event, data)
+        elif isinstance(event, events.PullRequestEvent):
+            return self.pull_request_event_handler(event, data)
+        else:
+            raise ValueError(
+                'event type `%s` not in supported list: %s' % (
+                    event.__class__, events))
+
+
+def get_auth(settings):
+    from requests.auth import HTTPBasicAuth
+    username = settings.get('username')
+    password = settings.get('password')
+    if username and password:
+        return HTTPBasicAuth(username, password)
+    return None
+
+
+def get_web_token(settings):
+    return settings['secret_token']
+
+
+def get_url_vars(url_vars):
+    return '\n'.join(
+        '{} - {}'.format('${' + key + '}', explanation)
+        for key, explanation in url_vars)
+
+
+def render_with_traceback(template, *args, **kwargs):
+    try:
+        return template.render(*args, **kwargs)
+    except Exception:
+        log.error(exceptions.text_error_template().render())
+        raise

rhodecode/integrations/types/email.py

@@ -29,7 +29,8 @@ from rhodecode import events
 from rhodecode.translation import _
 from rhodecode.lib.celerylib import run_task
 from rhodecode.lib.celerylib import tasks
-from rhodecode.integrations.types.base import IntegrationTypeBase
+from rhodecode.integrations.types.base import (
+    IntegrationTypeBase, render_with_traceback)
 
 
 log = logging.getLogger(__name__)
@@ -127,11 +128,15 @@ repo_push_template_html = Template('''
                 </td></tr>
                 <tr>
                     <td style="padding:15px;" valign="top">
-                        % for commit in data['push']['commits']:
-                        <a href="${commit['url']}">${commit['short_id']}</a> by ${commit['author']} at ${commit['date']} <br/>
-                        ${commit['message_html']} <br/>
-                        <br/>
-                        % endfor
+                        % if data['push']['commits']:
+                            % for commit in data['push']['commits']:
+                            <a href="${commit['url']}">${commit['short_id']}</a> by ${commit['author']} at ${commit['date']} <br/>
+                            ${commit['message_html']} <br/>
+                            <br/>
+                            % endfor
+                        % else:
+                            No commit data
+                        % endif
                     </td>
                 </tr>
             </table>
@@ -146,7 +151,34 @@ repo_push_template_html = Template('''
 </html>
 ''')
 
-email_icon = '''
+
+class EmailSettingsSchema(colander.Schema):
+    @colander.instantiate(validator=colander.Length(min=1))
+    class recipients(colander.SequenceSchema):
+        title = _('Recipients')
+        description = _('Email addresses to send push events to')
+        widget = deform.widget.SequenceWidget(min_len=1)
+
+        recipient = colander.SchemaNode(
+            colander.String(),
+            title=_('Email address'),
+            description=_('Email address'),
+            default='',
+            validator=colander.Email(),
+            widget=deform.widget.TextInputWidget(
+                placeholder='user@domain.com',
+            ),
+        )
+
+
+class EmailIntegrationType(IntegrationTypeBase):
+    key = 'email'
+    display_name = _('Email')
+    description = _('Send repo push summaries to a list of recipients via email')
+
+    @classmethod
+    def icon(cls):
+        return '''
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <svg
    xmlns:dc="http://purl.org/dc/elements/1.1/"
@@ -208,32 +240,6 @@ email_icon = '''
 </svg>
 '''
 
-
-class EmailSettingsSchema(colander.Schema):
-    @colander.instantiate(validator=colander.Length(min=1))
-    class recipients(colander.SequenceSchema):
-        title = _('Recipients')
-        description = _('Email addresses to send push events to')
-        widget = deform.widget.SequenceWidget(min_len=1)
-
-        recipient = colander.SchemaNode(
-            colander.String(),
-            title=_('Email address'),
-            description=_('Email address'),
-            default='',
-            validator=colander.Email(),
-            widget=deform.widget.TextInputWidget(
-                placeholder='user@domain.com',
-            ),
-        )
-
-
-class EmailIntegrationType(IntegrationTypeBase):
-    key = 'email'
-    display_name = _('Email')
-    description = _('Send repo push summaries to a list of recipients via email')
-    icon = email_icon
-
     def settings_schema(self):
         schema = EmailSettingsSchema()
         return schema
@@ -276,12 +282,14 @@ def repo_push_handler(data, settings):
             branches=', '.join(
                 branch['name'] for branch in data['push']['branches']))
 
-    email_body_plaintext = repo_push_template_plaintext.render(
+    email_body_plaintext = render_with_traceback(
+        repo_push_template_plaintext,
         data=data,
         subject=subject,
         instance_url=server_url)
 
-    email_body_html = repo_push_template_html.render(
+    email_body_html = render_with_traceback(
+        repo_push_template_html,
         data=data,
         subject=subject,
         instance_url=server_url)

rhodecode/integrations/types/hipchat.py

@@ -24,14 +24,14 @@ import logging
 import requests
 import colander
 import textwrap
-from collections import OrderedDict
 from mako.template import Template
 from rhodecode import events
 from rhodecode.translation import _
 from rhodecode.lib import helpers as h
 from rhodecode.lib.celerylib import run_task, async_task, RequestContextTask
 from rhodecode.lib.colander_utils import strip_whitespace
-from rhodecode.integrations.types.base import IntegrationTypeBase
+from rhodecode.integrations.types.base import (
+    IntegrationTypeBase, CommitParsingDataHandler, render_with_traceback)
 
 log = logging.getLogger(__name__)
 
@@ -81,23 +81,31 @@ repo_push_template = Template('''
 <ul>
 %for branch, branch_commits in branches_commits.items():
     <li>
+        % if branch:
         <a href="${branch_commits['branch']['url']}">branch: ${branch_commits['branch']['name']}</a>
+        % else:
+        to trunk
+        % endif
         <ul>
-    %for commit in branch_commits['commits']:
+        % for commit in branch_commits['commits']:
             <li><a href="${commit['url']}">${commit['short_id']}</a> - ${commit['message_html']}</li>
-    %endfor
+        % endfor
         </ul>
     </li>
 %endfor
 ''')
 
 
-class HipchatIntegrationType(IntegrationTypeBase):
+class HipchatIntegrationType(IntegrationTypeBase, CommitParsingDataHandler):
     key = 'hipchat'
     display_name = _('Hipchat')
     description = _('Send events such as repo pushes and pull requests to '
                     'your hipchat channel.')
-    icon = '''<?xml version="1.0" encoding="utf-8"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" viewBox="0 0 1000 1000" enable-background="new 0 0 1000 1000" xml:space="preserve"><g><g transform="translate(0.000000,511.000000) scale(0.100000,-0.100000)"><path fill="#205281" d="M4197.1,4662.4c-1661.5-260.4-3018-1171.6-3682.6-2473.3C219.9,1613.6,100,1120.3,100,462.6c0-1014,376.8-1918.4,1127-2699.4C2326.7-3377.6,3878.5-3898.3,5701-3730.5l486.5,44.5l208.9-123.3c637.2-373.4,1551.8-640.6,2240.4-650.9c304.9-6.9,335.7,0,417.9,75.4c185,174.7,147.3,411.1-89.1,548.1c-315.2,181.6-620,544.7-733.1,870.1l-51.4,157.6l472.7,472.7c349.4,349.4,520.7,551.5,657.7,774.2c784.5,1281.2,784.5,2788.5,0,4052.6c-236.4,376.8-794.8,966-1178.4,1236.7c-572.1,407.7-1264.1,709.1-1993.7,870.1c-267.2,58.2-479.6,75.4-1038,82.2C4714.4,4686.4,4310.2,4679.6,4197.1,4662.4z M5947.6,3740.9c1856.7-380.3,3127.6-1709.4,3127.6-3275c0-1000.3-534.4-1949.2-1466.2-2600.1c-188.4-133.6-287.8-226.1-301.5-284.4c-41.1-157.6,263.8-938.6,397.4-1020.8c20.5-10.3,34.3-44.5,34.3-75.4c0-167.8-811.9,195.3-1363.4,609.8l-181.6,137l-332.3-58.2c-445.3-78.8-1281.2-78.8-1702.6,0C2796-2569.2,1734.1-1832.6,1220.2-801.5C983.8-318.5,905,51.5,929,613.3c27.4,640.6,243.2,1192.1,685.1,1740.3c620,770.8,1661.5,1305.2,2822.8,1452.5C4806.9,3854,5553.7,3819.7,5947.6,3740.9z"/><path fill="#205281" d="M2381.5-345.9c-75.4-106.2-68.5-167.8,34.3-322c332.3-500.2,1010.6-928.4,1760.8-1120.2c417.9-106.2,1226.4-106.2,1644.3,0c712.5,181.6,1270.9,517.3,1685.4,1014C7681-561.7,7715.3-424.7,7616-325.4c-89.1,89.1-167.9,65.1-431.7-133.6c-835.8-630.3-2028-856.4-3086.5-585.8C3683.3-938.6,3142-685,2830.3-448.7C2576.8-253.4,2463.7-229.4,2381.5-345.9z"/></g></g><!-- Svg Vector Icons : http://www.onlinewebfonts.com/icon --></svg>'''
+
+    @classmethod
+    def icon(cls):
+        return '''<?xml version="1.0" encoding="utf-8"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" viewBox="0 0 1000 1000" enable-background="new 0 0 1000 1000" xml:space="preserve"><g><g transform="translate(0.000000,511.000000) scale(0.100000,-0.100000)"><path fill="#205281" d="M4197.1,4662.4c-1661.5-260.4-3018-1171.6-3682.6-2473.3C219.9,1613.6,100,1120.3,100,462.6c0-1014,376.8-1918.4,1127-2699.4C2326.7-3377.6,3878.5-3898.3,5701-3730.5l486.5,44.5l208.9-123.3c637.2-373.4,1551.8-640.6,2240.4-650.9c304.9-6.9,335.7,0,417.9,75.4c185,174.7,147.3,411.1-89.1,548.1c-315.2,181.6-620,544.7-733.1,870.1l-51.4,157.6l472.7,472.7c349.4,349.4,520.7,551.5,657.7,774.2c784.5,1281.2,784.5,2788.5,0,4052.6c-236.4,376.8-794.8,966-1178.4,1236.7c-572.1,407.7-1264.1,709.1-1993.7,870.1c-267.2,58.2-479.6,75.4-1038,82.2C4714.4,4686.4,4310.2,4679.6,4197.1,4662.4z M5947.6,3740.9c1856.7-380.3,3127.6-1709.4,3127.6-3275c0-1000.3-534.4-1949.2-1466.2-2600.1c-188.4-133.6-287.8-226.1-301.5-284.4c-41.1-157.6,263.8-938.6,397.4-1020.8c20.5-10.3,34.3-44.5,34.3-75.4c0-167.8-811.9,195.3-1363.4,609.8l-181.6,137l-332.3-58.2c-445.3-78.8-1281.2-78.8-1702.6,0C2796-2569.2,1734.1-1832.6,1220.2-801.5C983.8-318.5,905,51.5,929,613.3c27.4,640.6,243.2,1192.1,685.1,1740.3c620,770.8,1661.5,1305.2,2822.8,1452.5C4806.9,3854,5553.7,3819.7,5947.6,3740.9z"/><path fill="#205281" d="M2381.5-345.9c-75.4-106.2-68.5-167.8,34.3-322c332.3-500.2,1010.6-928.4,1760.8-1120.2c417.9-106.2,1226.4-106.2,1644.3,0c712.5,181.6,1270.9,517.3,1685.4,1014C7681-561.7,7715.3-424.7,7616-325.4c-89.1,89.1-167.9,65.1-431.7-133.6c-835.8-630.3-2028-856.4-3086.5-585.8C3683.3-938.6,3142-685,2830.3-448.7C2576.8-253.4,2463.7-229.4,2381.5-345.9z"/></g></g><!-- Svg Vector Icons : http://www.onlinewebfonts.com/icon --></svg>'''
+
     valid_events = [
         events.PullRequestCloseEvent,
         events.PullRequestMergeEvent,
@@ -213,20 +221,11 @@ class HipchatIntegrationType(IntegrationTypeBase):
         )
 
     def format_repo_push_event(self, data):
-        branch_data = {branch['name']: branch
-                       for branch in data['push']['branches']}
+        branches_commits = self.aggregate_branch_data(
+            data['push']['branches'], data['push']['commits'])
 
-        branches_commits = OrderedDict()
-        for commit in data['push']['commits']:
-            if commit['branch'] not in branches_commits:
-                branch_commits = {'branch': branch_data[commit['branch']],
-                                  'commits': []}
-                branches_commits[commit['branch']] = branch_commits
-
-            branch_commits = branches_commits[commit['branch']]
-            branch_commits['commits'].append(commit)
-
-        result = repo_push_template.render(
+        result = render_with_traceback(
+            repo_push_template,
             data=data,
             branches_commits=branches_commits,
         )

rhodecode/integrations/types/slack.py

@@ -28,14 +28,14 @@ import deform
 import requests
 import colander
 from mako.template import Template
-from collections import OrderedDict
 
 from rhodecode import events
 from rhodecode.translation import _
 from rhodecode.lib import helpers as h
 from rhodecode.lib.celerylib import run_task, async_task, RequestContextTask
 from rhodecode.lib.colander_utils import strip_whitespace
-from rhodecode.integrations.types.base import IntegrationTypeBase
+from rhodecode.integrations.types.base import (
+    IntegrationTypeBase, CommitParsingDataHandler, render_with_traceback)
 
 log = logging.getLogger(__name__)
 
@@ -87,12 +87,16 @@ class SlackSettingsSchema(colander.Schema):
     )
 
 
-class SlackIntegrationType(IntegrationTypeBase):
+class SlackIntegrationType(IntegrationTypeBase, CommitParsingDataHandler):
     key = 'slack'
     display_name = _('Slack')
     description = _('Send events such as repo pushes and pull requests to '
                     'your slack channel.')
-    icon = '''<?xml version="1.0" encoding="UTF-8" standalone="no"?><svg viewBox="0 0 256 256" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" preserveAspectRatio="xMidYMid"><g><path d="M165.963541,15.8384262 C162.07318,3.86308197 149.212328,-2.69009836 137.239082,1.20236066 C125.263738,5.09272131 118.710557,17.9535738 122.603016,29.9268197 L181.550164,211.292328 C185.597902,222.478689 197.682361,228.765377 209.282098,225.426885 C221.381246,221.943607 228.756984,209.093246 224.896,197.21023 C224.749115,196.756984 165.963541,15.8384262 165.963541,15.8384262" fill="#DFA22F"></path><path d="M74.6260984,45.515541 C70.7336393,33.5422951 57.8727869,26.9891148 45.899541,30.8794754 C33.9241967,34.7698361 27.3710164,47.6306885 31.2634754,59.6060328 L90.210623,240.971541 C94.2583607,252.157902 106.34282,258.44459 117.942557,255.104 C130.041705,251.62282 137.417443,238.772459 133.556459,226.887344 C133.409574,226.436197 74.6260984,45.515541 74.6260984,45.515541" fill="#3CB187"></path><path d="M240.161574,166.045377 C252.136918,162.155016 258.688,149.294164 254.797639,137.31882 C250.907279,125.345574 238.046426,118.792393 226.07318,122.682754 L44.7076721,181.632 C33.5213115,185.677639 27.234623,197.762098 30.5731148,209.361836 C34.0563934,221.460984 46.9067541,228.836721 58.7897705,224.975738 C59.2430164,224.828852 240.161574,166.045377 240.161574,166.045377" fill="#CE1E5B"></path><path d="M82.507541,217.270557 C94.312918,213.434754 109.528131,208.491016 125.855475,203.186361 C122.019672,191.380984 117.075934,176.163672 111.76918,159.83423 L68.4191475,173.924721 L82.507541,217.270557" fill="#392538"></path><path d="M173.847082,187.591344 C190.235279,182.267803 205.467279,177.31777 217.195016,173.507148 C213.359213,161.70177 208.413377,146.480262 203.106623,130.146623 L159.75659,144.237115 L173.847082,187.591344" fill="#BB242A"></path><path d="M210.484459,74.7058361 C222.457705,70.8154754 229.010885,57.954623 225.120525,45.9792787 C221.230164,34.0060328 208.369311,27.4528525 196.393967,31.3432131 L15.028459,90.292459 C3.84209836,94.3380984 -2.44459016,106.422557 0.896,118.022295 C4.37718033,130.121443 17.227541,137.49718 29.1126557,133.636197 C29.5638033,133.489311 210.484459,74.7058361 210.484459,74.7058361" fill="#72C5CD"></path><path d="M52.8220328,125.933115 C64.6274098,122.097311 79.8468197,117.151475 96.1762623,111.84682 C90.8527213,95.4565246 85.9026885,80.2245246 82.0920656,68.4946885 L38.731541,82.5872787 L52.8220328,125.933115" fill="#248C73"></path><path d="M144.159475,96.256 C160.551869,90.9303607 175.785967,85.9803279 187.515803,82.1676066 C182.190164,65.7752131 177.240131,50.5390164 173.42741,38.807082 L130.068984,52.8996721 L144.159475,96.256" fill="#62803A"></path></g></svg>'''
+
+    @classmethod
+    def icon(cls):
+        return '''<?xml version="1.0" encoding="UTF-8" standalone="no"?><svg viewBox="0 0 256 256" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" preserveAspectRatio="xMidYMid"><g><path d="M165.963541,15.8384262 C162.07318,3.86308197 149.212328,-2.69009836 137.239082,1.20236066 C125.263738,5.09272131 118.710557,17.9535738 122.603016,29.9268197 L181.550164,211.292328 C185.597902,222.478689 197.682361,228.765377 209.282098,225.426885 C221.381246,221.943607 228.756984,209.093246 224.896,197.21023 C224.749115,196.756984 165.963541,15.8384262 165.963541,15.8384262" fill="#DFA22F"></path><path d="M74.6260984,45.515541 C70.7336393,33.5422951 57.8727869,26.9891148 45.899541,30.8794754 C33.9241967,34.7698361 27.3710164,47.6306885 31.2634754,59.6060328 L90.210623,240.971541 C94.2583607,252.157902 106.34282,258.44459 117.942557,255.104 C130.041705,251.62282 137.417443,238.772459 133.556459,226.887344 C133.409574,226.436197 74.6260984,45.515541 74.6260984,45.515541" fill="#3CB187"></path><path d="M240.161574,166.045377 C252.136918,162.155016 258.688,149.294164 254.797639,137.31882 C250.907279,125.345574 238.046426,118.792393 226.07318,122.682754 L44.7076721,181.632 C33.5213115,185.677639 27.234623,197.762098 30.5731148,209.361836 C34.0563934,221.460984 46.9067541,228.836721 58.7897705,224.975738 C59.2430164,224.828852 240.161574,166.045377 240.161574,166.045377" fill="#CE1E5B"></path><path d="M82.507541,217.270557 C94.312918,213.434754 109.528131,208.491016 125.855475,203.186361 C122.019672,191.380984 117.075934,176.163672 111.76918,159.83423 L68.4191475,173.924721 L82.507541,217.270557" fill="#392538"></path><path d="M173.847082,187.591344 C190.235279,182.267803 205.467279,177.31777 217.195016,173.507148 C213.359213,161.70177 208.413377,146.480262 203.106623,130.146623 L159.75659,144.237115 L173.847082,187.591344" fill="#BB242A"></path><path d="M210.484459,74.7058361 C222.457705,70.8154754 229.010885,57.954623 225.120525,45.9792787 C221.230164,34.0060328 208.369311,27.4528525 196.393967,31.3432131 L15.028459,90.292459 C3.84209836,94.3380984 -2.44459016,106.422557 0.896,118.022295 C4.37718033,130.121443 17.227541,137.49718 29.1126557,133.636197 C29.5638033,133.489311 210.484459,74.7058361 210.484459,74.7058361" fill="#72C5CD"></path><path d="M52.8220328,125.933115 C64.6274098,122.097311 79.8468197,117.151475 96.1762623,111.84682 C90.8527213,95.4565246 85.9026885,80.2245246 82.0920656,68.4946885 L38.731541,82.5872787 L52.8220328,125.933115" fill="#248C73"></path><path d="M144.159475,96.256 C160.551869,90.9303607 175.785967,85.9803279 187.515803,82.1676066 C182.190164,65.7752131 177.240131,50.5390164 173.42741,38.807082 L130.068984,52.8996721 L144.159475,96.256" fill="#62803A"></path></g></svg>'''
+
     valid_events = [
         events.PullRequestCloseEvent,
         events.PullRequestMergeEvent,
@@ -190,32 +194,39 @@ class SlackIntegrationType(IntegrationTypeBase):
                 }
             ]
 
-        title = Template(textwrap.dedent(r'''
+        template = Template(textwrap.dedent(r'''
         *${data['actor']['username']}* left ${data['comment']['type']} on pull request <${data['pullrequest']['url']}|#${data['pullrequest']['pull_request_id']}>:
-        ''')).render(data=data, comment=event.comment)
+        '''))
+        title = render_with_traceback(
+            template, data=data, comment=event.comment)
 
-        text = Template(textwrap.dedent(r'''
+        template = Template(textwrap.dedent(r'''
         *pull request title*: ${pr_title}
         % if status_text:
         *submitted status*: `${status_text}`
         % endif
         >>> ${comment_text}
-        ''')).render(comment_text=comment_text,
-                     pr_title=data['pullrequest']['title'],
-                     status_text=status_text)
+        '''))
+        text = render_with_traceback(
+            template,
+            comment_text=comment_text,
+            pr_title=data['pullrequest']['title'],
+            status_text=status_text)
 
         return title, text, fields, overrides
 
     def format_pull_request_review_event(self, event, data):
-        title = Template(textwrap.dedent(r'''
+        template = Template(textwrap.dedent(r'''
         *${data['actor']['username']}* changed status of pull request <${data['pullrequest']['url']}|#${data['pullrequest']['pull_request_id']} to `${data['pullrequest']['status']}`>:
-        ''')).render(data=data)
+        '''))
+        title = render_with_traceback(template, data=data)
 
-        text = Template(textwrap.dedent(r'''
+        template = Template(textwrap.dedent(r'''
         *pull request title*: ${pr_title}
-        ''')).render(
-            pr_title=data['pullrequest']['title'],
-        )
+        '''))
+        text = render_with_traceback(
+            template,
+            pr_title=data['pullrequest']['title'])
 
         return title, text
 
@@ -227,50 +238,53 @@ class SlackIntegrationType(IntegrationTypeBase):
             events.PullRequestCreateEvent: 'created',
         }.get(event.__class__, str(event.__class__))
 
-        title = Template(textwrap.dedent(r'''
+        template = Template(textwrap.dedent(r'''
         *${data['actor']['username']}* `${action}` pull request <${data['pullrequest']['url']}|#${data['pullrequest']['pull_request_id']}>:
-        ''')).render(data=data, action=action)
+        '''))
+        title = render_with_traceback(template, data=data, action=action)
 
-        text = Template(textwrap.dedent(r'''
+        template = Template(textwrap.dedent(r'''
         *pull request title*: ${pr_title}
         %if data['pullrequest']['commits']:
         *commits*: ${len(data['pullrequest']['commits'])}
         %endif
-        ''')).render(
+        '''))
+        text = render_with_traceback(
+            template,
             pr_title=data['pullrequest']['title'],
-            data=data
-        )
+            data=data)
 
         return title, text
 
     def format_repo_push_event(self, data):
-        branch_data = {branch['name']: branch
-                       for branch in data['push']['branches']}
 
-        branches_commits = OrderedDict()
-        for commit in data['push']['commits']:
-            if commit['branch'] not in branches_commits:
-                branch_commits = {'branch': branch_data[commit['branch']],
-                                  'commits': []}
-                branches_commits[commit['branch']] = branch_commits
+        branches_commits = self.aggregate_branch_data(
+            data['push']['branches'], data['push']['commits'])
 
-            branch_commits = branches_commits[commit['branch']]
-            branch_commits['commits'].append(commit)
-
-        title = Template(r'''
+        template = Template(r'''
         *${data['actor']['username']}* pushed to repo <${data['repo']['url']}|${data['repo']['repo_name']}>:
-        ''').render(data=data)
+        ''')
+        title = render_with_traceback(template, data=data)
 
         repo_push_template = Template(textwrap.dedent(r'''
-        %for branch, branch_commits in branches_commits.items():
-        ${len(branch_commits['commits'])} ${'commit' if len(branch_commits['commits']) == 1 else 'commits'} on branch: <${branch_commits['branch']['url']}|${branch_commits['branch']['name']}>
-        %for commit in branch_commits['commits']:
+        <%
+            def branch_text(branch):
+                if branch:
+                    return 'on branch: <{}|{}>'.format(branch_commits['branch']['url'], branch_commits['branch']['name'])
+                else:
+                    ## case for SVN no branch push...
+                    return 'to trunk'
+        %> \
+        % for branch, branch_commits in branches_commits.items():
+        ${len(branch_commits['commits'])} ${'commit' if len(branch_commits['commits']) == 1 else 'commits'} ${branch_text(branch)}
+        % for commit in branch_commits['commits']:
         `<${commit['url']}|${commit['short_id']}>` - ${commit['message_html']|html_to_slack_links}
-        %endfor
-        %endfor
+        % endfor
+        % endfor
         '''))
 
-        text = repo_push_template.render(
+        text = render_with_traceback(
+            repo_push_template,
             data=data,
             branches_commits=branches_commits,
             html_to_slack_links=html_to_slack_links,
@@ -279,14 +293,16 @@ class SlackIntegrationType(IntegrationTypeBase):
         return title, text
 
     def format_repo_create_event(self, data):
-        title = Template(r'''
+        template = Template(r'''
         *${data['actor']['username']}* created new repository ${data['repo']['repo_name']}:
-        ''').render(data=data)
+        ''')
+        title = render_with_traceback(template, data=data)
 
-        text = Template(textwrap.dedent(r'''
+        template = Template(textwrap.dedent(r'''
         repo_url: ${data['repo']['url']}
         repo_type: ${data['repo']['repo_type']}
-        ''')).render(data=data)
+        '''))
+        text = render_with_traceback(template, data=data)
 
         return title, text
 

rhodecode/integrations/types/webhook.py

@@ -19,8 +19,6 @@
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 from __future__ import unicode_literals
-import string
-from collections import OrderedDict
 
 import deform
 import deform.widget
@@ -33,149 +31,18 @@ from requests.packages.urllib3.util.retry import Retry
 import rhodecode
 from rhodecode import events
 from rhodecode.translation import _
-from rhodecode.integrations.types.base import IntegrationTypeBase
+from rhodecode.integrations.types.base import (
+    IntegrationTypeBase, get_auth, get_web_token, get_url_vars,
+    WebhookDataHandler, WEBHOOK_URL_VARS)
 from rhodecode.lib.celerylib import run_task, async_task, RequestContextTask
+from rhodecode.model.validation_schema import widgets
 
 log = logging.getLogger(__name__)
 
 
 # updating this required to update the `common_vars` passed in url calling func
-WEBHOOK_URL_VARS = [
-    'repo_name',
-    'repo_type',
-    'repo_id',
-    'repo_url',
-    # extra repo fields
-    'extra:<extra_key_name>',
 
-    # special attrs below that we handle, using multi-call
-    'branch',
-    'commit_id',
-
-    # pr events vars
-    'pull_request_id',
-    'pull_request_url',
-
-    # user who triggers the call
-    'username',
-    'user_id',
-
-]
-URL_VARS = ', '.join('${' + x + '}' for x in WEBHOOK_URL_VARS)
-
-
-def get_auth(settings):
-    from requests.auth import HTTPBasicAuth
-    username = settings.get('username')
-    password = settings.get('password')
-    if username and password:
-        return HTTPBasicAuth(username, password)
-    return None
-
-
-class WebhookHandler(object):
-    def __init__(self, template_url, secret_token, headers):
-        self.template_url = template_url
-        self.secret_token = secret_token
-        self.headers = headers
-
-    def get_base_parsed_template(self, data):
-        """
-        initially parses the passed in template with some common variables
-        available on ALL calls
-        """
-        # note: make sure to update the `WEBHOOK_URL_VARS` if this changes
-        common_vars = {
-            'repo_name': data['repo']['repo_name'],
-            'repo_type': data['repo']['repo_type'],
-            'repo_id': data['repo']['repo_id'],
-            'repo_url': data['repo']['url'],
-            'username': data['actor']['username'],
-            'user_id': data['actor']['user_id']
-        }
-
-        extra_vars = {}
-        for extra_key, extra_val in data['repo']['extra_fields'].items():
-            extra_vars['extra__{}'.format(extra_key)] = extra_val
-        common_vars.update(extra_vars)
-
-        template_url = self.template_url.replace('${extra:', '${extra__')
-        return string.Template(template_url).safe_substitute(**common_vars)
-
-    def repo_push_event_handler(self, event, data):
-        url = self.get_base_parsed_template(data)
-        url_cals = []
-        branch_data = OrderedDict()
-        for obj in data['push']['branches']:
-            branch_data[obj['name']] = obj
-
-        branches_commits = OrderedDict()
-        for commit in data['push']['commits']:
-            if commit.get('git_ref_change'):
-                # special case for GIT that allows creating tags,
-                # deleting branches without associated commit
-                continue
-
-            if commit['branch'] not in branches_commits:
-                branch_commits = {'branch': branch_data[commit['branch']],
-                                  'commits': []}
-                branches_commits[commit['branch']] = branch_commits
-
-            branch_commits = branches_commits[commit['branch']]
-            branch_commits['commits'].append(commit)
-
-        if '${branch}' in url:
-            # call it multiple times, for each branch if used in variables
-            for branch, commit_ids in branches_commits.items():
-                branch_url = string.Template(url).safe_substitute(branch=branch)
-                # call further down for each commit if used
-                if '${commit_id}' in branch_url:
-                    for commit_data in commit_ids['commits']:
-                        commit_id = commit_data['raw_id']
-                        commit_url = string.Template(branch_url).safe_substitute(
-                            commit_id=commit_id)
-                        # register per-commit call
-                        log.debug(
-                            'register webhook call(%s) to url %s', event, commit_url)
-                        url_cals.append((commit_url, self.secret_token, self.headers, data))
-
-                else:
-                    # register per-branch call
-                    log.debug(
-                        'register webhook call(%s) to url %s', event, branch_url)
-                    url_cals.append((branch_url, self.secret_token, self.headers, data))
-
-        else:
-            log.debug(
-                'register webhook call(%s) to url %s', event, url)
-            url_cals.append((url, self.secret_token, self.headers, data))
-
-        return url_cals
-
-    def repo_create_event_handler(self, event, data):
-        url = self.get_base_parsed_template(data)
-        log.debug(
-            'register webhook call(%s) to url %s', event, url)
-        return [(url, self.secret_token, self.headers, data)]
-
-    def pull_request_event_handler(self, event, data):
-        url = self.get_base_parsed_template(data)
-        log.debug(
-            'register webhook call(%s) to url %s', event, url)
-        url = string.Template(url).safe_substitute(
-            pull_request_id=data['pullrequest']['pull_request_id'],
-            pull_request_url=data['pullrequest']['url'])
-        return [(url, self.secret_token, self.headers, data)]
-
-    def __call__(self, event, data):
-        if isinstance(event, events.RepoPushEvent):
-            return self.repo_push_event_handler(event, data)
-        elif isinstance(event, events.RepoCreateEvent):
-            return self.repo_create_event_handler(event, data)
-        elif isinstance(event, events.PullRequestEvent):
-            return self.pull_request_event_handler(event, data)
-        else:
-            raise ValueError('event type not supported: %s' % events)
+URL_VARS = get_url_vars(WEBHOOK_URL_VARS)
 
 
 class WebhookSettingsSchema(colander.Schema):
@@ -183,17 +50,21 @@ class WebhookSettingsSchema(colander.Schema):
         colander.String(),
         title=_('Webhook URL'),
         description=
-            _('URL to which Webhook should submit data. Following variables '
-              'are allowed to be used: {vars}. Some of the variables would '
-              'trigger multiple calls, like ${{branch}} or ${{commit_id}}. '
-              'Webhook will be called as many times as unique objects in '
-              'data in such cases.').format(vars=URL_VARS),
+            _('URL to which Webhook should submit data. If used some of the '
+              'variables would trigger multiple calls, like ${branch} or '
+              '${commit_id}. Webhook will be called as many times as unique '
+              'objects in data in such cases.'),
         missing=colander.required,
         required=True,
         validator=colander.url,
-        widget=deform.widget.TextInputWidget(
-            placeholder='https://www.example.com/webhook'
-        ),
+        widget=widgets.CodeMirrorWidget(
+            help_block_collapsable_name='Show url variables',
+            help_block_collapsable=(
+                'E.g http://my-serv/trigger_job/${{event_name}}'
+                '?PR_ID=${{pull_request_id}}'
+                '\nFull list of vars:\n{}'.format(URL_VARS)),
+            codemirror_mode='text',
+            codemirror_options='{"lineNumbers": false, "lineWrapping": true}'),
     )
     secret_token = colander.SchemaNode(
         colander.String(),
@@ -234,7 +105,7 @@ class WebhookSettingsSchema(colander.Schema):
         default='',
         missing='',
         widget=deform.widget.TextInputWidget(
-            placeholder='e.g.Authorization'
+            placeholder='e.g: Authorization'
         ),
     )
     custom_header_val = colander.SchemaNode(
@@ -244,7 +115,7 @@ class WebhookSettingsSchema(colander.Schema):
         default='',
         missing='',
         widget=deform.widget.TextInputWidget(
-            placeholder='e.g. RcLogin auth=xxxx'
+            placeholder='e.g. Basic XxXxXx'
         ),
     )
     method_type = colander.SchemaNode(
@@ -264,8 +135,11 @@ class WebhookSettingsSchema(colander.Schema):
 class WebhookIntegrationType(IntegrationTypeBase):
     key = 'webhook'
     display_name = _('Webhook')
-    description = _('Post json events to a Webhook endpoint')
-    icon = '''<?xml version="1.0" encoding="UTF-8" standalone="no"?><svg viewBox="0 0 256 239" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" preserveAspectRatio="xMidYMid"><g><path d="M119.540432,100.502743 C108.930124,118.338815 98.7646301,135.611455 88.3876025,152.753617 C85.7226696,157.154315 84.4040417,160.738531 86.5332204,166.333309 C92.4107024,181.787152 84.1193605,196.825836 68.5350381,200.908244 C53.8383677,204.759349 39.5192953,195.099955 36.6032893,179.365384 C34.0194114,165.437749 44.8274148,151.78491 60.1824106,149.608284 C61.4694072,149.424428 62.7821041,149.402681 64.944891,149.240571 C72.469175,136.623655 80.1773157,123.700312 88.3025935,110.073173 C73.611854,95.4654658 64.8677898,78.3885437 66.803227,57.2292132 C68.1712787,42.2715849 74.0527146,29.3462646 84.8033863,18.7517722 C105.393354,-1.53572199 136.805164,-4.82141828 161.048542,10.7510424 C184.333097,25.7086706 194.996783,54.8450075 185.906752,79.7822957 C179.052655,77.9239597 172.151111,76.049808 164.563565,73.9917997 C167.418285,60.1274266 165.306899,47.6765751 155.95591,37.0109123 C149.777932,29.9690049 141.850349,26.2780332 132.835442,24.9178894 C114.764113,22.1877169 97.0209573,33.7983633 91.7563309,51.5355878 C85.7800012,71.6669027 94.8245623,88.1111998 119.540432,100.502743 L119.540432,100.502743 Z" fill="#C73A63"></path><path d="M149.841194,79.4106285 C157.316054,92.5969067 164.905578,105.982857 172.427885,119.246236 C210.44865,107.483365 239.114472,128.530009 249.398582,151.063322 C261.81978,178.282014 253.328765,210.520191 228.933162,227.312431 C203.893073,244.551464 172.226236,241.605803 150.040866,219.46195 C155.694953,214.729124 161.376716,209.974552 167.44794,204.895759 C189.360489,219.088306 208.525074,218.420096 222.753207,201.614016 C234.885769,187.277151 234.622834,165.900356 222.138374,151.863988 C207.730339,135.66681 188.431321,135.172572 165.103273,150.721309 C155.426087,133.553447 145.58086,116.521995 136.210101,99.2295848 C133.05093,93.4015266 129.561608,90.0209366 122.440622,88.7873178 C110.547271,86.7253555 102.868785,76.5124151 102.408155,65.0698097 C101.955433,53.7537294 108.621719,43.5249733 119.04224,39.5394355 C129.363912,35.5914599 141.476705,38.7783085 148.419765,47.554004 C154.093621,54.7244134 155.896602,62.7943365 152.911402,71.6372484 C152.081082,74.1025091 151.00562,76.4886916 149.841194,79.4106285 L149.841194,79.4106285 Z" fill="#4B4B4B"></path><path d="M167.706921,187.209935 L121.936499,187.209935 C117.54964,205.253587 108.074103,219.821756 91.7464461,229.085759 C79.0544063,236.285822 65.3738898,238.72736 50.8136292,236.376762 C24.0061432,232.053165 2.08568567,207.920497 0.156179306,180.745298 C-2.02835403,149.962159 19.1309765,122.599149 47.3341915,116.452801 C49.2814904,123.524363 51.2485589,130.663141 53.1958579,137.716911 C27.3195169,150.919004 18.3639187,167.553089 25.6054984,188.352614 C31.9811726,206.657224 50.0900643,216.690262 69.7528413,212.809503 C89.8327554,208.847688 99.9567329,192.160226 98.7211371,165.37844 C117.75722,165.37844 136.809118,165.180745 155.847178,165.475311 C163.280522,165.591951 169.019617,164.820939 174.620326,158.267339 C183.840836,147.48306 200.811003,148.455721 210.741239,158.640984 C220.88894,169.049642 220.402609,185.79839 209.663799,195.768166 C199.302587,205.38802 182.933414,204.874012 173.240413,194.508846 C171.247644,192.37176 169.677943,189.835329 167.706921,187.209935 L167.706921,187.209935 Z" fill="#4A4A4A"></path></g></svg>'''
+    description = _('send JSON data to a url endpoint')
+
+    @classmethod
+    def icon(cls):
+        return '''<?xml version="1.0" encoding="UTF-8" standalone="no"?><svg viewBox="0 0 256 239" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" preserveAspectRatio="xMidYMid"><g><path d="M119.540432,100.502743 C108.930124,118.338815 98.7646301,135.611455 88.3876025,152.753617 C85.7226696,157.154315 84.4040417,160.738531 86.5332204,166.333309 C92.4107024,181.787152 84.1193605,196.825836 68.5350381,200.908244 C53.8383677,204.759349 39.5192953,195.099955 36.6032893,179.365384 C34.0194114,165.437749 44.8274148,151.78491 60.1824106,149.608284 C61.4694072,149.424428 62.7821041,149.402681 64.944891,149.240571 C72.469175,136.623655 80.1773157,123.700312 88.3025935,110.073173 C73.611854,95.4654658 64.8677898,78.3885437 66.803227,57.2292132 C68.1712787,42.2715849 74.0527146,29.3462646 84.8033863,18.7517722 C105.393354,-1.53572199 136.805164,-4.82141828 161.048542,10.7510424 C184.333097,25.7086706 194.996783,54.8450075 185.906752,79.7822957 C179.052655,77.9239597 172.151111,76.049808 164.563565,73.9917997 C167.418285,60.1274266 165.306899,47.6765751 155.95591,37.0109123 C149.777932,29.9690049 141.850349,26.2780332 132.835442,24.9178894 C114.764113,22.1877169 97.0209573,33.7983633 91.7563309,51.5355878 C85.7800012,71.6669027 94.8245623,88.1111998 119.540432,100.502743 L119.540432,100.502743 Z" fill="#C73A63"></path><path d="M149.841194,79.4106285 C157.316054,92.5969067 164.905578,105.982857 172.427885,119.246236 C210.44865,107.483365 239.114472,128.530009 249.398582,151.063322 C261.81978,178.282014 253.328765,210.520191 228.933162,227.312431 C203.893073,244.551464 172.226236,241.605803 150.040866,219.46195 C155.694953,214.729124 161.376716,209.974552 167.44794,204.895759 C189.360489,219.088306 208.525074,218.420096 222.753207,201.614016 C234.885769,187.277151 234.622834,165.900356 222.138374,151.863988 C207.730339,135.66681 188.431321,135.172572 165.103273,150.721309 C155.426087,133.553447 145.58086,116.521995 136.210101,99.2295848 C133.05093,93.4015266 129.561608,90.0209366 122.440622,88.7873178 C110.547271,86.7253555 102.868785,76.5124151 102.408155,65.0698097 C101.955433,53.7537294 108.621719,43.5249733 119.04224,39.5394355 C129.363912,35.5914599 141.476705,38.7783085 148.419765,47.554004 C154.093621,54.7244134 155.896602,62.7943365 152.911402,71.6372484 C152.081082,74.1025091 151.00562,76.4886916 149.841194,79.4106285 L149.841194,79.4106285 Z" fill="#4B4B4B"></path><path d="M167.706921,187.209935 L121.936499,187.209935 C117.54964,205.253587 108.074103,219.821756 91.7464461,229.085759 C79.0544063,236.285822 65.3738898,238.72736 50.8136292,236.376762 C24.0061432,232.053165 2.08568567,207.920497 0.156179306,180.745298 C-2.02835403,149.962159 19.1309765,122.599149 47.3341915,116.452801 C49.2814904,123.524363 51.2485589,130.663141 53.1958579,137.716911 C27.3195169,150.919004 18.3639187,167.553089 25.6054984,188.352614 C31.9811726,206.657224 50.0900643,216.690262 69.7528413,212.809503 C89.8327554,208.847688 99.9567329,192.160226 98.7211371,165.37844 C117.75722,165.37844 136.809118,165.180745 155.847178,165.475311 C163.280522,165.591951 169.019617,164.820939 174.620326,158.267339 C183.840836,147.48306 200.811003,148.455721 210.741239,158.640984 C220.88894,169.049642 220.402609,185.79839 209.663799,195.768166 C199.302587,205.38802 182.933414,204.874012 173.240413,194.508846 C171.247644,192.37176 169.677943,189.835329 167.706921,187.209935 L167.706921,187.209935 Z" fill="#4A4A4A"></path></g></svg>'''
 
     valid_events = [
         events.PullRequestCloseEvent,
@@ -293,8 +167,8 @@ class WebhookIntegrationType(IntegrationTypeBase):
         return schema
 
     def send_event(self, event):
-        log.debug('handling event %s with Webhook integration %s',
-            event.name, self)
+        log.debug(
+            'handling event %s with Webhook integration %s', event.name, self)
 
         if event.__class__ not in self.valid_events:
             log.debug('event not valid: %r' % event)
@@ -313,8 +187,7 @@ class WebhookIntegrationType(IntegrationTypeBase):
         if head_key and head_val:
             headers = {head_key: head_val}
 
-        handler = WebhookHandler(
-            template_url, self.settings['secret_token'], headers)
+        handler = WebhookDataHandler(template_url, headers)
 
         url_calls = handler(event, data)
         log.debug('webhook: calling following urls: %s',
@@ -370,7 +243,10 @@ def post_to_webhook(url_calls, settings):
             rhodecode.__version__)
     }  # updated below with custom ones, allows override
 
-    for url, token, headers, data in url_calls:
+    auth = get_auth(settings)
+    token = get_web_token(settings)
+
+    for url, headers, data in url_calls:
         req_session = requests.Session()
         req_session.mount(  # retry max N times
             'http://', requests.adapters.HTTPAdapter(max_retries=retries))
@@ -380,7 +256,6 @@ def post_to_webhook(url_calls, settings):
 
         headers = headers or {}
         call_headers.update(headers)
-        auth = get_auth(settings)
 
         log.debug('calling Webhook with method: %s, and auth:%s',
                   call_method, auth)
@@ -392,4 +267,8 @@ def post_to_webhook(url_calls, settings):
         }, headers=call_headers, auth=auth)
         log.debug('Got Webhook response: %s', resp)
 
-        resp.raise_for_status()  # raise exception on a failed request
+        try:
+            resp.raise_for_status()  # raise exception on a failed request
+        except Exception:
+            log.error(resp.text)
+            raise

rhodecode/lib/caching_query.py

@@ -99,8 +99,11 @@ class CachingQuery(Query):
 
         """
         if hasattr(self, '_cache_parameters'):
-            return self.get_value(createfunc=lambda:
-                                  list(Query.__iter__(self)))
+
+            def caching_query():
+                return list(Query.__iter__(self))
+
+            return self.get_value(createfunc=caching_query)
         else:
             return Query.__iter__(self)
 

rhodecode/lib/codeblocks.py

@@ -28,8 +28,9 @@ from pygments.lexers.special import TextLexer, Token
 
 from rhodecode.lib.helpers import (
     get_lexer_for_filenode, html_escape, get_custom_lexer)
-from rhodecode.lib.utils2 import AttributeDict
+from rhodecode.lib.utils2 import AttributeDict, StrictAttributeDict
 from rhodecode.lib.vcs.nodes import FileNode
+from rhodecode.lib.vcs.exceptions import VCSError, NodeDoesNotExistError
 from rhodecode.lib.diff_match_patch import diff_match_patch
 from rhodecode.lib.diffs import LimitedDiffContainer
 from pygments.lexers import get_lexer_by_name
@@ -38,7 +39,7 @@ plain_text_lexer = get_lexer_by_name(
     'text', stripall=False, stripnl=False, ensurenl=False)
 
 
-log = logging.getLogger()
+log = logging.getLogger(__name__)
 
 
 def filenode_as_lines_tokens(filenode, lexer=None):
@@ -351,6 +352,16 @@ def tokens_diff(old_tokens, new_tokens, use_diff_match_patch=True):
     return old_tokens_result, new_tokens_result, similarity
 
 
+def diffset_node_getter(commit):
+    def get_node(fname):
+        try:
+            return commit.get_node(fname)
+        except NodeDoesNotExistError:
+            return None
+
+    return get_node
+
+
 class DiffSet(object):
     """
     An object for parsing the diff result from diffs.DiffProcessor and
@@ -400,7 +411,12 @@ class DiffSet(object):
         for patch in patchset:
             diffset.file_stats[patch['filename']] = patch['stats']
             filediff = self.render_patch(patch)
-            filediff.diffset = diffset
+            filediff.diffset = StrictAttributeDict(dict(
+                source_ref=diffset.source_ref,
+                target_ref=diffset.target_ref,
+                repo_name=diffset.repo_name,
+                source_repo_name=diffset.source_repo_name,
+            ))
             diffset.files.append(filediff)
             diffset.changed_files += 1
             if not patch['stats']['binary']:
@@ -510,6 +526,7 @@ class DiffSet(object):
         if target_file_path in self.comments_store:
             for lineno, comments in self.comments_store[target_file_path].items():
                 left_comments[lineno] = comments
+
         # left comments are one that we couldn't place in diff lines.
         # could be outdated, or the diff changed and this line is no
         # longer available
@@ -546,7 +563,7 @@ class DiffSet(object):
 
         result.lines.extend(
             self.parse_lines(before, after, source_file, target_file))
-        result.unified = self.as_unified(result.lines)
+        result.unified = list(self.as_unified(result.lines))
         result.sideside = result.lines
 
         return result
@@ -601,8 +618,9 @@ class DiffSet(object):
                 original.lineno = before['old_lineno']
                 original.content = before['line']
                 original.action = self.action_to_op(before['action'])
-                original.comments = self.get_comments_for('old',
-                    source_file, before['old_lineno'])
+
+                original.get_comment_args = (
+                    source_file, 'o', before['old_lineno'])
 
             if after:
                 if after['action'] == 'new-no-nl':
@@ -614,8 +632,9 @@ class DiffSet(object):
                 modified.lineno = after['new_lineno']
                 modified.content = after['line']
                 modified.action = self.action_to_op(after['action'])
-                modified.comments = self.get_comments_for('new',
-                    target_file, after['new_lineno'])
+
+                modified.get_comment_args = (
+                    target_file, 'n', after['new_lineno'])
 
             # diff the lines
             if before_tokens and after_tokens:
@@ -644,23 +663,6 @@ class DiffSet(object):
 
         return lines
 
-    def get_comments_for(self, version, filename, line_number):
-        if hasattr(filename, 'unicode_path'):
-            filename = filename.unicode_path
-
-        if not isinstance(filename, basestring):
-            return None
-
-        line_key = {
-            'old': 'o',
-            'new': 'n',
-        }[version] + str(line_number)
-
-        if filename in self.comments_store:
-            file_comments = self.comments_store[filename]
-            if line_key in file_comments:
-                return file_comments.pop(line_key)
-
     def get_line_tokens(self, line_text, line_number, file=None):
         filenode = None
         filename = None
@@ -717,25 +719,25 @@ class DiffSet(object):
                     if line.original.action == ' ':
                         yield (line.original.lineno, line.modified.lineno,
                                line.original.action, line.original.content,
-                               line.original.comments)
+                               line.original.get_comment_args)
                         continue
 
                     if line.original.action == '-':
                         yield (line.original.lineno, None,
                                line.original.action, line.original.content,
-                               line.original.comments)
+                               line.original.get_comment_args)
 
                     if line.modified.action == '+':
                         buf.append((
                             None, line.modified.lineno,
                             line.modified.action, line.modified.content,
-                            line.modified.comments))
+                            line.modified.get_comment_args))
                         continue
 
                 if line.modified:
                     yield (None, line.modified.lineno,
                            line.modified.action, line.modified.content,
-                           line.modified.comments)
+                           line.modified.get_comment_args)
 
             for b in buf:
                 yield b

rhodecode/lib/db_manage.py

@@ -126,7 +126,7 @@ class DbManage(object):
         log.info("Deleting (%s) cache keys now...", total)
         CacheKey.delete_all_cache()
 
-    def upgrade(self):
+    def upgrade(self, version=None):
         """
         Upgrades given database schema to given revision following
         all needed steps, to perform the upgrade
@@ -157,9 +157,9 @@ class DbManage(object):
         db_uri = self.dburi
 
         try:
-            curr_version = api.db_version(db_uri, repository_path)
-            msg = ('Found current database under version '
-                   'control with version %s' % curr_version)
+            curr_version = version or api.db_version(db_uri, repository_path)
+            msg = ('Found current database db_uri under version '
+                   'control with version {}'.format(curr_version))
 
         except (RuntimeError, DatabaseNotControlledError):
             curr_version = 1