diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 984990c9..de82363c 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 4.11.6 +current_version = 4.12.0 message = release: Bump version {current_version} to {new_version} [bumpversion:file:rhodecode/VERSION] diff --git a/.release.cfg b/.release.cfg index e9745397..8a4cddf8 100644 --- a/.release.cfg +++ b/.release.cfg @@ -5,26 +5,21 @@ done = false done = true [task:rc_tools_pinned] -done = true [task:fixes_on_stable] -done = true [task:pip2nix_generated] -done = true [task:changelog_updated] -done = true [task:generate_api_docs] -done = true - -[release] -state = prepared -version = 4.11.6 [task:updated_translation] +[release] +state = in_progress +version = 4.12.0 + [task:generate_js_routes] [task:updated_trial_license] diff --git a/MANIFEST.in b/MANIFEST.in index 3e292ac8..a7a5ef7d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -16,9 +16,6 @@ recursive-include configs * # translations recursive-include rhodecode/i18n * -# hook templates -recursive-include rhodecode/config/hook_templates * - # non-python core stuff recursive-include rhodecode *.cfg recursive-include rhodecode *.json diff --git a/configs/development.ini b/configs/development.ini index 51a71cda..962aeff1 100644 --- a/configs/development.ini +++ b/configs/development.ini @@ -190,9 +190,6 @@ force_https = false ## use Strict-Transport-Security headers use_htsts = false -## number of commits stats will parse on each iteration -commit_parse_limit = 25 - ## git rev filter option, --all is the default filter, if you need to ## hide all refs in changelog switch this to --branches --tags git_rev_filter = --branches --tags @@ -708,12 +705,12 @@ formatter = color_formatter_sql [formatter_generic] class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter -format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s +format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s datefmt = %Y-%m-%d %H:%M:%S [formatter_color_formatter] class = rhodecode.lib.logging_formatter.ColorFormatter -format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s +format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s datefmt = %Y-%m-%d %H:%M:%S [formatter_color_formatter_sql] diff --git a/configs/gunicorn_config.py b/configs/gunicorn_config.py index 9abfbdd8..db796506 100644 --- a/configs/gunicorn_config.py +++ b/configs/gunicorn_config.py @@ -55,12 +55,13 @@ keepalive = 2 # SERVER MECHANICS # None == system temp dir +# worker_tmp_dir is recommended to be set to some tmpfs worker_tmp_dir = None tmp_upload_dir = None # Custom log format access_log_format = ( - '%(t)s GNCRN %(p)-8s %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"') + '%(t)s [%(p)-8s] GNCRN %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"') # self adjust workers based on CPU count # workers = multiprocessing.cpu_count() * 2 + 1 @@ -119,16 +120,16 @@ def child_exit(server, worker): def pre_request(worker, req): - return - worker.log.debug("[<%-10s>] PRE WORKER: %s %s", - worker.pid, req.method, req.path) + worker.start_time = time.time() + worker.log.debug( + "GNCRN PRE WORKER [cnt:%s]: %s %s", worker.nr, req.method, req.path) def post_request(worker, req, environ, resp): - return - worker.log.debug("[<%-10s>] POST WORKER: %s %s resp: %s", worker.pid, - req.method, req.path, resp.status_code) - + total_time = time.time() - worker.start_time + worker.log.debug( + "GNCRN POST WORKER [cnt:%s]: %s %s resp: %s, Load Time: %.3fs", + worker.nr, req.method, req.path, resp.status_code, total_time) class RhodeCodeLogger(Logger): diff --git a/configs/production.ini b/configs/production.ini index c169ce7c..814c0512 100644 --- a/configs/production.ini +++ b/configs/production.ini @@ -165,9 +165,6 @@ force_https = false ## use Strict-Transport-Security headers use_htsts = false -## number of commits stats will parse on each iteration -commit_parse_limit = 25 - ## git rev filter option, --all is the default filter, if you need to ## hide all refs in changelog switch this to --branches --tags git_rev_filter = --branches --tags @@ -678,12 +675,12 @@ formatter = generic [formatter_generic] class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter -format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s +format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s datefmt = %Y-%m-%d %H:%M:%S [formatter_color_formatter] class = rhodecode.lib.logging_formatter.ColorFormatter -format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s +format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s datefmt = %Y-%m-%d %H:%M:%S [formatter_color_formatter_sql] diff --git a/docs/admin/sec-sophos-umc.rst b/docs/admin/sec-sophos-umc.rst index 7021b95a..264ab568 100644 --- a/docs/admin/sec-sophos-umc.rst +++ b/docs/admin/sec-sophos-umc.rst @@ -1,4 +1,4 @@ -.. _sec-your-server: +.. _sec-sophos-umc: Securing Your Server via Sophos UTM 9 ------------------------------------- diff --git a/docs/admin/svn-path-permissions.rst b/docs/admin/svn-path-permissions.rst new file mode 100644 index 00000000..19aff10b --- /dev/null +++ b/docs/admin/svn-path-permissions.rst @@ -0,0 +1,80 @@ +.. _svn-path-permissions: + +|svn| Enabling Path Permissions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Because |RCEE| uses standard svn apache mod_svn we can take advantage of the +authz configuration to protect paths and branches. + + +Configuring RhodeCode +===================== + + +1. To configure path based permissions first we need to use a customized + mod_dav_svn.conf. + + Open :file:`home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file. + And find `svn.proxy.config_template` setting. Now set a new path to read + the template from. For example: + + .. code-block:: ini + + svn.proxy.config_template = /home/ubuntu/rhodecode/custom_mod_dav_svn.conf.mako + + +2. Create the file as in example: `/home/ubuntu/rhodecode/custom_mod_dav_svn.conf.mako` + You can download one from: + + ``_ + +3. Add (if not yet exists) a section `AuthzSVNReposRelativeAccessFile` in order + to read the path auth file. + + Example modified config section enabling reading the authz file relative + to repository path. Means located in `/storage_dir/repo_name/conf/authz` + + .. code-block:: text + + + # snip ... + + # use specific SVN conf/authz file for each repository + AuthzSVNReposRelativeAccessFile authz + + Allow from all + # snip ... + + .. note:: + + The `AuthzSVNReposRelativeAccessFile` should go above the `Allow from all` + directive. + + +4. Restart RhodeCode, Go to + the :menuselection:`Admin --> Settings --> VCS` page, and + click :guilabel:`Generate Apache Config`. + This will now generate a new configuration with enabled changes to read + the authz file. You can verify if changes were made by checking the generated + mod_dav_svn.conf file which is included in your apache configuration. + +5. Specify new rules in the repository authz configuration. + edit a file in :file:`repo_name/conf/authz`. For example, we specify that + only admin is allowed to push to develop branch + + .. code-block:: ini + + [/branches/develop] + * = r + admin = rw + + + For more example see: + ``_ + + Those rules also work for paths, so not only branches but all different + paths inside the repository can be specified. + +6. Reload Apache. If all is configured correctly it should not be allowed to + commit according to specified rules. + diff --git a/docs/admin/system-admin.rst b/docs/admin/system-admin.rst index 82fb040c..0666ac76 100644 --- a/docs/admin/system-admin.rst +++ b/docs/admin/system-admin.rst @@ -19,6 +19,7 @@ The following are the most common system administration tasks. config-files-overview vcs-server svn-http + svn-path-permissions gunicorn-ssl-support apache-config nginx-config diff --git a/docs/api/api.rst b/docs/api/api.rst index 1f861fad..74873511 100644 --- a/docs/api/api.rst +++ b/docs/api/api.rst @@ -196,6 +196,7 @@ are not required in args. .. --- API DEFS MARKER --- .. toctree:: + methods/views methods/license-methods methods/deprecated-methods methods/gist-methods diff --git a/docs/api/methods/repo-methods.rst b/docs/api/methods/repo-methods.rst index 0cf9787b..bda6a676 100644 --- a/docs/api/methods/repo-methods.rst +++ b/docs/api/methods/repo-methods.rst @@ -66,7 +66,7 @@ comment_commit create_repo ----------- -.. py:function:: create_repo(apiuser, repo_name, repo_type, owner=>, description=, private=, clone_uri=, landing_rev=, enable_statistics=, enable_locking=, enable_downloads=, copy_permissions=) +.. py:function:: create_repo(apiuser, repo_name, repo_type, owner=>, description=, private=, clone_uri=, push_uri=, landing_rev=, enable_statistics=, enable_locking=, enable_downloads=, copy_permissions=) Creates a repository. @@ -95,6 +95,8 @@ create_repo :type private: bool :param clone_uri: set clone_uri :type clone_uri: str + :param push_uri: set push_uri + :type push_uri: str :param landing_rev: : :type landing_rev: str :param enable_locking: @@ -789,7 +791,7 @@ maintenance pull ---- -.. py:function:: pull(apiuser, repoid) +.. py:function:: pull(apiuser, repoid, remote_uri=) Triggers a pull on the given repository from a remote location. You can use this to keep remote repositories up-to-date. @@ -804,6 +806,8 @@ pull :type apiuser: AuthUser :param repoid: The repository name or repository ID. :type repoid: str or int + :param remote_uri: Optional remote URI to pass in for pull + :type remote_uri: str Example output: @@ -811,7 +815,7 @@ pull id : result : { - "msg": "Pulled from ``" + "msg": "Pulled from url `` on repo ``" "repository": "" } error : null @@ -823,7 +827,7 @@ pull id : result : null error : { - "Unable to pull changes from ``" + "Unable to push changes from ``" } @@ -976,7 +980,7 @@ strip update_repo ----------- -.. py:function:: update_repo(apiuser, repoid, repo_name=, owner=>, description=, private=, clone_uri=, landing_rev=, fork_of=, enable_statistics=, enable_locking=, enable_downloads=, fields=) +.. py:function:: update_repo(apiuser, repoid, repo_name=, owner=>, description=, private=, clone_uri=, push_uri=, landing_rev=, fork_of=, enable_statistics=, enable_locking=, enable_downloads=, fields=) Updates a repository with the given information. diff --git a/docs/api/methods/user-group-methods.rst b/docs/api/methods/user-group-methods.rst index cb8b4179..85707108 100644 --- a/docs/api/methods/user-group-methods.rst +++ b/docs/api/methods/user-group-methods.rst @@ -51,7 +51,7 @@ add_user_to_user_group create_user_group ----------------- -.. py:function:: create_user_group(apiuser, group_name, description=, owner=>, active=) +.. py:function:: create_user_group(apiuser, group_name, description=, owner=>, active=, sync=) Creates a new user group. @@ -71,6 +71,9 @@ create_user_group :type owner: Optional(str or int) :param active: Set this group as active. :type active: Optional(``True`` | ``False``) + :param sync: Set enabled or disabled the automatically sync from + external authentication types like ldap. + :type sync: Optional(``True`` | ``False``) Example output: @@ -368,7 +371,7 @@ revoke_user_permission_from_user_group update_user_group ----------------- -.. py:function:: update_user_group(apiuser, usergroupid, group_name=, description=, owner=, active=) +.. py:function:: update_user_group(apiuser, usergroupid, group_name=, description=, owner=, active=, sync=) Updates the specified `user group` with the details provided. @@ -387,6 +390,9 @@ update_user_group :type owner: Optional(str or int) :param active: Set the group as active. :type active: Optional(``True`` | ``False``) + :param sync: Set enabled or disabled the automatically sync from + external authentication types like ldap. + :type sync: Optional(``True`` | ``False``) Example output: diff --git a/docs/api/methods/views.rst b/docs/api/methods/views.rst new file mode 100644 index 00000000..c47d733e --- /dev/null +++ b/docs/api/methods/views.rst @@ -0,0 +1,48 @@ +.. _views-ref: + +views +===== + +push (EE only) +-------------- + +.. py:function:: push(apiuser, repoid, remote_uri=) + + Triggers a push on the given repository from a remote location. You + can use this to keep remote repositories up-to-date. + + This command can only be run using an |authtoken| with admin + rights to the specified repository. For more information, + see :ref:`config-token-ref`. + + This command takes the following options: + + :param apiuser: This is filled automatically from the |authtoken|. + :type apiuser: AuthUser + :param repoid: The repository name or repository ID. + :type repoid: str or int + :param remote_uri: Optional remote URI to pass in for push + :type remote_uri: str + + Example output: + + .. code-block:: bash + + id : + result : { + "msg": "Pushed to url `` on repo ``" + "repository": "" + } + error : null + + Example error output: + + .. code-block:: bash + + id : + result : null + error : { + "Unable to push changes to ``" + } + + diff --git a/docs/auth/crowd-auth.rst b/docs/auth/auth-crowd.rst similarity index 100% rename from docs/auth/crowd-auth.rst rename to docs/auth/auth-crowd.rst diff --git a/docs/auth/auth-ldap-groups.rst b/docs/auth/auth-ldap-groups.rst new file mode 100644 index 00000000..7aa0dd36 --- /dev/null +++ b/docs/auth/auth-ldap-groups.rst @@ -0,0 +1,112 @@ +.. _config-ldap-groups-ref: + +LDAP/AD With User Groups Sync +----------------------------- + +|RCM| supports LDAP (Lightweight Directory Access Protocol) or +AD (active Directory) authentication. +All LDAP versions are supported, with the following |RCM| plugins managing each: + +* For LDAP/AD with user group sync use ``LDAP + User Groups (egg:rhodecode-enterprise-ee#ldap_group)`` + +RhodeCode reads all data defined from plugin and creates corresponding +accounts on local database after receiving data from LDAP. This is done on +every user log-in including operations like pushing/pulling/checkout. +In addition group membership is read from LDAP and following operations are done: + +- automatic addition of user to |RCM| user group +- automatic removal of user from any other |RCM| user groups not specified in LDAP. + The removal is done *only* on groups that are marked to be synced from ldap. + This setting can be changed in advanced settings on user groups +- automatic creation of user groups if they aren't yet existing in |RCM| +- marking user as super-admins if he is a member of any admin group defined in plugin settings + +This plugin is available only in EE Edition. + +.. important:: + + The email used with your |RCE| super-admin account needs to match the email + address attached to your admin profile in LDAP. This is because + within |RCE| the user email needs to be unique, and multiple users + cannot share an email account. + + Likewise, if as an admin you also have a user account, the email address + attached to the user account needs to be different. + + +LDAP Configuration Steps +^^^^^^^^^^^^^^^^^^^^^^^^ + +To configure |LDAP|, use the following steps: + +1. From the |RCM| interface, select + :menuselection:`Admin --> Authentication` +2. Enable the ldap+ groups plugin and select :guilabel:`Save` +3. Select the :guilabel:`Enabled` check box in the plugin configuration section +4. Add the required LDAP information and :guilabel:`Save`, for more details, + see :ref:`config-ldap-groups-examples` + +For a more detailed description of LDAP objects, see :ref:`ldap-gloss-ref`: + +.. _config-ldap-groups-examples: + +Example LDAP configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. code-block:: bash + + # Auth Cache TTL, Defines the caching for authentication to offload LDAP server. + # This means that cache result will be saved for 3600 before contacting LDAP server to verify the user access + 3600 + # Host, comma seperated format is optionally possible to specify more than 1 server + https://ldap1.server.com/ldap-admin/,https://ldap2.server.com/ldap-admin/ + # Default LDAP Port, use 689 for LDAPS + 389 + # Account, used for SimpleBind if LDAP server requires an authentication + e.g admin@server.com + # Password used for simple bind + ldap-user-password + # LDAP connection security + LDAPS + # Certificate checks level + DEMAND + # Base DN + cn=Rufus Magillacuddy,ou=users,dc=rhodecode,dc=com + # User Search Base + ou=groups,ou=users + # LDAP search filter to narrow the results + (objectClass=person) + # LDAP search scope + SUBTREE + # Login attribute + sAMAccountName + # First Name Attribute to read + givenName + # Last Name Attribute to read + sn + # Email Attribute to read email address from + mail + # group extraction method + rfc2307bis + # Group search base + ou=RC-Groups + # Group Name Attribute, field to read the group name from + sAMAAccountName + # User Member of Attribute, field in which groups are stored + memberOf + # LDAP Group Search Filter, allows narrowing the results + + # Admin Groups. Comma separated list of groups. If user is member of + # any of those he will be marked as super-admin in RhodeCode + admins, management + + +Below is example setup that can be used with Active Directory and ldap groups. + +.. image:: ../images/ldap-groups-example.png + :alt: LDAP/AD setup example + :scale: 50 % + +.. toctree:: + + ldap-active-directory + ldap-authentication \ No newline at end of file diff --git a/docs/auth/ldap-config-steps.rst b/docs/auth/auth-ldap.rst similarity index 56% rename from docs/auth/ldap-config-steps.rst rename to docs/auth/auth-ldap.rst index 9a600f1b..76729027 100644 --- a/docs/auth/ldap-config-steps.rst +++ b/docs/auth/auth-ldap.rst @@ -1,14 +1,17 @@ .. _config-ldap-ref: -LDAP ----- +LDAP/AD +------- |RCM| supports LDAP (Lightweight Directory Access Protocol) or AD (active Directory) authentication. All LDAP versions are supported, with the following |RCM| plugins managing each: -* For LDAPv3 use ``LDAP (egg:rhodecode-enterprise-ce#ldap)`` -* For LDAPv3 with user group sync use ``LDAP + User Groups (egg:rhodecode-enterprise-ee#ldap_group)`` +* For LDAP or Active Directory use ``LDAP (egg:rhodecode-enterprise-ce#ldap)`` + +RhodeCode reads all data defined from plugin and creates corresponding +accounts on local database after receiving data from LDAP. This is done on +every user log-in including operations like pushing/pulling/checkout. .. important:: @@ -21,6 +24,7 @@ All LDAP versions are supported, with the following |RCM| plugins managing each: Likewise, if as an admin you also have a user account, the email address attached to the user account needs to be different. + LDAP Configuration Steps ^^^^^^^^^^^^^^^^^^^^^^^^ @@ -28,7 +32,7 @@ To configure |LDAP|, use the following steps: 1. From the |RCM| interface, select :menuselection:`Admin --> Authentication` -2. Enable the required plugin and select :guilabel:`Save` +2. Enable the ldap plugin and select :guilabel:`Save` 3. Select the :guilabel:`Enabled` check box in the plugin configuration section 4. Add the required LDAP information and :guilabel:`Save`, for more details, see :ref:`config-ldap-examples` @@ -41,15 +45,16 @@ Example LDAP configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^ .. code-block:: bash - # Auth Cache TTL + # Auth Cache TTL, Defines the caching for authentication to offload LDAP server. + # This means that cache result will be saved for 3600 before contacting LDAP server to verify the user access 3600 - # Host + # Host, comma seperated format is optionally possible to specify more than 1 server https://ldap1.server.com/ldap-admin/,https://ldap2.server.com/ldap-admin/ - # Port + # Default LDAP Port, use 689 for LDAPS 389 - # Account - cn=admin,dc=rhodecode,dc=com - # Password + # Account, used for SimpleBind if LDAP server requires an authentication + e.g admin@server.com + # Password used for simple bind ldap-user-password # LDAP connection security LDAPS @@ -57,32 +62,26 @@ Example LDAP configuration DEMAND # Base DN cn=Rufus Magillacuddy,ou=users,dc=rhodecode,dc=com - # User Search Base - ou=groups,ou=users - # LDAP search filter + # LDAP search filter to narrow the results (objectClass=person) # LDAP search scope SUBTREE # Login attribute - rmagillacuddy - # First Name Attribute - Rufus - # Last Name Attribute - Magillacuddy - # Email Attribute - LDAP-Registered@email.ac - # User Member of Attribute - Organizational Role - # Group search base - cn=users,ou=groups,dc=rhodecode,dc=com - # LDAP Group Search Filter - (objectclass=posixGroup) - # Group Name Attribute - users - # Group Member Of Attribute - cn - # Admin Groups - admin,devops,qa + sAMAccountName + # First Name Attribute to read + givenName + # Last Name Attribute to read + sn + # Email Attribute to read email address from + mail + + +Below is example setup that can be used with Active Directory/LDAP server. + +.. image:: ../images/ldap-example.png + :alt: LDAP/AD setup example + :scale: 50 % + .. toctree:: diff --git a/docs/auth/pam-auth.rst b/docs/auth/auth-pam.rst similarity index 100% rename from docs/auth/pam-auth.rst rename to docs/auth/auth-pam.rst diff --git a/docs/auth/token-auth.rst b/docs/auth/auth-token.rst similarity index 100% rename from docs/auth/token-auth.rst rename to docs/auth/auth-token.rst diff --git a/docs/auth/auth.rst b/docs/auth/auth.rst index 20aa4ef6..f2e197d2 100644 --- a/docs/auth/auth.rst +++ b/docs/auth/auth.rst @@ -3,35 +3,30 @@ Authentication Options ====================== -|RCE| provides a built in authentication plugin -``rhodecode.lib.auth_rhodecode``. This is enabled by default and accessed -through the administrative interface. Additionally, -|RCE| provides a Pluggable Authentication System (PAS). This gives the +|RCE| provides a built in authentication against its own database. This is +implemented using ``rhodecode.lib.auth_rhodecode`` plugin. This plugin is +enabled by default. +Additionally, |RCE| provides a Pluggable Authentication System. This gives the administrator greater control over how users authenticate with the system. .. important:: You can disable the built in |RCM| authentication plugin ``rhodecode.lib.auth_rhodecode`` and force all authentication to go - through your authentication plugin. However, if you do this, - and your external authentication tools fails, you will be unable to - access |RCM|. + through your authentication plugin of choice e.g LDAP only. + However, if you do this, and your external authentication tools fails, + you will be unable to access |RCM|. |RCM| comes with the following user authentication management plugins: -.. only:: latex - - * :ref:`config-ldap-ref` - * :ref:`config-pam-ref` - * :ref:`config-crowd-ref` - * :ref:`config-token-ref` .. toctree:: - ldap-config-steps - crowd-auth - pam-auth - token-auth + auth-ldap + auth-ldap-groups + auth-crowd + auth-pam + auth-token ssh-connection diff --git a/docs/auth/ldap-configuration-example.rst b/docs/auth/ldap-configuration-example.rst deleted file mode 100644 index 62f6d276..00000000 --- a/docs/auth/ldap-configuration-example.rst +++ /dev/null @@ -1,13 +0,0 @@ - LDAP Host hostname1,hostname2 # testing here - Port port - Account: `uid=admin,cn=users,cn=accounts,dc=localdomain,dc=tld` - Password: userpassword #Testing this here - Connection Security LDAPS - Certificate Checks ``NEVER`` - Base DN cn=users,cn=accounts,dc=localdomain,dc=tld - LDAP Search Filter (objectClass=person) - LDAP Search Scope SUBTREE - Login Attribute uid - First Name Attribute givenname - Last Name Attribute sn - Email Attribute mail diff --git a/docs/conf.py b/docs/conf.py index 8ab7fabc..15573a7b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -34,7 +34,7 @@ import common extensions = [ 'sphinx.ext.intersphinx', 'sphinx.ext.todo', - 'sphinx.ext.pngmath' + 'sphinx.ext.imgmath' ] intersphinx_mapping = { @@ -104,7 +104,6 @@ exclude_patterns = [ # Other RST files 'admin/rhodecode-backup.rst', - 'auth/ldap-configuration-example.rst', 'issue-trackers/redmine.rst', 'known-issues/error-msg-guide.rst', 'tutorials/docs-build.rst', diff --git a/docs/contributing/dev-setup.rst b/docs/contributing/dev-setup.rst index 343274d5..4e6cd6df 100644 --- a/docs/contributing/dev-setup.rst +++ b/docs/contributing/dev-setup.rst @@ -34,6 +34,13 @@ following commands: Update your channels frequently by running ``nix-channel --update``. +.. note:: + + To uninstall nix run the following: + + remove the . "$HOME/.nix-profile/etc/profile.d/nix.sh" line in your ~/.profile or ~/.bash_profile + rm -rf $HOME/{.nix-channels,.nix-defexpr,.nix-profile,.config/nixpkgs} + sudo rm -rf /nix Switch nix to the latest STABLE channel --------------------------------------- diff --git a/docs/default.nix b/docs/default.nix index a6f2cbc5..3798f997 100644 --- a/docs/default.nix +++ b/docs/default.nix @@ -169,17 +169,6 @@ let }; }; - setuptools = buildPythonPackage { - name = "setuptools-36.6.0"; - buildInputs = []; - doCheck = false; - propagatedBuildInputs = []; - src = fetchurl { - url = "https://pypi.python.org/packages/45/29/8814bf414e7cd1031e1a3c8a4169218376e284ea2553cc0822a6ea1c2d78/setuptools-36.6.0.zip"; - md5 = "74663b15117d9a2cc5295d76011e6fd1"; - }; - }; - six = buildPythonPackage { name = "six-1.11.0"; buildInputs = []; @@ -259,6 +248,9 @@ let }; + # Avoid that setuptools is replaced, this leads to trouble + # with buildPythonPackage. + setuptools = pkgs.python27Packages.setuptools; in python.buildEnv.override { inherit python; diff --git a/docs/images/ldap-example.png b/docs/images/ldap-example.png new file mode 100644 index 00000000..418d6d27 Binary files /dev/null and b/docs/images/ldap-example.png differ diff --git a/docs/release-notes/release-notes-4.12.0.rst b/docs/release-notes/release-notes-4.12.0.rst new file mode 100644 index 00000000..abae5262 --- /dev/null +++ b/docs/release-notes/release-notes-4.12.0.rst @@ -0,0 +1,139 @@ +|RCE| 4.12.0 |RNS| +------------------ + +Release Date +^^^^^^^^^^^^ + +- 2018-04-24 + + +New Features +^^^^^^^^^^^^ + +- Svn: added support for RhodeCode integration framework. All integrations like + slack, email, Jenkins now also fully work for SVN. +- Integrations: added new dedicated Jenkins integration with the support of + CSRF authentication. Available in EE edition only. +- Automation: added new bi-directional remote sync. RhodeCode instances can now + automatically push or pull from/to remote locations. This feature is powered + by the Scheduler of 4.11 release, and it is required to be enabled for this feature to work. + Available in EE edition only. +- Mercurial: path-based permissions. RhodeCode can now use Mercurials narrowhg + to implement path-based permissions. All permissions are read from .hg/hgacl. + Thanks to the great contribution from Sandu Turcan. +- VCS: added new diff caches. Available as an option under vcs settings. + Diff caches work on pull-request, or individual commits for greater + performance and reduced memory usage. This feature increases speed of large + pull requests significantly. In addition for pull requests it will allow + showing old closed pull requests even if commits from source were removed, + further enhancing auditing capabilities. +- Audit: added few new audit log entries especially around changing permissions. +- LDAP: added connection pinning and timeout option to ldap plugin. This should + prevent problems when connection to LDAP is not stable causing RhodeCode + instances to freeze waiting on LDAP connections. +- User groups: expose public user group profiles. Allows to see members of a user + groups by other team members, if they have proper permissions. +- UI: show pull request page in quick nav menu on my account for quicker access. +- UI: hidden/outdated comments now have visible markers next to line numbers. + This allows access to them without showing all hidden comments. + + +General +^^^^^^^ + +- Ssh: show conflicting fingerprint when adding an already existing key. + Helps to track why adding a key failed. +- System info: added ulimit to system info. This is causing lots of problems + when we hit any of those limits, that is why it's important to show this. +- Repository settings: add hidden view to force re-install hooks. + Available under /{repo_name}/settings/advanced/hooks +- Integrations: Webhook now handles response errors and show response for + easier debugging. +- Cli: speed up CLI execution start by skipping auth plugin search/registry. +- SVN: added an example in the docs on how to enable path-based permissions. +- LDAP: enable connection recycling on LDAP plugin. +- Auth plugins: use a nicer visual display of auth plugins that would + highlight that order of enabled plugins does matter. +- Events: expose shadow repo build url. +- Events: expose pull request title and uid in event data. +- API: enable setting sync flag for user groups on create/edit. +- API: update pull method with a possible specification of the url +- Logging: improved consistency of auth plugins logs. +- Logging: improved log for ssl required +- Dependencies: bumped mercurial to 4.4 series +- Dependencies: bumped zope.cachedescriptors==4.3.1 +- Dependencies: bumped zope.deprecation==4.3.0 +- Dependencies: bumped zope.event==4.3.0 +- Dependencies: bumped zope.interface==4.4.3 +- Dependencies: bumped graphviz 0.8.2 +- Dependencies: bumped to ipaddress 0.1.19 +- Dependencies: bumped pyexpect to 4.3.1 +- Dependencies: bumped ws4py to 0.4.3 +- Dependencies: bumped bleach to 2.1.2 +- Dependencies: bumped html5lib 1.0.1 +- Dependencies: bumped greenlet to 0.4.13 +- Dependencies: bumped markdown to 2.6.11 +- Dependencies: bumped psutil to 5.4.3 +- Dependencies: bumped beaker to 1.9.1 +- Dependencies: bumped alembic to 0.6.8 release. +- Dependencies: bumped supervisor to 3.3.4 +- Dependencies: bumped pyexpect to 4.4.0 and scandir to 1.7 +- Dependencies: bumped appenlight client to 0.6.25 +- Dependencies: don't require full mysql lib for the db driver. + Reduces installation package size by around 100MB. + + +Security +^^^^^^^^ + +- My account: changing email in my account now requires providing user + access password. This is a case for only RhodeCode built-in accounts. + Prevents adding recovery email by unauthorized users who gain + access to logged in session of user. +- Logging: fix leaking of tokens to logging. +- General: serialize the repo name in repo checks to prevent potential + html injections by providing a malformed url. + + +Performance +^^^^^^^^^^^ + +- Diffs: don't use recurred diffset attachment in diffs. This makes +this structure much harder to garbage collect. Reduces memory usage. +- Diff cache: added caching for better performance of large pull requests. + + +Fixes +^^^^^ + +- Age helper: fix issues with proper timezone detection for certain timezones. + Fixes wrong age display in few cases. +- API: added audit logs for user group related calls that were + accidentally missing. +- Diffs: fix and improve line selections and anchor links. +- Pull requests: fixed cases with default expected refs are closed or unavailable. + For Mercurial with closed default branch a compare across forks could fail. +- Core: properly report 502 errors for gevent and gunicorn. + Gevent wtih Gunicorn doesn't raise normal pycurl errors. +- Auth plugins: fixed problem with cache of settings in multi-worker mode. + The previous implementation had a bug that cached the settings in each class, + caused not refreshing the update of settings in multi-worker mode. + Only restart of RhodeCode loaded new settings. +- Audit logs: properly handle query syntax in the search field. +- Repositories: better handling of missing requirements errors for repositories. +- API: fixed problems with repository fork/create using celery backend. +- VCS settings: added missing flash message on validation errors to prevent + missing out some field input validation problems. + + +Upgrade notes +^^^^^^^^^^^^^ + +- This release adds support for SVN hook. This required lots of changes on how we +handle SVN protocol. We did thoughtful tests for SVN compatibility. +Please be advised to check the behaviour of SVN repositories during this update. + +- Diff caches are turned off by default for backward compatibility. We however recommend +turning them on either individually for bigger repositories or globally for every repository. +This setting can be found in admin > settings > vcs, or repository > settings > vcs + diff --git a/docs/release-notes/release-notes.rst b/docs/release-notes/release-notes.rst index f207a8b7..98022470 100644 --- a/docs/release-notes/release-notes.rst +++ b/docs/release-notes/release-notes.rst @@ -9,6 +9,7 @@ Release Notes .. toctree:: :maxdepth: 1 + release-notes-4.12.0.rst release-notes-4.11.6.rst release-notes-4.11.5.rst release-notes-4.11.4.rst diff --git a/docs/tutorials/deploy-from-host.rst b/docs/tutorials/deploy-from-host.rst index 3c63f2c9..ebd5cca1 100644 --- a/docs/tutorials/deploy-from-host.rst +++ b/docs/tutorials/deploy-from-host.rst @@ -88,7 +88,7 @@ Install |RCC| The |RCC| installer is now on your server, and you can read the full instructions here -:ref:`Install RhodeCode Control `, +:ref:`Install RhodeCode Control ` , but below is the example shortcut. .. code-block:: bash diff --git a/pkgs/patch-beaker-lock-func-debug.diff b/pkgs/patch-beaker-lock-func-debug.diff new file mode 100644 index 00000000..bede0d66 --- /dev/null +++ b/pkgs/patch-beaker-lock-func-debug.diff @@ -0,0 +1,20 @@ +diff -rup Beaker-1.9.1-orig/beaker/container.py Beaker-1.9.1/beaker/container.py +--- Beaker-1.9.1-orig/beaker/container.py 2018-04-10 10:23:04.000000000 +0200 ++++ Beaker-1.9.1/beaker/container.py 2018-04-10 10:23:34.000000000 +0200 +@@ -353,13 +353,13 @@ class Value(object): + debug("get_value returning old value while new one is created") + return value + else: +- debug("lock_creatfunc (didnt wait)") ++ debug("lock_creatfunc `%s` (didnt wait)", self.createfunc.__name__) + has_createlock = True + + if not has_createlock: +- debug("lock_createfunc (waiting)") ++ debug("lock_createfunc `%s` (waiting)", self.createfunc.__name__) + creation_lock.acquire() +- debug("lock_createfunc (waited)") ++ debug("lock_createfunc `%s` (waited)", self.createfunc.__name__) + + try: + # see if someone created the value already diff --git a/pkgs/python-packages-overrides.nix b/pkgs/python-packages-overrides.nix index ef1db0f3..a5581580 100644 --- a/pkgs/python-packages-overrides.nix +++ b/pkgs/python-packages-overrides.nix @@ -25,6 +25,12 @@ self: super: { }; }); + Beaker = super.Beaker.override (attrs: { + patches = [ + ./patch-beaker-lock-func-debug.diff + ]; + }); + future = super.future.override (attrs: { meta = { license = [ pkgs.lib.licenses.mit ]; @@ -82,7 +88,7 @@ self: super: { pkgs.openssl ]; propagatedBuildInputs = attrs.propagatedBuildInputs ++ [ - pkgs.mysql.lib + pkgs.libmysql pkgs.zlib ]; }); diff --git a/pkgs/python-packages.nix b/pkgs/python-packages.nix index 84d4802e..09edc90d 100644 --- a/pkgs/python-packages.nix +++ b/pkgs/python-packages.nix @@ -16,13 +16,13 @@ }; }; Beaker = super.buildPythonPackage { - name = "Beaker-1.9.0"; + name = "Beaker-1.9.1"; buildInputs = with self; []; doCheck = false; propagatedBuildInputs = with self; [funcsigs]; src = fetchurl { - url = "https://pypi.python.org/packages/93/b2/12de6937b06e9615dbb3cb3a1c9af17f133f435bdef59f4ad42032b6eb49/Beaker-1.9.0.tar.gz"; - md5 = "38b3fcdfa24faf97c6cf66991eb54e9c"; + url = "https://pypi.python.org/packages/ca/14/a626188d0d0c7b55dd7cf1902046c2743bd392a7078bb53073e13280eb1e/Beaker-1.9.1.tar.gz"; + md5 = "46fda0a164e2b0d24ccbda51a2310301"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; @@ -94,13 +94,13 @@ }; }; Markdown = super.buildPythonPackage { - name = "Markdown-2.6.9"; + name = "Markdown-2.6.11"; buildInputs = with self; []; doCheck = false; propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/29/82/dfe242bcfd9eec0e7bf93a80a8f8d8515a95b980c44f5c0b45606397a423/Markdown-2.6.9.tar.gz"; - md5 = "56547d362a9abcf30955b8950b08b5e3"; + url = "https://pypi.python.org/packages/b3/73/fc5c850f44af5889192dff783b7b0d8f3fe8d30b65c8e3f78f8f0265fecf/Markdown-2.6.11.tar.gz"; + md5 = "a67c1b2914f7d74eeede2ebe0fdae470"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; @@ -315,13 +315,13 @@ }; }; alembic = super.buildPythonPackage { - name = "alembic-0.9.6"; + name = "alembic-0.9.8"; buildInputs = with self; []; doCheck = false; propagatedBuildInputs = with self; [SQLAlchemy Mako python-editor python-dateutil]; src = fetchurl { - url = "https://pypi.python.org/packages/bf/b3/b28ea715824f8455635ece3c12f59d5d205f98cc378858e414e3aa6ebdbc/alembic-0.9.6.tar.gz"; - md5 = "fcb096bccc87c8770bd07a04606cb989"; + url = "https://pypi.python.org/packages/a1/95/2252783859df9ec76b9a25d968c2827ed75a43ba34c6e8d38f87a5c0fb26/alembic-0.9.8.tar.gz"; + md5 = "5cfef58641c9a94d4a5d547e951a7dda"; }; meta = { license = [ pkgs.lib.licenses.mit ]; @@ -341,13 +341,13 @@ }; }; appenlight-client = super.buildPythonPackage { - name = "appenlight-client-0.6.22"; + name = "appenlight-client-0.6.25"; buildInputs = with self; []; doCheck = false; propagatedBuildInputs = with self; [WebOb requests six]; src = fetchurl { - url = "https://pypi.python.org/packages/73/37/0a64460fa9670b17c061adc433bc8be5079cba21e8b3a92d824adccb12bc/appenlight_client-0.6.22.tar.gz"; - md5 = "641afc114a9a3b3af4f75b11c70968ee"; + url = "https://pypi.python.org/packages/fa/44/2911ef85ea4f4fe65058fd22959d8dad598fab6a3c84e5bcb569d15c8783/appenlight_client-0.6.25.tar.gz"; + md5 = "76dd2f9d42659fae8f290982078dc880"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; @@ -406,13 +406,13 @@ }; }; bleach = super.buildPythonPackage { - name = "bleach-2.1.1"; + name = "bleach-2.1.2"; buildInputs = with self; []; doCheck = false; propagatedBuildInputs = with self; [six html5lib]; src = fetchurl { - url = "https://pypi.python.org/packages/d4/3f/d517089af35b01bb9bc4eac5ea04bae342b37a5e9abbb27b7c3ce0eae070/bleach-2.1.1.tar.gz"; - md5 = "7c5dfb1d66ea979b5a465afb12c82ec4"; + url = "https://pypi.python.org/packages/b3/5f/0da670d30d3ffbc57cc97fa82947f81bbe3eab8d441e2d42e661f215baf2/bleach-2.1.2.tar.gz"; + md5 = "d0b14ae43a437ee0c650e04c6063eedd"; }; meta = { license = [ pkgs.lib.licenses.asl20 ]; @@ -710,8 +710,8 @@ doCheck = false; propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/5e/1a/0aa2c8195a204a9f51284018562dea77e25511f02fe924fac202fc012172/functools32-3.2.3-2.zip"; - md5 = "d55232eb132ec779e6893c902a0bc5ad"; + url = "https://pypi.python.org/packages/c5/60/6ac26ad05857c601308d8fb9e87fa36d0ebf889423f47c3502ef034365db/functools32-3.2.3-2.tar.gz"; + md5 = "09f24ffd9af9f6cd0f63cb9f4e23d4b2"; }; meta = { license = [ pkgs.lib.licenses.psfl ]; @@ -783,26 +783,26 @@ }; }; graphviz = super.buildPythonPackage { - name = "graphviz-0.8.1"; + name = "graphviz-0.8.2"; buildInputs = with self; []; doCheck = false; propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/a9/a6/ee6721349489a2da6eedd3dba124f2b5ac15ee1e0a7bd4d3cfdc4fff0327/graphviz-0.8.1.zip"; - md5 = "88d8efa88c02a735b3659fe0feaf0b96"; + url = "https://pypi.python.org/packages/fa/d1/63b62dee9e55368f60b5ea445e6afb361bb47e692fc27553f3672e16efb8/graphviz-0.8.2.zip"; + md5 = "50866e780f43e1cb0d073c70424fcaff"; }; meta = { license = [ pkgs.lib.licenses.mit ]; }; }; greenlet = super.buildPythonPackage { - name = "greenlet-0.4.12"; + name = "greenlet-0.4.13"; buildInputs = with self; []; doCheck = false; propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/be/76/82af375d98724054b7e273b5d9369346937324f9bcc20980b45b068ef0b0/greenlet-0.4.12.tar.gz"; - md5 = "e8637647d58a26c4a1f51ca393e53c00"; + url = "https://pypi.python.org/packages/13/de/ba92335e9e76040ca7274224942282a80d54f85e342a5e33c5277c7f87eb/greenlet-0.4.13.tar.gz"; + md5 = "6e0b9dd5385f81d478451ec8ed1d62b3"; }; meta = { license = [ pkgs.lib.licenses.mit ]; @@ -822,13 +822,13 @@ }; }; html5lib = super.buildPythonPackage { - name = "html5lib-1.0b10"; + name = "html5lib-1.0.1"; buildInputs = with self; []; doCheck = false; - propagatedBuildInputs = with self; [six webencodings setuptools]; + propagatedBuildInputs = with self; [six webencodings]; src = fetchurl { - url = "https://pypi.python.org/packages/97/16/982214624095c1420c75f3bd295d9e658794aafb95fc075823de107e0ae4/html5lib-1.0b10.tar.gz"; - md5 = "5ada1243b7a863624b2f35245b2186e9"; + url = "https://pypi.python.org/packages/85/3e/cf449cf1b5004e87510b9368e7a5f1acd8831c2d6691edd3c62a0823f98f/html5lib-1.0.1.tar.gz"; + md5 = "942a0688d6bdf20d087c9805c40182ad"; }; meta = { license = [ pkgs.lib.licenses.mit ]; @@ -874,13 +874,13 @@ }; }; ipaddress = super.buildPythonPackage { - name = "ipaddress-1.0.18"; + name = "ipaddress-1.0.19"; buildInputs = with self; []; doCheck = false; propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/4e/13/774faf38b445d0b3a844b65747175b2e0500164b7c28d78e34987a5bfe06/ipaddress-1.0.18.tar.gz"; - md5 = "310c2dfd64eb6f0df44aa8c59f2334a7"; + url = "https://pypi.python.org/packages/f0/ba/860a4a3e283456d6b7e2ab39ce5cf11a3490ee1a363652ac50abf9f0f5df/ipaddress-1.0.19.tar.gz"; + md5 = "d0687efaf93a32476d81e90ba0609c57"; }; meta = { license = [ pkgs.lib.licenses.psfl ]; @@ -1048,8 +1048,8 @@ doCheck = false; propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/15/45/30273ee91feb60dabb8fbb2da7868520525f02cf910279b3047182feed80/mock-1.0.1.zip"; - md5 = "869f08d003c289a97c1a6610faf5e913"; + url = "https://pypi.python.org/packages/a2/52/7edcd94f0afb721a2d559a5b9aae8af4f8f2c79bc63fdbe8a8a6c9b23bbe/mock-1.0.1.tar.gz"; + md5 = "c3971991738caa55ec7c356bbc154ee2"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; @@ -1160,13 +1160,13 @@ }; }; pexpect = super.buildPythonPackage { - name = "pexpect-4.3.0"; + name = "pexpect-4.4.0"; buildInputs = with self; []; doCheck = false; propagatedBuildInputs = with self; [ptyprocess]; src = fetchurl { - url = "https://pypi.python.org/packages/f8/44/5466c30e49762bb92e442bbdf4472d6904608d211258eb3198a11f0309a4/pexpect-4.3.0.tar.gz"; - md5 = "047a486dcd26134b74f2e67046bb61a0"; + url = "https://pypi.python.org/packages/fa/c3/60c0cbf96f242d0b47a82e9ca634dcd6dcb043832cf05e17540812e1c707/pexpect-4.4.0.tar.gz"; + md5 = "e9b07f0765df8245ac72201d757baaef"; }; meta = { license = [ pkgs.lib.licenses.isc { fullName = "ISC License (ISCL)"; } ]; @@ -1225,13 +1225,13 @@ }; }; psutil = super.buildPythonPackage { - name = "psutil-5.4.0"; + name = "psutil-5.4.3"; buildInputs = with self; []; doCheck = false; propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/8d/96/1fc6468be91521192861966c40bd73fdf8b065eae6d82dd0f870b9825a65/psutil-5.4.0.tar.gz"; - md5 = "01af6219b1e8fcfd53603023967713bf"; + url = "https://pypi.python.org/packages/e2/e1/600326635f97fee89bf8426fef14c5c29f4849c79f68fd79f433d8c1bd96/psutil-5.4.3.tar.gz"; + md5 = "3b291833dbea631db9d271aa602a169a"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; @@ -1360,8 +1360,8 @@ doCheck = false; propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/2e/26/e8fb5b4256a5f5036be7ce115ef8db8d06bc537becfbdc46c6af008314ee/pyparsing-1.5.7.zip"; - md5 = "b86854857a368d6ccb4d5b6e76d0637f"; + url = "https://pypi.python.org/packages/6f/2c/47457771c02a8ff0f302b695e094ec309e30452232bd79198ee94fda689f/pyparsing-1.5.7.tar.gz"; + md5 = "9be0fcdcc595199c646ab317c1d9a709"; }; meta = { license = [ pkgs.lib.licenses.mit ]; @@ -1602,13 +1602,13 @@ }; }; pytz = super.buildPythonPackage { - name = "pytz-2017.3"; + name = "pytz-2018.3"; buildInputs = with self; []; doCheck = false; propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/60/88/d3152c234da4b2a1f7a989f89609ea488225eaea015bc16fbde2b3fdfefa/pytz-2017.3.zip"; - md5 = "7006b56c0d68a162d9fe57d4249c3171"; + url = "https://pypi.python.org/packages/1b/50/4cdc62fc0753595fc16c8f722a89740f487c6e5670c644eb8983946777be/pytz-2018.3.tar.gz"; + md5 = "abb07c09c79f78d7c04f222a550c99ef"; }; meta = { license = [ pkgs.lib.licenses.mit ]; @@ -1680,10 +1680,10 @@ }; }; rhodecode-enterprise-ce = super.buildPythonPackage { - name = "rhodecode-enterprise-ce-4.11.6"; + name = "rhodecode-enterprise-ce-4.12.0"; buildInputs = with self; [pytest py pytest-cov pytest-sugar pytest-runner pytest-catchlog pytest-profiling gprof2dot pytest-timeout mock WebTest cov-core coverage configobj]; doCheck = true; - propagatedBuildInputs = with self; [setuptools-scm amqp authomatic Babel Beaker celery Chameleon channelstream click colander configobj cssselect decorator deform docutils dogpile.cache dogpile.core ecdsa FormEncode future futures gnureadline infrae.cache iso8601 itsdangerous Jinja2 billiard kombu lxml Mako Markdown MarkupSafe msgpack-python MySQL-python objgraph packaging Paste PasteDeploy PasteScript pathlib2 peppercorn psutil psycopg2 py-bcrypt pycrypto pycurl pyflakes pygments-markdown-lexer Pygments pyparsing pyramid-beaker pyramid-debugtoolbar pyramid-jinja2 pyramid-mako pyramid pysqlite python-dateutil python-ldap python-memcached python-pam pytz pyzmq py-gfm recaptcha-client redis repoze.lru requests Routes setproctitle simplejson six SQLAlchemy sshpubkeys subprocess32 supervisor Tempita translationstring trollius urllib3 URLObject venusian WebError WebHelpers2 WebHelpers WebOb Whoosh wsgiref zope.cachedescriptors zope.deprecation zope.event zope.interface nbconvert bleach nbformat jupyter-client alembic invoke bumpversion transifex-client gevent greenlet gunicorn waitress uWSGI ipdb ipython CProfileV bottle rhodecode-tools appenlight-client pytest py pytest-cov pytest-sugar pytest-runner pytest-catchlog pytest-profiling gprof2dot pytest-timeout mock WebTest cov-core coverage]; + propagatedBuildInputs = with self; [setuptools-scm amqp authomatic Babel Beaker celery Chameleon channelstream click colander configobj cssselect decorator deform docutils dogpile.cache dogpile.core ecdsa FormEncode future futures gnureadline infrae.cache iso8601 itsdangerous Jinja2 billiard kombu lxml Mako Markdown MarkupSafe msgpack-python MySQL-python objgraph packaging Paste PasteDeploy PasteScript pathlib2 peppercorn psutil psycopg2 py-bcrypt pycrypto pycurl pyflakes pygments-markdown-lexer Pygments pyparsing pyramid-beaker pyramid-debugtoolbar pyramid-jinja2 pyramid-mako pyramid pysqlite python-dateutil python-ldap python-memcached python-pam pytz tzlocal pyzmq py-gfm recaptcha-client redis repoze.lru requests Routes setproctitle simplejson six SQLAlchemy sshpubkeys subprocess32 supervisor Tempita translationstring trollius urllib3 URLObject venusian WebError WebHelpers2 WebHelpers WebOb Whoosh wsgiref zope.cachedescriptors zope.deprecation zope.event zope.interface nbconvert bleach nbformat jupyter-client alembic invoke bumpversion transifex-client gevent greenlet gunicorn waitress uWSGI ipdb ipython CProfileV bottle rhodecode-tools appenlight-client pytest py pytest-cov pytest-sugar pytest-runner pytest-catchlog pytest-profiling gprof2dot pytest-timeout mock WebTest cov-core coverage]; src = ./.; meta = { license = [ { fullName = "Affero GNU General Public License v3 or later (AGPLv3+)"; } { fullName = "AGPLv3, and Commercial License"; } ]; @@ -1703,13 +1703,13 @@ }; }; scandir = super.buildPythonPackage { - name = "scandir-1.6"; + name = "scandir-1.7"; buildInputs = with self; []; doCheck = false; propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/77/3f/916f524f50ee65e3f465a280d2851bd63685250fddb3020c212b3977664d/scandir-1.6.tar.gz"; - md5 = "0180ddb97c96cbb2d4f25d2ae11c64ac"; + url = "https://pypi.python.org/packages/13/bb/e541b74230bbf7a20a3949a2ee6631be299378a784f5445aa5d0047c192b/scandir-1.7.tar.gz"; + md5 = "037e5f24d1a0e78b17faca72dea9555f"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal { fullName = "New BSD License"; } ]; @@ -1820,13 +1820,13 @@ }; }; supervisor = super.buildPythonPackage { - name = "supervisor-3.3.3"; + name = "supervisor-3.3.4"; buildInputs = with self; []; doCheck = false; propagatedBuildInputs = with self; [meld3]; src = fetchurl { - url = "https://pypi.python.org/packages/31/7e/788fc6566211e77c395ea272058eb71299c65cc5e55b6214d479c6c2ec9a/supervisor-3.3.3.tar.gz"; - md5 = "0fe86dfec4e5c5d98324d24c4cf944bd"; + url = "https://pypi.python.org/packages/44/60/698e54b4a4a9b956b2d709b4b7b676119c833d811d53ee2500f1b5e96dc3/supervisor-3.3.4.tar.gz"; + md5 = "f1814d71d820ddfa8c86d46a72314cec"; }; meta = { license = [ { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ]; @@ -1910,6 +1910,19 @@ license = [ pkgs.lib.licenses.asl20 ]; }; }; + tzlocal = super.buildPythonPackage { + name = "tzlocal-1.5.1"; + buildInputs = with self; []; + doCheck = false; + propagatedBuildInputs = with self; [pytz]; + src = fetchurl { + url = "https://pypi.python.org/packages/cb/89/e3687d3ed99bc882793f82634e9824e62499fdfdc4b1ae39e211c5b05017/tzlocal-1.5.1.tar.gz"; + md5 = "4553be891efa0812c4adfb0c6e818eec"; + }; + meta = { + license = [ pkgs.lib.licenses.mit ]; + }; + }; uWSGI = super.buildPythonPackage { name = "uWSGI-2.0.15"; buildInputs = with self; []; @@ -2002,13 +2015,13 @@ }; }; ws4py = super.buildPythonPackage { - name = "ws4py-0.4.2"; + name = "ws4py-0.4.3"; buildInputs = with self; []; doCheck = false; propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/b8/98/a90f1d96ffcb15dfc220af524ce23e0a5881258dafa197673357ce1683dd/ws4py-0.4.2.tar.gz"; - md5 = "f0603ae376707a58d205bd87a67758a2"; + url = "https://pypi.python.org/packages/fa/a1/33c43a4304ac3b4dc81deb93cbd329de9297dd034d75c47cce64fda806bc/ws4py-0.4.3.tar.gz"; + md5 = "d5834cf7d3965bb0da31bbb02bd8513a"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; @@ -2028,52 +2041,52 @@ }; }; zope.cachedescriptors = super.buildPythonPackage { - name = "zope.cachedescriptors-4.0.0"; + name = "zope.cachedescriptors-4.3.1"; buildInputs = with self; []; doCheck = false; propagatedBuildInputs = with self; [setuptools]; src = fetchurl { - url = "https://pypi.python.org/packages/40/33/694b6644c37f28553f4b9f20b3c3a20fb709a22574dff20b5bdffb09ecd5/zope.cachedescriptors-4.0.0.tar.gz"; - md5 = "8d308de8c936792c8e758058fcb7d0f0"; + url = "https://pypi.python.org/packages/2f/89/ebe1890cc6d3291ebc935558fa764d5fffe571018dbbee200e9db78762cb/zope.cachedescriptors-4.3.1.tar.gz"; + md5 = "42f3693f43bc93f3b1eb86940f58acf3"; }; meta = { license = [ pkgs.lib.licenses.zpt21 ]; }; }; zope.deprecation = super.buildPythonPackage { - name = "zope.deprecation-4.1.2"; + name = "zope.deprecation-4.3.0"; buildInputs = with self; []; doCheck = false; propagatedBuildInputs = with self; [setuptools]; src = fetchurl { - url = "https://pypi.python.org/packages/c1/d3/3919492d5e57d8dd01b36f30b34fc8404a30577392b1eb817c303499ad20/zope.deprecation-4.1.2.tar.gz"; - md5 = "e9a663ded58f4f9f7881beb56cae2782"; + url = "https://pypi.python.org/packages/a1/18/2dc5e6bfe64fdc3b79411b67464c55bb0b43b127051a20f7f492ab767758/zope.deprecation-4.3.0.tar.gz"; + md5 = "2166b2cb7e0e96a21104e6f8f9b696bb"; }; meta = { license = [ pkgs.lib.licenses.zpt21 ]; }; }; zope.event = super.buildPythonPackage { - name = "zope.event-4.0.3"; + name = "zope.event-4.3.0"; buildInputs = with self; []; doCheck = false; propagatedBuildInputs = with self; [setuptools]; src = fetchurl { - url = "https://pypi.python.org/packages/c1/29/91ba884d7d6d96691df592e9e9c2bfa57a47040ec1ff47eff18c85137152/zope.event-4.0.3.tar.gz"; - md5 = "9a3780916332b18b8b85f522bcc3e249"; + url = "https://pypi.python.org/packages/9e/d0/54ba59f19a0635f6591b74be259cf6fbf67e73f4edda27b5cd0cf4d26efa/zope.event-4.3.0.tar.gz"; + md5 = "8ca737960741c6fd112972f3313303bd"; }; meta = { license = [ pkgs.lib.licenses.zpt21 ]; }; }; zope.interface = super.buildPythonPackage { - name = "zope.interface-4.1.3"; + name = "zope.interface-4.4.3"; buildInputs = with self; []; doCheck = false; propagatedBuildInputs = with self; [setuptools]; src = fetchurl { - url = "https://pypi.python.org/packages/9d/81/2509ca3c6f59080123c1a8a97125eb48414022618cec0e64eb1313727bfe/zope.interface-4.1.3.tar.gz"; - md5 = "9ae3d24c0c7415deb249dd1a132f0f79"; + url = "https://pypi.python.org/packages/bd/d2/25349ed41f9dcff7b3baf87bd88a4c82396cf6e02f1f42bb68657a3132af/zope.interface-4.4.3.tar.gz"; + md5 = "8700a4f527c1203b34b10c2b4e7a6912"; }; meta = { license = [ pkgs.lib.licenses.zpt21 ]; diff --git a/requirements.txt b/requirements.txt index 742bab56..f380e4f5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ setuptools-scm==1.15.6 amqp==2.2.2 authomatic==0.1.0.post1 Babel==1.3 -Beaker==1.9.0 +Beaker==1.9.1 celery==4.1.0 Chameleon==2.24 channelstream==0.5.2 @@ -31,7 +31,7 @@ billiard==3.5.0.3 kombu==4.1.0 lxml==3.7.3 Mako==1.0.7 -Markdown==2.6.9 +Markdown==2.6.11 MarkupSafe==1.0.0 msgpack-python==0.4.8 MySQL-python==1.2.5 @@ -42,7 +42,7 @@ PasteDeploy==1.5.2 PasteScript==2.0.2 pathlib2==2.3.0 peppercorn==0.5 -psutil==5.4.0 +psutil==5.4.3 psycopg2==2.7.3.2 py-bcrypt==0.4 pycrypto==2.6.1 @@ -61,7 +61,8 @@ python-dateutil python-ldap==2.4.45 python-memcached==1.58 python-pam==1.8.2 -pytz==2017.3 +pytz==2018.3 +tzlocal==1.5.1 pyzmq==14.6.0 py-gfm==0.1.3 recaptcha-client==1.0.6 @@ -75,7 +76,7 @@ six==1.11.0 SQLAlchemy==1.1.15 sshpubkeys==2.2.0 subprocess32==3.2.7 -supervisor==3.3.3 +supervisor==3.3.4 Tempita==0.5.2 translationstring==1.3 trollius==1.0.4 @@ -88,29 +89,29 @@ WebHelpers==1.3 WebOb==1.7.4 Whoosh==2.7.4 wsgiref==0.1.2 -zope.cachedescriptors==4.0.0 -zope.deprecation==4.1.2 -zope.event==4.0.3 -zope.interface==4.1.3 +zope.cachedescriptors==4.3.1 +zope.deprecation==4.3.0 +zope.event==4.3.0 +zope.interface==4.4.3 # IPYTHON RENDERING # entrypoints backport, pypi version doesn't support egg installs https://code.rhodecode.com/upstream/entrypoints/archive/96e6d645684e1af3d7df5b5272f3fe85a546b233.tar.gz?md5=7db37771aea9ac9fefe093e5d6987313#egg=entrypoints==0.2.2.rhodecode-upstream1 nbconvert==5.3.1 -bleach==2.1.1 +bleach==2.1.2 nbformat==4.4.0 jupyter_client==5.0.0 ## cli tools -alembic==0.9.6 +alembic==0.9.8 invoke==0.13.0 bumpversion==0.5.3 transifex-client==0.12.5 ## http servers gevent==1.2.2 -greenlet==0.4.12 +greenlet==0.4.13 gunicorn==19.7.1 waitress==1.1.0 uWSGI==2.0.15 @@ -125,7 +126,7 @@ bottle==0.12.13 https://code.rhodecode.com/rhodecode-tools-ce/archive/v0.14.1.tar.gz?md5=0b9c2caad160b68889f8172ea54af7b2#egg=rhodecode-tools==0.14.1 ## appenlight -appenlight-client==0.6.22 +appenlight-client==0.6.25 ## test related requirements -r requirements_test.txt diff --git a/rhodecode/VERSION b/rhodecode/VERSION index 345711f8..bcd250ed 100644 --- a/rhodecode/VERSION +++ b/rhodecode/VERSION @@ -1 +1 @@ -4.11.6 \ No newline at end of file +4.12.0 \ No newline at end of file diff --git a/rhodecode/__init__.py b/rhodecode/__init__.py index bd65114c..34dd7c91 100644 --- a/rhodecode/__init__.py +++ b/rhodecode/__init__.py @@ -51,7 +51,7 @@ PYRAMID_SETTINGS = {} EXTENSIONS = {} __version__ = ('.'.join((str(each) for each in VERSION[:3]))) -__dbversion__ = 85 # defines current db version for migrations +__dbversion__ = 86 # defines current db version for migrations __platform__ = platform.system() __license__ = 'AGPLv3, and Commercial License' __author__ = 'RhodeCode GmbH' diff --git a/rhodecode/api/tests/test_get_repo_changeset.py b/rhodecode/api/tests/test_get_repo_changeset.py index b6737998..cc38a28a 100644 --- a/rhodecode/api/tests/test_get_repo_changeset.py +++ b/rhodecode/api/tests/test_get_repo_changeset.py @@ -42,7 +42,8 @@ class TestGetRepoChangeset(object): if details == 'full': assert result['refs']['bookmarks'] == getattr( commit, 'bookmarks', []) - assert result['refs']['branches'] == [commit.branch] + branches = [commit.branch] if commit.branch else [] + assert result['refs']['branches'] == branches assert result['refs']['tags'] == commit.tags @pytest.mark.parametrize("details", ['basic', 'extended', 'full']) diff --git a/rhodecode/api/tests/test_pull.py b/rhodecode/api/tests/test_pull.py index 1c871a81..48bc902f 100644 --- a/rhodecode/api/tests/test_pull.py +++ b/rhodecode/api/tests/test_pull.py @@ -33,12 +33,14 @@ class TestPull(object): def test_api_pull(self, backend): r = backend.create_repo() repo_name = r.repo_name - r.clone_uri = os.path.join(TESTS_TMP_PATH, backend.repo_name) + clone_uri = os.path.join(TESTS_TMP_PATH, backend.repo_name) + r.clone_uri = clone_uri id_, params = build_data(self.apikey, 'pull', repoid=repo_name,) response = api_call(self.app, params) - - expected = {'msg': 'Pulled from `%s`' % (repo_name,), + msg = 'Pulled from url `%s` on repo `%s`' % ( + clone_uri, repo_name) + expected = {'msg': msg, 'repository': repo_name} assert_ok(id_, expected, given=response.body) @@ -47,5 +49,5 @@ class TestPull(object): self.apikey, 'pull', repoid=backend.repo_name) response = api_call(self.app, params) - expected = 'Unable to pull changes from `%s`' % (backend.repo_name,) + expected = 'Unable to pull changes from `None`' assert_error(id_, expected, given=response.body) diff --git a/rhodecode/api/tests/test_update_repo.py b/rhodecode/api/tests/test_update_repo.py index e14b675c..52a192d5 100644 --- a/rhodecode/api/tests/test_update_repo.py +++ b/rhodecode/api/tests/test_update_repo.py @@ -56,6 +56,9 @@ class TestApiUpdateRepo(object): ({'clone_uri': ''}, {'clone_uri': ''}), + ({'push_uri': ''}, + {'push_uri': ''}), + ({'landing_rev': 'rev:tip'}, {'landing_rev': ['rev', 'tip']}), diff --git a/rhodecode/api/tests/test_update_user_group.py b/rhodecode/api/tests/test_update_user_group.py index 841c2ced..d51d6c1b 100644 --- a/rhodecode/api/tests/test_update_user_group.py +++ b/rhodecode/api/tests/test_update_user_group.py @@ -36,7 +36,9 @@ class TestUpdateUserGroup(object): # ('owner', {'owner': TEST_USER_REGULAR_LOGIN}), ('owner_email', {'owner_email': TEST_USER_ADMIN_EMAIL}), ('active', {'active': False}), - ('active', {'active': True}) + ('active', {'active': True}), + ('sync', {'sync': False}), + ('sync', {'sync': True}) ]) def test_api_update_user_group(self, changing_attr, updates, user_util): user_group = user_util.create_user_group() @@ -49,6 +51,12 @@ class TestUpdateUserGroup(object): **updates) response = api_call(self.app, params) + # special case for sync + if changing_attr == 'sync' and updates['sync'] is False: + expected_api_data['sync'] = None + elif changing_attr == 'sync' and updates['sync'] is True: + expected_api_data['sync'] = 'manual_api' + expected = { 'msg': 'updated user group ID:%s %s' % ( user_group.users_group_id, user_group.users_group_name), @@ -63,7 +71,9 @@ class TestUpdateUserGroup(object): # ('owner', {'owner': TEST_USER_REGULAR_LOGIN}), ('owner_email', {'owner_email': TEST_USER_ADMIN_EMAIL}), ('active', {'active': False}), - ('active', {'active': True}) + ('active', {'active': True}), + ('sync', {'sync': False}), + ('sync', {'sync': True}) ]) def test_api_update_user_group_regular_user( self, changing_attr, updates, user_util): @@ -72,7 +82,6 @@ class TestUpdateUserGroup(object): expected_api_data = user_group.get_api_data() expected_api_data.update(updates) - # grant permission to this user user = UserModel().get_by_username(self.TEST_USER_LOGIN) @@ -82,6 +91,12 @@ class TestUpdateUserGroup(object): self.apikey_regular, 'update_user_group', usergroupid=group_name, **updates) response = api_call(self.app, params) + # special case for sync + if changing_attr == 'sync' and updates['sync'] is False: + expected_api_data['sync'] = None + elif changing_attr == 'sync' and updates['sync'] is True: + expected_api_data['sync'] = 'manual_api' + expected = { 'msg': 'updated user group ID:%s %s' % ( user_group.users_group_id, user_group.users_group_name), diff --git a/rhodecode/api/views/repo_api.py b/rhodecode/api/views/repo_api.py index 2f2cdaca..6bf2941b 100644 --- a/rhodecode/api/views/repo_api.py +++ b/rhodecode/api/views/repo_api.py @@ -563,6 +563,7 @@ def create_repo( description=Optional(''), private=Optional(False), clone_uri=Optional(None), + push_uri=Optional(None), landing_rev=Optional('rev:tip'), enable_statistics=Optional(False), enable_locking=Optional(False), @@ -596,6 +597,8 @@ def create_repo( :type private: bool :param clone_uri: set clone_uri :type clone_uri: str + :param push_uri: set push_uri + :type push_uri: str :param landing_rev: : :type landing_rev: str :param enable_locking: @@ -639,6 +642,7 @@ def create_repo( description = Optional.extract(description) copy_permissions = Optional.extract(copy_permissions) clone_uri = Optional.extract(clone_uri) + push_uri = Optional.extract(push_uri) landing_commit_ref = Optional.extract(landing_rev) defs = SettingsModel().get_default_repo_settings(strip_prefix=True) @@ -667,6 +671,7 @@ def create_repo( repo_description=description, repo_landing_commit_ref=landing_commit_ref, repo_clone_uri=clone_uri, + repo_push_uri=push_uri, repo_private=private, repo_copy_permissions=copy_permissions, repo_enable_statistics=enable_statistics, @@ -685,6 +690,7 @@ def create_repo( 'repo_description': schema_data['repo_description'], 'repo_private': schema_data['repo_private'], 'clone_uri': schema_data['repo_clone_uri'], + 'push_uri': schema_data['repo_push_uri'], 'repo_landing_rev': schema_data['repo_landing_commit_ref'], 'enable_statistics': schema_data['repo_enable_statistics'], 'enable_locking': schema_data['repo_enable_locking'], @@ -692,7 +698,7 @@ def create_repo( 'repo_copy_permissions': schema_data['repo_copy_permissions'], } - task = RepoModel().create(form_data=data, cur_user=owner) + task = RepoModel().create(form_data=data, cur_user=owner.user_id) task_id = get_task_id(task) # no commit, it's done in RepoModel, or async via celery return { @@ -800,7 +806,8 @@ def remove_field_from_repo(request, apiuser, repoid, key): def update_repo( request, apiuser, repoid, repo_name=Optional(None), owner=Optional(OAttr('apiuser')), description=Optional(''), - private=Optional(False), clone_uri=Optional(None), + private=Optional(False), + clone_uri=Optional(None), push_uri=Optional(None), landing_rev=Optional('rev:tip'), fork_of=Optional(None), enable_statistics=Optional(False), enable_locking=Optional(False), @@ -877,6 +884,9 @@ def update_repo( clone_uri=clone_uri if not isinstance(clone_uri, Optional) else repo.clone_uri, + push_uri=push_uri + if not isinstance(push_uri, Optional) else repo.push_uri, + repo_landing_rev=landing_rev if not isinstance(landing_rev, Optional) else repo._landing_revision, @@ -910,6 +920,7 @@ def update_repo( repo_owner=updates['user'], repo_description=updates['repo_description'], repo_clone_uri=updates['clone_uri'], + repo_push_uri=updates['push_uri'], repo_fork_of=updates['fork_id'], repo_private=updates['repo_private'], repo_landing_commit_ref=updates['repo_landing_rev'], @@ -928,6 +939,7 @@ def update_repo( repo_description=schema_data['repo_description'], repo_private=schema_data['repo_private'], clone_uri=schema_data['repo_clone_uri'], + push_uri=schema_data['repo_push_uri'], repo_landing_rev=schema_data['repo_landing_commit_ref'], repo_enable_statistics=schema_data['repo_enable_statistics'], repo_enable_locking=schema_data['repo_enable_locking'], @@ -1084,7 +1096,7 @@ def fork_repo(request, apiuser, repoid, fork_name, 'landing_rev': schema_data['repo_landing_commit_ref'], } - task = RepoModel().create_fork(data, cur_user=owner) + task = RepoModel().create_fork(data, cur_user=owner.user_id) # no commit, it's done in RepoModel, or async via celery task_id = get_task_id(task) @@ -1749,7 +1761,7 @@ def revoke_user_group_permission(request, apiuser, repoid, usergroupid): @jsonrpc_method() -def pull(request, apiuser, repoid): +def pull(request, apiuser, repoid, remote_uri=Optional(None)): """ Triggers a pull on the given repository from a remote location. You can use this to keep remote repositories up-to-date. @@ -1764,6 +1776,8 @@ def pull(request, apiuser, repoid): :type apiuser: AuthUser :param repoid: The repository name or repository ID. :type repoid: str or int + :param remote_uri: Optional remote URI to pass in for pull + :type remote_uri: str Example output: @@ -1771,7 +1785,7 @@ def pull(request, apiuser, repoid): id : result : { - "msg": "Pulled from ``" + "msg": "Pulled from url `` on repo ``" "repository": "" } error : null @@ -1783,27 +1797,31 @@ def pull(request, apiuser, repoid): id : result : null error : { - "Unable to pull changes from ``" + "Unable to push changes from ``" } """ repo = get_repo_or_error(repoid) + remote_uri = Optional.extract(remote_uri) + remote_uri_display = remote_uri or repo.clone_uri_hidden if not has_superadmin_permission(apiuser): _perms = ('repository.admin',) validate_repo_permissions(apiuser, repoid, repo, _perms) try: - ScmModel().pull_changes(repo.repo_name, apiuser.username) + ScmModel().pull_changes( + repo.repo_name, apiuser.username, remote_uri=remote_uri) return { - 'msg': 'Pulled from `%s`' % repo.repo_name, + 'msg': 'Pulled from url `%s` on repo `%s`' % ( + remote_uri_display, repo.repo_name), 'repository': repo.repo_name } except Exception: log.exception("Exception occurred while trying to " "pull changes from remote location") raise JSONRPCError( - 'Unable to pull changes from `%s`' % repo.repo_name + 'Unable to pull changes from `%s`' % remote_uri_display ) diff --git a/rhodecode/api/views/user_group_api.py b/rhodecode/api/views/user_group_api.py index 456db118..49f67277 100644 --- a/rhodecode/api/views/user_group_api.py +++ b/rhodecode/api/views/user_group_api.py @@ -168,7 +168,8 @@ def get_user_groups(request, apiuser): @jsonrpc_method() def create_user_group( request, apiuser, group_name, description=Optional(''), - owner=Optional(OAttr('apiuser')), active=Optional(True)): + owner=Optional(OAttr('apiuser')), active=Optional(True), + sync=Optional(None)): """ Creates a new user group. @@ -188,6 +189,9 @@ def create_user_group( :type owner: Optional(str or int) :param active: Set this group as active. :type active: Optional(``True`` | ``False``) + :param sync: Set enabled or disabled the automatically sync from + external authentication types like ldap. + :type sync: Optional(``True`` | ``False``) Example output: @@ -227,6 +231,15 @@ def create_user_group( owner = get_user_or_error(owner) active = Optional.extract(active) description = Optional.extract(description) + sync = Optional.extract(sync) + + # set the sync option based on group_data + group_data = None + if sync: + group_data = { + 'extern_type': 'manual_api', + 'extern_type_set_by': apiuser.username + } schema = user_group_schema.UserGroupSchema().bind( # user caller @@ -246,7 +259,7 @@ def create_user_group( name=schema_data['user_group_name'], description=schema_data['user_group_description'], owner=owner, - active=schema_data['user_group_active']) + active=schema_data['user_group_active'], group_data=group_data) Session().flush() creation_data = user_group.get_api_data() audit_logger.store_api( @@ -265,7 +278,7 @@ def create_user_group( @jsonrpc_method() def update_user_group(request, apiuser, usergroupid, group_name=Optional(''), description=Optional(''), owner=Optional(None), - active=Optional(True)): + active=Optional(True), sync=Optional(None)): """ Updates the specified `user group` with the details provided. @@ -284,6 +297,9 @@ def update_user_group(request, apiuser, usergroupid, group_name=Optional(''), :type owner: Optional(str or int) :param active: Set the group as active. :type active: Optional(``True`` | ``False``) + :param sync: Set enabled or disabled the automatically sync from + external authentication types like ldap. + :type sync: Optional(``True`` | ``False``) Example output: @@ -329,8 +345,21 @@ def update_user_group(request, apiuser, usergroupid, group_name=Optional(''), store_update(updates, description, 'user_group_description') store_update(updates, owner, 'user') store_update(updates, active, 'users_group_active') + + sync = Optional.extract(sync) + group_data = None + if sync is True: + group_data = { + 'extern_type': 'manual_api', + 'extern_type_set_by': apiuser.username + } + if sync is False: + group_data = user_group.group_data + if group_data and "extern_type" in group_data: + del group_data["extern_type"] + try: - UserGroupModel().update(user_group, updates) + UserGroupModel().update(user_group, updates, group_data=group_data) audit_logger.store_api( 'user_group.edit', action_data={'old_data': old_data}, user=apiuser) @@ -609,8 +638,18 @@ def grant_user_permission_to_user_group( perm = get_perm_or_error(perm, prefix='usergroup.') try: - UserGroupModel().grant_user_permission( + changes = UserGroupModel().grant_user_permission( user_group=user_group, user=user, perm=perm) + + action_data = { + 'added': changes['added'], + 'updated': changes['updated'], + 'deleted': changes['deleted'], + } + audit_logger.store_api( + 'user_group.edit.permissions', action_data=action_data, + user=apiuser) + Session().commit() return { 'msg': @@ -669,8 +708,17 @@ def revoke_user_permission_from_user_group( user = get_user_or_error(userid) try: - UserGroupModel().revoke_user_permission( + changes = UserGroupModel().revoke_user_permission( user_group=user_group, user=user) + action_data = { + 'added': changes['added'], + 'updated': changes['updated'], + 'deleted': changes['deleted'], + } + audit_logger.store_api( + 'user_group.edit.permissions', action_data=action_data, + user=apiuser) + Session().commit() return { 'msg': 'Revoked perm for user: `%s` in user group: `%s`' % ( @@ -735,11 +783,20 @@ def grant_user_group_permission_to_user_group( 'user group `%s` does not exist' % (sourceusergroupid,)) try: - UserGroupModel().grant_user_group_permission( + changes = UserGroupModel().grant_user_group_permission( target_user_group=target_user_group, user_group=user_group, perm=perm) - Session().commit() + action_data = { + 'added': changes['added'], + 'updated': changes['updated'], + 'deleted': changes['deleted'], + } + audit_logger.store_api( + 'user_group.edit.permissions', action_data=action_data, + user=apiuser) + + Session().commit() return { 'msg': 'Granted perm: `%s` for user group: `%s` ' 'in user group: `%s`' % ( @@ -806,8 +863,17 @@ def revoke_user_group_permission_from_user_group( 'user group `%s` does not exist' % (sourceusergroupid,)) try: - UserGroupModel().revoke_user_group_permission( + changes = UserGroupModel().revoke_user_group_permission( target_user_group=target_user_group, user_group=user_group) + action_data = { + 'added': changes['added'], + 'updated': changes['updated'], + 'deleted': changes['deleted'], + } + audit_logger.store_api( + 'user_group.edit.permissions', action_data=action_data, + user=apiuser) + Session().commit() return { diff --git a/rhodecode/apps/_base/__init__.py b/rhodecode/apps/_base/__init__.py index fbc6bf60..d295cf1c 100644 --- a/rhodecode/apps/_base/__init__.py +++ b/rhodecode/apps/_base/__init__.py @@ -22,9 +22,9 @@ import time import logging import operator -from pyramid.httpexceptions import HTTPFound +from pyramid.httpexceptions import HTTPFound, HTTPForbidden -from rhodecode.lib import helpers as h +from rhodecode.lib import helpers as h, diffs from rhodecode.lib.utils2 import StrictAttributeDict, safe_int, datetime_to_time from rhodecode.lib.vcs.exceptions import RepositoryRequirementError from rhodecode.model import repo @@ -33,6 +33,7 @@ from rhodecode.model import user_group from rhodecode.model import user from rhodecode.model.db import User from rhodecode.model.scm import ScmModel +from rhodecode.model.settings import VcsSettingsModel log = logging.getLogger(__name__) @@ -204,28 +205,47 @@ class RepoAppView(BaseAppView): c.rhodecode_db_repo = self.db_repo c.repo_name = self.db_repo_name c.repository_pull_requests = self.db_repo_pull_requests + self.path_filter = PathFilter(None) - c.repository_requirements_missing = False + c.repository_requirements_missing = {} try: self.rhodecode_vcs_repo = self.db_repo.scm_instance() + if self.rhodecode_vcs_repo: + path_perms = self.rhodecode_vcs_repo.get_path_permissions( + c.auth_user.username) + self.path_filter = PathFilter(path_perms) except RepositoryRequirementError as e: - c.repository_requirements_missing = True + c.repository_requirements_missing = {'error': str(e)} self._handle_missing_requirements(e) self.rhodecode_vcs_repo = None - if (not c.repository_requirements_missing - and self.rhodecode_vcs_repo is None): + c.path_filter = self.path_filter # used by atom_feed_entry.mako + + if self.rhodecode_vcs_repo is None: # unable to fetch this repo as vcs instance, report back to user h.flash(_( "The repository `%(repo_name)s` cannot be loaded in filesystem. " "Please check if it exist, or is not damaged.") % {'repo_name': c.repo_name}, category='error', ignore_duplicate=True) - raise HTTPFound(h.route_path('home')) + if c.repository_requirements_missing: + route = self.request.matched_route.name + if route.startswith(('edit_repo', 'repo_summary')): + # allow summary and edit repo on missing requirements + return c + + raise HTTPFound( + h.route_path('repo_summary', repo_name=self.db_repo_name)) + + else: # redirect if we don't show missing requirements + raise HTTPFound(h.route_path('home')) return c - def _get_f_path(self, matchdict, default=None): + def _get_f_path_unchecked(self, matchdict, default=None): + """ + Should only be used by redirects, everything else should call _get_f_path + """ f_path = matchdict.get('f_path') if f_path: # fix for multiple initial slashes that causes errors for GIT @@ -233,6 +253,63 @@ class RepoAppView(BaseAppView): return default + def _get_f_path(self, matchdict, default=None): + f_path_match = self._get_f_path_unchecked(matchdict, default) + return self.path_filter.assert_path_permissions(f_path_match) + + def _get_general_setting(self, target_repo, settings_key, default=False): + settings_model = VcsSettingsModel(repo=target_repo) + settings = settings_model.get_general_settings() + return settings.get(settings_key, default) + + +class PathFilter(object): + + # Expects and instance of BasePathPermissionChecker or None + def __init__(self, permission_checker): + self.permission_checker = permission_checker + + def assert_path_permissions(self, path): + if path and self.permission_checker and not self.permission_checker.has_access(path): + raise HTTPForbidden() + return path + + def filter_patchset(self, patchset): + if not self.permission_checker or not patchset: + return patchset, False + had_filtered = False + filtered_patchset = [] + for patch in patchset: + filename = patch.get('filename', None) + if not filename or self.permission_checker.has_access(filename): + filtered_patchset.append(patch) + else: + had_filtered = True + if had_filtered: + if isinstance(patchset, diffs.LimitedDiffContainer): + filtered_patchset = diffs.LimitedDiffContainer(patchset.diff_limit, patchset.cur_diff_size, filtered_patchset) + return filtered_patchset, True + else: + return patchset, False + + def render_patchset_filtered(self, diffset, patchset, source_ref=None, target_ref=None): + filtered_patchset, has_hidden_changes = self.filter_patchset(patchset) + result = diffset.render_patchset(filtered_patchset, source_ref=source_ref, target_ref=target_ref) + result.has_hidden_changes = has_hidden_changes + return result + + def get_raw_patch(self, diff_processor): + if self.permission_checker is None: + return diff_processor.as_raw() + elif self.permission_checker.has_full_access: + return diff_processor.as_raw() + else: + return '# Repository has user-specific filters, raw patch generation is disabled.' + + @property + def is_enabled(self): + return self.permission_checker is not None + class RepoGroupAppView(BaseAppView): def __init__(self, context, request): diff --git a/rhodecode/apps/admin/__init__.py b/rhodecode/apps/admin/__init__.py index 5a87da84..5fdea03c 100644 --- a/rhodecode/apps/admin/__init__.py +++ b/rhodecode/apps/admin/__init__.py @@ -169,6 +169,11 @@ def admin_routes(config): name='admin_settings_labs_update', pattern='/settings/labs/update') + # Automation EE feature + config.add_route( + 'admin_settings_automation', + pattern=ADMIN_PREFIX + '/settings/automation') + # global permissions config.add_route( diff --git a/rhodecode/apps/admin/navigation.py b/rhodecode/apps/admin/navigation.py index 8932f65b..3ec8e2fc 100644 --- a/rhodecode/apps/admin/navigation.py +++ b/rhodecode/apps/admin/navigation.py @@ -95,7 +95,8 @@ class NavigationRegistry(object): 'admin_settings_sessions'), NavEntry('open_source', _('Open Source Licenses'), 'admin_settings_open_source'), - + NavEntry('automation', _('Automation'), + 'admin_settings_automation') ] _labs_entry = NavEntry('labs', _('Labs'), diff --git a/rhodecode/apps/admin/tests/test_admin_auth_settings.py b/rhodecode/apps/admin/tests/test_admin_auth_settings.py index d8d5777b..09a3e513 100644 --- a/rhodecode/apps/admin/tests/test_admin_auth_settings.py +++ b/rhodecode/apps/admin/tests/test_admin_auth_settings.py @@ -86,6 +86,7 @@ class TestAuthSettingsView(object): 'host': 'dc.example.com', 'port': '999', + 'timeout': 3600, 'tls_kind': 'PLAIN', 'tls_reqcert': 'NEVER', diff --git a/rhodecode/apps/admin/tests/test_admin_users_ssh_keys.py b/rhodecode/apps/admin/tests/test_admin_users_ssh_keys.py index 7cc8dcbc..1ca4ad6b 100644 --- a/rhodecode/apps/admin/tests/test_admin_users_ssh_keys.py +++ b/rhodecode/apps/admin/tests/test_admin_users_ssh_keys.py @@ -67,6 +67,7 @@ class TestAdminUsersSshKeysView(TestController): 'I4fG8+hBHzpeFxUGvSGNtXPUbwaAY8j/oHYrTpMgkj6pUEFsiKfC5zPq' \ 'PFR5HyKTCHW0nFUJnZsbyFT5hMiF/hZkJc9A0ZbdSvJwCRQ/g3bmdL ' \ 'your_email@example.com' + FINGERPRINT = 'MD5:01:4f:ad:29:22:6e:01:37:c9:d2:52:26:52:b0:2d:93' def test_ssh_keys_default_user(self): self.log_user() @@ -111,9 +112,11 @@ class TestAdminUsersSshKeysView(TestController): route_path('edit_user_ssh_keys_add', user_id=user_id), {'description': desc, 'key_data': key_data, 'csrf_token': self.csrf_token}) + + err = 'Such key with fingerprint `{}` already exists, ' \ + 'please use a different one'.format(self.FINGERPRINT) assert_session_flash(response, 'An error occurred during ssh key ' - 'saving: Such key already exists, ' - 'please use a different one') + 'saving: {}'.format(err)) def test_add_ssh_key(self, user_util): self.log_user() diff --git a/rhodecode/apps/admin/views/process_management.py b/rhodecode/apps/admin/views/process_management.py index 5cd2f496..99eee6f2 100644 --- a/rhodecode/apps/admin/views/process_management.py +++ b/rhodecode/apps/admin/views/process_management.py @@ -28,7 +28,7 @@ from rhodecode.apps._base import BaseAppView from rhodecode.apps.admin.navigation import navigation_list from rhodecode.lib.auth import ( LoginRequired, HasPermissionAllDecorator, CSRFRequired) -from rhodecode.lib.utils2 import safe_int +from rhodecode.lib.utils2 import safe_int, StrictAttributeDict log = logging.getLogger(__name__) @@ -36,9 +36,41 @@ log = logging.getLogger(__name__) class AdminProcessManagementView(BaseAppView): def load_default_context(self): c = self._get_local_tmpl_context() - return c + def _format_proc(self, proc, with_children=False): + try: + mem = proc.memory_info() + proc_formatted = StrictAttributeDict({ + 'pid': proc.pid, + 'name': proc.name(), + 'mem_rss': mem.rss, + 'mem_vms': mem.vms, + 'cpu_percent': proc.cpu_percent(), + 'create_time': proc.create_time(), + 'cmd': ' '.join(proc.cmdline()), + }) + + if with_children: + proc_formatted.update({ + 'children': [self._format_proc(x) + for x in proc.children(recursive=True)] + }) + except Exception: + log.exception('Failed to load proc') + proc_formatted = None + return proc_formatted + + def get_processes(self): + proc_list = [] + for p in psutil.process_iter(): + if 'gunicorn' in p.name(): + proc = self._format_proc(p, with_children=True) + if proc: + proc_list.append(proc) + + return proc_list + @LoginRequired() @HasPermissionAllDecorator('hg.admin') @view_config( @@ -50,8 +82,7 @@ class AdminProcessManagementView(BaseAppView): c.active = 'process_management' c.navlist = navigation_list(self.request) - c.gunicorn_processes = ( - p for p in psutil.process_iter() if 'gunicorn' in p.name()) + c.gunicorn_processes = self.get_processes() return self._get_template_context(c) @LoginRequired() @@ -62,8 +93,7 @@ class AdminProcessManagementView(BaseAppView): def process_management_data(self): _ = self.request.translate c = self.load_default_context() - c.gunicorn_processes = ( - p for p in psutil.process_iter() if 'gunicorn' in p.name()) + c.gunicorn_processes = self.get_processes() return self._get_template_context(c) @LoginRequired() @@ -75,9 +105,10 @@ class AdminProcessManagementView(BaseAppView): def process_management_signal(self): pids = self.request.json.get('pids', []) result = [] + def on_terminate(proc): msg = "process `PID:{}` terminated with exit code {}".format( - proc.pid, proc.returncode) + proc.pid, proc.returncode or 0) result.append(msg) procs = [] @@ -91,15 +122,22 @@ class AdminProcessManagementView(BaseAppView): children = proc.children(recursive=True) if children: - print('Wont kill Master Process') + log.warning('Wont kill Master Process') else: procs.append(proc) for p in procs: - p.terminate() + try: + p.terminate() + except psutil.AccessDenied as e: + log.warning('Access denied: {}'.format(e)) + gone, alive = psutil.wait_procs(procs, timeout=10, callback=on_terminate) for p in alive: - p.kill() + try: + p.kill() + except psutil.AccessDenied as e: + log.warning('Access denied: {}'.format(e)) return {'result': result} diff --git a/rhodecode/apps/admin/views/settings.py b/rhodecode/apps/admin/views/settings.py index cc4f7b63..87421587 100644 --- a/rhodecode/apps/admin/views/settings.py +++ b/rhodecode/apps/admin/views/settings.py @@ -314,6 +314,9 @@ class AdminSettingsView(BaseAppView): try: form_result = application_form.to_python(dict(self.request.POST)) except formencode.Invalid as errors: + h.flash( + _("Some form inputs contain invalid data."), + category='error') data = render('rhodecode:templates/admin/settings/settings.mako', self._get_template_context(c), self.request) html = formencode.htmlfill.render( @@ -386,6 +389,9 @@ class AdminSettingsView(BaseAppView): try: form_result = application_form.to_python(dict(self.request.POST)) except formencode.Invalid as errors: + h.flash( + _("Some form inputs contain invalid data."), + category='error') data = render('rhodecode:templates/admin/settings/settings.mako', self._get_template_context(c), self.request) html = formencode.htmlfill.render( @@ -666,6 +672,17 @@ class AdminSettingsView(BaseAppView): return self._get_template_context(c) + @LoginRequired() + @HasPermissionAllDecorator('hg.admin') + @view_config( + route_name='admin_settings_automation', request_method='GET', + renderer='rhodecode:templates/admin/settings/settings.mako') + def settings_automation(self): + c = self.load_default_context() + c.active = 'automation' + + return self._get_template_context(c) + @LoginRequired() @HasPermissionAllDecorator('hg.admin') @view_config( @@ -705,7 +722,7 @@ class AdminSettingsView(BaseAppView): form_result = application_form.to_python(dict(self.request.POST)) except formencode.Invalid as errors: h.flash( - _('Some form inputs contain invalid data.'), + _("Some form inputs contain invalid data."), category='error') data = render('rhodecode:templates/admin/settings/settings.mako', self._get_template_context(c), self.request) diff --git a/rhodecode/apps/admin/views/system_info.py b/rhodecode/apps/admin/views/system_info.py index 0a98ec9e..945a1c64 100644 --- a/rhodecode/apps/admin/views/system_info.py +++ b/rhodecode/apps/admin/views/system_info.py @@ -123,6 +123,9 @@ class AdminSystemInfoSettingsView(BaseAppView): (_('Uptime'), val('uptime')['text'], state('uptime')), ('', '', ''), # spacer + # ulimit + (_('Ulimit'), val('ulimit')['text'], state('ulimit')), + # Repo storage (_('Storage location'), val('storage')['path'], state('storage')), (_('Storage info'), val('storage')['text'], state('storage')), diff --git a/rhodecode/apps/admin/views/user_groups.py b/rhodecode/apps/admin/views/user_groups.py index 57763adf..a34d377c 100644 --- a/rhodecode/apps/admin/views/user_groups.py +++ b/rhodecode/apps/admin/views/user_groups.py @@ -88,8 +88,8 @@ class AdminUserGroupsView(BaseAppView, DataGridAppView): _render = self.request.get_partial_renderer( 'rhodecode:templates/data_table/_dt_elements.mako') - def user_group_name(user_group_id, user_group_name): - return _render("user_group_name", user_group_id, user_group_name) + def user_group_name(user_group_name): + return _render("user_group_name", user_group_name) def user_group_actions(user_group_id, user_group_name): return _render("user_group_actions", user_group_id, user_group_name) @@ -153,15 +153,14 @@ class AdminUserGroupsView(BaseAppView, DataGridAppView): user_groups_data = [] for user_gr in auth_user_group_list: user_groups_data.append({ - "users_group_name": user_group_name( - user_gr.users_group_id, h.escape(user_gr.users_group_name)), + "users_group_name": user_group_name(user_gr.users_group_name), "name_raw": h.escape(user_gr.users_group_name), "description": h.escape(user_gr.user_group_description), "members": user_gr.member_count, # NOTE(marcink): because of advanced query we # need to load it like that - "sync": UserGroup._load_group_data( - user_gr.group_data).get('extern_type'), + "sync": UserGroup._load_sync( + UserGroup._load_group_data(user_gr.group_data)), "active": h.bool2icon(user_gr.users_group_active), "owner": user_profile(user_gr.User.username), "action": user_group_actions( diff --git a/rhodecode/apps/admin/views/users.py b/rhodecode/apps/admin/views/users.py index 1e0c038a..2de12237 100644 --- a/rhodecode/apps/admin/views/users.py +++ b/rhodecode/apps/admin/views/users.py @@ -817,6 +817,7 @@ class UsersView(UserAppView): key_data = self.request.POST.get('key_data') description = self.request.POST.get('description') + fingerprint = 'unknown' try: if not key_data: raise ValueError('Please add a valid public key') @@ -841,8 +842,9 @@ class UsersView(UserAppView): except IntegrityError: log.exception("Exception during ssh key saving") - h.flash(_('An error occurred during ssh key saving: {}').format( - 'Such key already exists, please use a different one'), + err = 'Such key with fingerprint `{}` already exists, ' \ + 'please use a different one'.format(fingerprint) + h.flash(_('An error occurred during ssh key saving: {}').format(err), category='error') except Exception as e: log.exception("Exception during ssh key saving") diff --git a/rhodecode/apps/my_account/tests/test_my_account_edit.py b/rhodecode/apps/my_account/tests/test_my_account_edit.py index 8d906600..d249d098 100644 --- a/rhodecode/apps/my_account/tests/test_my_account_edit.py +++ b/rhodecode/apps/my_account/tests/test_my_account_edit.py @@ -109,7 +109,7 @@ class TestMyAccountEdit(TestController): # ('extern_name', {'extern_name': None}), ('active', {'active': False}), ('active', {'active': True}), - ('email', {'email': 'some@email.com'}), + ('email', {'email': u'some@email.com'}), ]) def test_my_account_update(self, name, attrs, user_util): usr = user_util.create_user(password='qweqwe') @@ -120,13 +120,17 @@ class TestMyAccountEdit(TestController): params.update({'password_confirmation': ''}) params.update({'new_password': ''}) - params.update({'extern_type': 'rhodecode'}) - params.update({'extern_name': 'rhodecode'}) + params.update({'extern_type': u'rhodecode'}) + params.update({'extern_name': u'rhodecode'}) params.update({'csrf_token': self.csrf_token}) params.update(attrs) # my account page cannot set language param yet, only for admins del params['language'] + if name == 'email': + uem = user_util.create_additional_user_email(usr, attrs['email']) + email_before = User.get(user_id).email + response = self.app.post(route_path('my_account_update'), params) assert_session_flash( @@ -146,7 +150,7 @@ class TestMyAccountEdit(TestController): params['language'] = updated_params['language'] if name == 'email': - params['emails'] = [attrs['email']] + params['emails'] = [attrs['email'], email_before] if name == 'extern_type': # cannot update this via form, expected value is original one params['extern_type'] = "rhodecode" @@ -162,10 +166,10 @@ class TestMyAccountEdit(TestController): assert params == updated_params - def test_my_account_update_err_email_exists(self): + def test_my_account_update_err_email_not_exists_in_emails(self): self.log_user() - new_email = 'test_regular@mail.com' # already existing email + new_email = 'test_regular@mail.com' # not in emails params = { 'username': 'test_admin', 'new_password': 'test12', @@ -179,7 +183,7 @@ class TestMyAccountEdit(TestController): response = self.app.post(route_path('my_account_update'), params=params) - response.mustcontain('This e-mail address is already taken') + response.mustcontain('"test_regular@mail.com" is not one of test_admin@mail.com') def test_my_account_update_bad_email_address(self): self.log_user('test_regular2', 'test12') @@ -197,7 +201,4 @@ class TestMyAccountEdit(TestController): response = self.app.post(route_path('my_account_update'), params=params) - response.mustcontain('An email address must contain a single @') - msg = u'Username "%(username)s" already exists' - msg = h.html_escape(msg % {'username': 'test_admin'}) - response.mustcontain(u"%s" % msg) + response.mustcontain('"newmail.pl" is not one of test_regular2@mail.com') diff --git a/rhodecode/apps/my_account/tests/test_my_account_emails.py b/rhodecode/apps/my_account/tests/test_my_account_emails.py index 3c21b1e9..3b5d52a1 100644 --- a/rhodecode/apps/my_account/tests/test_my_account_emails.py +++ b/rhodecode/apps/my_account/tests/test_my_account_emails.py @@ -24,7 +24,7 @@ from rhodecode.apps._base import ADMIN_PREFIX from rhodecode.model.db import User, UserEmailMap from rhodecode.tests import ( TestController, TEST_USER_ADMIN_LOGIN, TEST_USER_REGULAR_EMAIL, - assert_session_flash) + assert_session_flash, TEST_USER_REGULAR_PASS) from rhodecode.tests.fixture import Fixture fixture = Fixture() @@ -47,30 +47,14 @@ class TestMyAccountEmails(TestController): response = self.app.get(route_path('my_account_emails')) response.mustcontain('No additional emails specified') - def test_my_account_my_emails_add_existing_email(self): - self.log_user() - response = self.app.get(route_path('my_account_emails')) - response.mustcontain('No additional emails specified') - response = self.app.post(route_path('my_account_emails_add'), - {'new_email': TEST_USER_REGULAR_EMAIL, - 'csrf_token': self.csrf_token}) - assert_session_flash(response, 'This e-mail address is already taken') - - def test_my_account_my_emails_add_mising_email_in_form(self): - self.log_user() - response = self.app.get(route_path('my_account_emails')) - response.mustcontain('No additional emails specified') - response = self.app.post(route_path('my_account_emails_add'), - {'csrf_token': self.csrf_token}) - assert_session_flash(response, 'Please enter an email address') - def test_my_account_my_emails_add_remove(self): self.log_user() response = self.app.get(route_path('my_account_emails')) response.mustcontain('No additional emails specified') response = self.app.post(route_path('my_account_emails_add'), - {'new_email': 'foo@barz.com', + {'email': 'foo@barz.com', + 'current_password': TEST_USER_REGULAR_PASS, 'csrf_token': self.csrf_token}) response = self.app.get(route_path('my_account_emails')) diff --git a/rhodecode/apps/my_account/tests/test_my_account_ssh_keys.py b/rhodecode/apps/my_account/tests/test_my_account_ssh_keys.py index 4a68d971..fcab22d3 100644 --- a/rhodecode/apps/my_account/tests/test_my_account_ssh_keys.py +++ b/rhodecode/apps/my_account/tests/test_my_account_ssh_keys.py @@ -66,6 +66,7 @@ class TestMyAccountSshKeysView(TestController): 'I4fG8+hBHzpeFxUGvSGNtXPUbwaAY8j/oHYrTpMgkj6pUEFsiKfC5zPq' \ 'PFR5HyKTCHW0nFUJnZsbyFT5hMiF/hZkJc9A0ZbdSvJwCRQ/g3bmdL ' \ 'your_email@example.com' + FINGERPRINT = 'MD5:01:4f:ad:29:22:6e:01:37:c9:d2:52:26:52:b0:2d:93' def test_add_ssh_key_error(self, user_util): user = user_util.create_user(password='qweqwe') @@ -100,9 +101,11 @@ class TestMyAccountSshKeysView(TestController): route_path('my_account_ssh_keys_add'), {'description': desc, 'key_data': key_data, 'csrf_token': self.csrf_token}) + + err = 'Such key with fingerprint `{}` already exists, ' \ + 'please use a different one'.format(self.FINGERPRINT) assert_session_flash(response, 'An error occurred during ssh key ' - 'saving: Such key already exists, ' - 'please use a different one') + 'saving: {}'.format(err)) def test_add_ssh_key(self, user_util): user = user_util.create_user(password='qweqwe') diff --git a/rhodecode/apps/my_account/views/my_account.py b/rhodecode/apps/my_account/views/my_account.py index 73d5d4e3..b937bf53 100644 --- a/rhodecode/apps/my_account/views/my_account.py +++ b/rhodecode/apps/my_account/views/my_account.py @@ -232,40 +232,59 @@ class MyAccountView(BaseAppView, DataGridAppView): c.user_email_map = UserEmailMap.query()\ .filter(UserEmailMap.user == c.user).all() + + schema = user_schema.AddEmailSchema().bind( + username=c.user.username, user_emails=c.user.emails) + + form = forms.RcForm(schema, + action=h.route_path('my_account_emails_add'), + buttons=(forms.buttons.save, forms.buttons.reset)) + + c.form = form return self._get_template_context(c) @LoginRequired() @NotAnonymous() @CSRFRequired() @view_config( - route_name='my_account_emails_add', request_method='POST') + route_name='my_account_emails_add', request_method='POST', + renderer='rhodecode:templates/admin/my_account/my_account.mako') def my_account_emails_add(self): _ = self.request.translate c = self.load_default_context() + c.active = 'emails' - email = self.request.POST.get('new_email') + schema = user_schema.AddEmailSchema().bind( + username=c.user.username, user_emails=c.user.emails) + form = forms.RcForm( + schema, action=h.route_path('my_account_emails_add'), + buttons=(forms.buttons.save, forms.buttons.reset)) + + controls = self.request.POST.items() try: - form = UserExtraEmailForm(self.request.translate)() - data = form.to_python({'email': email}) - email = data['email'] - - UserModel().add_extra_email(c.user.user_id, email) + valid_data = form.validate(controls) + UserModel().add_extra_email(c.user.user_id, valid_data['email']) audit_logger.store_web( 'user.edit.email.add', action_data={ - 'data': {'email': email, 'user': 'self'}}, + 'data': {'email': valid_data['email'], 'user': 'self'}}, user=self._rhodecode_user,) - Session().commit() - h.flash(_("Added new email address `%s` for user account") % email, - category='success') except formencode.Invalid as error: h.flash(h.escape(error.error_dict['email']), category='error') + except forms.ValidationFailure as e: + c.user_email_map = UserEmailMap.query() \ + .filter(UserEmailMap.user == c.user).all() + c.form = e + return self._get_template_context(c) except Exception: - log.exception("Exception in my_account_emails") - h.flash(_('An error occurred during email saving'), + log.exception("Exception adding email") + h.flash(_('Error occurred during adding email'), category='error') - return HTTPFound(h.route_path('my_account_emails')) + else: + h.flash(_("Successfully added email"), category='success') + + raise HTTPFound(self.request.route_path('my_account_emails')) @LoginRequired() @NotAnonymous() @@ -414,22 +433,23 @@ class MyAccountView(BaseAppView, DataGridAppView): def my_account_edit(self): c = self.load_default_context() c.active = 'profile_edit' - - c.perm_user = c.auth_user c.extern_type = c.user.extern_type c.extern_name = c.user.extern_name - defaults = c.user.get_dict() + schema = user_schema.UserProfileSchema().bind( + username=c.user.username, user_emails=c.user.emails) + appstruct = { + 'username': c.user.username, + 'email': c.user.email, + 'firstname': c.user.firstname, + 'lastname': c.user.lastname, + } + c.form = forms.RcForm( + schema, appstruct=appstruct, + action=h.route_path('my_account_update'), + buttons=(forms.buttons.save, forms.buttons.reset)) - data = render('rhodecode:templates/admin/my_account/my_account.mako', - self._get_template_context(c), self.request) - html = formencode.htmlfill.render( - data, - defaults=defaults, - encoding="UTF-8", - force_defaults=False - ) - return Response(html) + return self._get_template_context(c) @LoginRequired() @NotAnonymous() @@ -442,55 +462,40 @@ class MyAccountView(BaseAppView, DataGridAppView): _ = self.request.translate c = self.load_default_context() c.active = 'profile_edit' - c.perm_user = c.auth_user c.extern_type = c.user.extern_type c.extern_name = c.user.extern_name - _form = UserForm(self.request.translate, edit=True, - old_data={'user_id': self._rhodecode_user.user_id, - 'email': self._rhodecode_user.email})() - form_result = {} + schema = user_schema.UserProfileSchema().bind( + username=c.user.username, user_emails=c.user.emails) + form = forms.RcForm( + schema, buttons=(forms.buttons.save, forms.buttons.reset)) + + controls = self.request.POST.items() try: - post_data = dict(self.request.POST) - post_data['new_password'] = '' - post_data['password_confirmation'] = '' - form_result = _form.to_python(post_data) - # skip updating those attrs for my account + valid_data = form.validate(controls) skip_attrs = ['admin', 'active', 'extern_type', 'extern_name', 'new_password', 'password_confirmation'] - # TODO: plugin should define if username can be updated if c.extern_type != "rhodecode": # forbid updating username for external accounts skip_attrs.append('username') - + old_email = c.user.email UserModel().update_user( - self._rhodecode_user.user_id, skip_attrs=skip_attrs, - **form_result) - h.flash(_('Your account was updated successfully'), - category='success') + self._rhodecode_user.user_id, skip_attrs=skip_attrs, + **valid_data) + if old_email != valid_data['email']: + old = UserEmailMap.query() \ + .filter(UserEmailMap.user == c.user).filter(UserEmailMap.email == valid_data['email']).first() + old.email = old_email + h.flash(_('Your account was updated successfully'), category='success') Session().commit() - - except formencode.Invalid as errors: - data = render( - 'rhodecode:templates/admin/my_account/my_account.mako', - self._get_template_context(c), self.request) - - html = formencode.htmlfill.render( - data, - defaults=errors.value, - errors=errors.error_dict or {}, - prefix_error=False, - encoding="UTF-8", - force_defaults=False) - return Response(html) - + except forms.ValidationFailure as e: + c.form = e + return self._get_template_context(c) except Exception: log.exception("Exception updating user") - h.flash(_('Error occurred during update of user %s') - % form_result.get('username'), category='error') - raise HTTPFound(h.route_path('my_account_profile')) - + h.flash(_('Error occurred during update of user'), + category='error') raise HTTPFound(h.route_path('my_account_profile')) def _get_pull_requests_list(self, statuses): diff --git a/rhodecode/apps/my_account/views/my_account_ssh_keys.py b/rhodecode/apps/my_account/views/my_account_ssh_keys.py index bd7304e7..78036171 100644 --- a/rhodecode/apps/my_account/views/my_account_ssh_keys.py +++ b/rhodecode/apps/my_account/views/my_account_ssh_keys.py @@ -89,7 +89,7 @@ class MyAccountSshKeysView(BaseAppView, DataGridAppView): user_data = c.user.get_api_data() key_data = self.request.POST.get('key_data') description = self.request.POST.get('description') - + fingerprint = 'unknown' try: if not key_data: raise ValueError('Please add a valid public key') @@ -114,8 +114,9 @@ class MyAccountSshKeysView(BaseAppView, DataGridAppView): except IntegrityError: log.exception("Exception during ssh key saving") - h.flash(_('An error occurred during ssh key saving: {}').format( - 'Such key already exists, please use a different one'), + err = 'Such key with fingerprint `{}` already exists, ' \ + 'please use a different one'.format(fingerprint) + h.flash(_('An error occurred during ssh key saving: {}').format(err), category='error') except Exception as e: log.exception("Exception during ssh key saving") diff --git a/rhodecode/apps/repository/__init__.py b/rhodecode/apps/repository/__init__.py index d1796de1..a34f6462 100644 --- a/rhodecode/apps/repository/__init__.py +++ b/rhodecode/apps/repository/__init__.py @@ -331,6 +331,10 @@ def includeme(config): name='edit_repo_advanced_fork', pattern='/{repo_name:.*?[^/]}/settings/advanced/fork', repo_route=True) + config.add_route( + name='edit_repo_advanced_hooks', + pattern='/{repo_name:.*?[^/]}/settings/advanced/hooks', repo_route=True) + # Caches config.add_route( name='edit_repo_caches', @@ -373,6 +377,9 @@ def includeme(config): config.add_route( name='edit_repo_remote_pull', pattern='/{repo_name:.*?[^/]}/settings/remote/pull', repo_route=True) + config.add_route( + name='edit_repo_remote_push', + pattern='/{repo_name:.*?[^/]}/settings/remote/push', repo_route=True) # Statistics config.add_route( @@ -418,6 +425,11 @@ def includeme(config): name='repo_default_reviewers_data', pattern='/{repo_name:.*?[^/]}/settings/review/default-reviewers', repo_route=True) + # Repo Automation (EE feature) + config.add_route( + name='repo_automation', + pattern='/{repo_name:.*?[^/]}/settings/automation', repo_route=True) + # Strip config.add_route( name='edit_repo_strip', diff --git a/rhodecode/apps/repository/tests/test_repo_summary.py b/rhodecode/apps/repository/tests/test_repo_summary.py index 740e950e..ab0c8032 100644 --- a/rhodecode/apps/repository/tests/test_repo_summary.py +++ b/rhodecode/apps/repository/tests/test_repo_summary.py @@ -234,7 +234,8 @@ class TestSummaryView(object): Repository, 'scm_instance', side_effect=RepositoryRequirementError) with scm_patcher: - response = self.app.get(route_path('repo_summary', repo_name=repo_name)) + response = self.app.get( + route_path('repo_summary', repo_name=repo_name)) assert_response = AssertResponse(response) assert_response.element_contains( '.main .alert-warning strong', 'Missing requirements') diff --git a/rhodecode/apps/repository/views/repo_automation.py b/rhodecode/apps/repository/views/repo_automation.py new file mode 100644 index 00000000..c253b3e9 --- /dev/null +++ b/rhodecode/apps/repository/views/repo_automation.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2016-2018 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +import logging + +from pyramid.view import view_config + +from rhodecode.apps._base import RepoAppView +from rhodecode.apps.repository.utils import get_default_reviewers_data +from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator + +log = logging.getLogger(__name__) + + +class RepoAutomationView(RepoAppView): + def load_default_context(self): + c = self._get_local_tmpl_context() + return c + + @LoginRequired() + @HasRepoPermissionAnyDecorator('repository.admin') + @view_config( + route_name='repo_automation', request_method='GET', + renderer='rhodecode:templates/admin/repos/repo_edit.mako') + def repo_automation(self): + c = self.load_default_context() + c.active = 'automation' + + return self._get_template_context(c) diff --git a/rhodecode/apps/repository/views/repo_caches.py b/rhodecode/apps/repository/views/repo_caches.py index f335819b..1dc0e66e 100644 --- a/rhodecode/apps/repository/views/repo_caches.py +++ b/rhodecode/apps/repository/views/repo_caches.py @@ -18,6 +18,7 @@ # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ +import os import logging from pyramid.httpexceptions import HTTPFound @@ -27,6 +28,7 @@ from rhodecode.apps._base import RepoAppView from rhodecode.lib.auth import ( LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired) from rhodecode.lib import helpers as h +from rhodecode.lib import system_info from rhodecode.model.meta import Session from rhodecode.model.scm import ScmModel @@ -36,8 +38,6 @@ log = logging.getLogger(__name__) class RepoCachesView(RepoAppView): def load_default_context(self): c = self._get_local_tmpl_context() - - return c @LoginRequired() @@ -48,6 +48,11 @@ class RepoCachesView(RepoAppView): def repo_caches(self): c = self.load_default_context() c.active = 'caches' + cached_diffs_dir = c.rhodecode_db_repo.cached_diffs_dir + c.cached_diff_count = len(c.rhodecode_db_repo.cached_diffs()) + c.cached_diff_size = 0 + if os.path.isdir(cached_diffs_dir): + c.cached_diff_size = system_info.get_storage_size(cached_diffs_dir) return self._get_template_context(c) diff --git a/rhodecode/apps/repository/views/repo_commits.py b/rhodecode/apps/repository/views/repo_commits.py index b9e891cb..2a584b8a 100644 --- a/rhodecode/apps/repository/views/repo_commits.py +++ b/rhodecode/apps/repository/views/repo_commits.py @@ -34,17 +34,18 @@ from rhodecode.lib.auth import ( LoginRequired, HasRepoPermissionAnyDecorator, NotAnonymous, CSRFRequired) from rhodecode.lib.compat import OrderedDict +from rhodecode.lib.diffs import cache_diff, load_cached_diff, diff_cache_exist from rhodecode.lib.exceptions import StatusChangeOnClosedPullRequestError import rhodecode.lib.helpers as h -from rhodecode.lib.utils2 import safe_unicode +from rhodecode.lib.utils2 import safe_unicode, str2bool from rhodecode.lib.vcs.backends.base import EmptyCommit from rhodecode.lib.vcs.exceptions import ( - RepositoryError, CommitDoesNotExistError, NodeDoesNotExistError) + RepositoryError, CommitDoesNotExistError) from rhodecode.model.db import ChangesetComment, ChangesetStatus from rhodecode.model.changeset_status import ChangesetStatusModel from rhodecode.model.comment import CommentsModel from rhodecode.model.meta import Session - +from rhodecode.model.settings import VcsSettingsModel log = logging.getLogger(__name__) @@ -152,6 +153,12 @@ class RepoCommitsView(RepoAppView): return c + def _is_diff_cache_enabled(self, target_repo): + caching_enabled = self._get_general_setting( + target_repo, 'rhodecode_diff_cache') + log.debug('Diff caching enabled: %s', caching_enabled) + return caching_enabled + def _commit(self, commit_id_range, method): _ = self.request.translate c = self.load_default_context() @@ -240,45 +247,65 @@ class RepoCommitsView(RepoAppView): commit2 = commit commit1 = commit.parents[0] if commit.parents else EmptyCommit() - _diff = self.rhodecode_vcs_repo.get_diff( - commit1, commit2, - ignore_whitespace=ign_whitespace_lcl, context=context_lcl) - diff_processor = diffs.DiffProcessor( - _diff, format='newdiff', diff_limit=diff_limit, - file_limit=file_limit, show_full_diff=c.fulldiff) - - commit_changes = OrderedDict() if method == 'show': - _parsed = diff_processor.prepare() - c.limited_diff = isinstance(_parsed, diffs.LimitedDiffContainer) - - _parsed = diff_processor.prepare() - - def _node_getter(commit): - def get_node(fname): - try: - return commit.get_node(fname) - except NodeDoesNotExistError: - return None - return get_node - inline_comments = CommentsModel().get_inline_comments( self.db_repo.repo_id, revision=commit.raw_id) c.inline_cnt = CommentsModel().get_inline_comments_count( inline_comments) + c.inline_comments = inline_comments - diffset = codeblocks.DiffSet( - repo_name=self.db_repo_name, - source_node_getter=_node_getter(commit1), - target_node_getter=_node_getter(commit2), - comments=inline_comments) - diffset = diffset.render_patchset( - _parsed, commit1.raw_id, commit2.raw_id) + cache_path = self.rhodecode_vcs_repo.get_create_shadow_cache_pr_path( + self.db_repo) + cache_file_path = diff_cache_exist( + cache_path, 'diff', commit.raw_id, + ign_whitespace_lcl, context_lcl, c.fulldiff) + caching_enabled = self._is_diff_cache_enabled(self.db_repo) + force_recache = str2bool(self.request.GET.get('force_recache')) + + cached_diff = None + if caching_enabled: + cached_diff = load_cached_diff(cache_file_path) + + has_proper_diff_cache = cached_diff and cached_diff.get('diff') + if not force_recache and has_proper_diff_cache: + diffset = cached_diff['diff'] + else: + vcs_diff = self.rhodecode_vcs_repo.get_diff( + commit1, commit2, + ignore_whitespace=ign_whitespace_lcl, + context=context_lcl) + + diff_processor = diffs.DiffProcessor( + vcs_diff, format='newdiff', diff_limit=diff_limit, + file_limit=file_limit, show_full_diff=c.fulldiff) + + _parsed = diff_processor.prepare() + + diffset = codeblocks.DiffSet( + repo_name=self.db_repo_name, + source_node_getter=codeblocks.diffset_node_getter(commit1), + target_node_getter=codeblocks.diffset_node_getter(commit2)) + + diffset = self.path_filter.render_patchset_filtered( + diffset, _parsed, commit1.raw_id, commit2.raw_id) + + # save cached diff + if caching_enabled: + cache_diff(cache_file_path, diffset, None) + + c.limited_diff = diffset.limited_diff c.changes[commit.raw_id] = diffset else: + # TODO(marcink): no cache usage here... + _diff = self.rhodecode_vcs_repo.get_diff( + commit1, commit2, + ignore_whitespace=ign_whitespace_lcl, context=context_lcl) + diff_processor = diffs.DiffProcessor( + _diff, format='newdiff', diff_limit=diff_limit, + file_limit=file_limit, show_full_diff=c.fulldiff) # downloads/raw we only need RAW diff nothing else - diff = diff_processor.as_raw() + diff = self.path_filter.get_raw_patch(diff_processor) c.changes[commit.raw_id] = [None, None, None, None, diff, None, None] # sort comments by how they were generated diff --git a/rhodecode/apps/repository/views/repo_compare.py b/rhodecode/apps/repository/views/repo_compare.py index 33726e03..300dad8c 100644 --- a/rhodecode/apps/repository/views/repo_compare.py +++ b/rhodecode/apps/repository/views/repo_compare.py @@ -295,22 +295,13 @@ class RepoCompareView(RepoAppView): file_limit=file_limit, show_full_diff=c.fulldiff) _parsed = diff_processor.prepare() - def _node_getter(commit): - """ Returns a function that returns a node for a commit or None """ - def get_node(fname): - try: - return commit.get_node(fname) - except NodeDoesNotExistError: - return None - return get_node - diffset = codeblocks.DiffSet( repo_name=source_repo.repo_name, - source_node_getter=_node_getter(source_commit), - target_node_getter=_node_getter(target_commit), + source_node_getter=codeblocks.diffset_node_getter(source_commit), + target_node_getter=codeblocks.diffset_node_getter(target_commit), ) - c.diffset = diffset.render_patchset( - _parsed, source_ref, target_ref) + c.diffset = self.path_filter.render_patchset_filtered( + diffset, _parsed, source_ref, target_ref) c.preview_mode = merge c.source_commit = source_commit diff --git a/rhodecode/apps/repository/views/repo_feed.py b/rhodecode/apps/repository/views/repo_feed.py index 0c4e3eb1..acf2de27 100644 --- a/rhodecode/apps/repository/views/repo_feed.py +++ b/rhodecode/apps/repository/views/repo_feed.py @@ -90,13 +90,15 @@ class RepoFeedView(RepoAppView): _renderer = self.request.get_partial_renderer( 'rhodecode:templates/feed/atom_feed_entry.mako') diff_processor, parsed_diff, limited_diff = self._changes(commit) + filtered_parsed_diff, has_hidden_changes = self.path_filter.filter_patchset(parsed_diff) return _renderer( 'body', commit=commit, - parsed_diff=parsed_diff, + parsed_diff=filtered_parsed_diff, limited_diff=limited_diff, feed_include_diff=self.feed_include_diff, diff_processor=diff_processor, + has_hidden_changes=has_hidden_changes ) def _set_timezone(self, date, tzinfo=pytz.utc): @@ -122,8 +124,7 @@ class RepoFeedView(RepoAppView): """ self.load_default_context() - @cache_region('long_term') - def _generate_feed(cache_key): + def _generate_feed(): feed = Atom1Feed( title=self.title % self.db_repo_name, link=h.route_url('repo_summary', repo_name=self.db_repo_name), @@ -146,12 +147,18 @@ class RepoFeedView(RepoAppView): return feed.mime_type, feed.writeString('utf-8') - invalidator_context = CacheKey.repo_context_cache( - _generate_feed, self.db_repo_name, CacheKey.CACHE_TYPE_ATOM) + @cache_region('long_term') + def _generate_feed_and_cache(cache_key): + return _generate_feed() - with invalidator_context as context: - context.invalidate() - mime_type, feed = context.compute() + if self.path_filter.is_enabled: + invalidator_context = CacheKey.repo_context_cache( + _generate_feed_and_cache, self.db_repo_name, CacheKey.CACHE_TYPE_ATOM) + with invalidator_context as context: + context.invalidate() + mime_type, feed = context.compute() + else: + mime_type, feed = _generate_feed() response = Response(feed) response.content_type = mime_type @@ -169,8 +176,7 @@ class RepoFeedView(RepoAppView): """ self.load_default_context() - @cache_region('long_term') - def _generate_feed(cache_key): + def _generate_feed(): feed = Rss201rev2Feed( title=self.title % self.db_repo_name, link=h.route_url('repo_summary', repo_name=self.db_repo_name), @@ -193,12 +199,19 @@ class RepoFeedView(RepoAppView): return feed.mime_type, feed.writeString('utf-8') - invalidator_context = CacheKey.repo_context_cache( - _generate_feed, self.db_repo_name, CacheKey.CACHE_TYPE_RSS) + @cache_region('long_term') + def _generate_feed_and_cache(cache_key): + return _generate_feed() - with invalidator_context as context: - context.invalidate() - mime_type, feed = context.compute() + if self.path_filter.is_enabled: + invalidator_context = CacheKey.repo_context_cache( + _generate_feed_and_cache, self.db_repo_name, CacheKey.CACHE_TYPE_RSS) + + with invalidator_context as context: + context.invalidate() + mime_type, feed = context.compute() + else: + mime_type, feed = _generate_feed() response = Response(feed) response.content_type = mime_type diff --git a/rhodecode/apps/repository/views/repo_files.py b/rhodecode/apps/repository/views/repo_files.py index a7243f79..8bf08a04 100644 --- a/rhodecode/apps/repository/views/repo_files.py +++ b/rhodecode/apps/repository/views/repo_files.py @@ -426,7 +426,7 @@ class RepoFilesView(RepoAppView): context=line_context) diff = diffs.DiffProcessor(_diff, format='gitdiff') - response = Response(diff.as_raw()) + response = Response(self.path_filter.get_raw_patch(diff)) response.content_type = 'text/plain' response.content_disposition = ( 'attachment; filename=%s_%s_vs_%s.diff' % (f_path, diff1, diff2) @@ -442,7 +442,7 @@ class RepoFilesView(RepoAppView): context=line_context) diff = diffs.DiffProcessor(_diff, format='gitdiff') - response = Response(diff.as_raw()) + response = Response(self.path_filter.get_raw_patch(diff)) response.content_type = 'text/plain' charset = self._get_default_encoding(c) if charset: @@ -462,7 +462,7 @@ class RepoFilesView(RepoAppView): """ Kept only to make OLD links work """ - f_path = self._get_f_path(self.request.matchdict) + f_path = self._get_f_path_unchecked(self.request.matchdict) diff1 = self.request.GET.get('diff1', '') diff2 = self.request.GET.get('diff2', '') diff --git a/rhodecode/apps/repository/views/repo_pull_requests.py b/rhodecode/apps/repository/views/repo_pull_requests.py index 4638e5ca..6bcc6b0e 100644 --- a/rhodecode/apps/repository/views/repo_pull_requests.py +++ b/rhodecode/apps/repository/views/repo_pull_requests.py @@ -34,6 +34,7 @@ from rhodecode.apps._base import RepoAppView, DataGridAppView from rhodecode.lib import helpers as h, diffs, codeblocks, channelstream from rhodecode.lib.base import vcs_operation_context +from rhodecode.lib.diffs import load_cached_diff, cache_diff, diff_cache_exist from rhodecode.lib.ext_json import json from rhodecode.lib.auth import ( LoginRequired, HasRepoPermissionAny, HasRepoPermissionAnyDecorator, @@ -41,7 +42,7 @@ from rhodecode.lib.auth import ( from rhodecode.lib.utils2 import str2bool, safe_str, safe_unicode from rhodecode.lib.vcs.backends.base import EmptyCommit, UpdateFailureReason from rhodecode.lib.vcs.exceptions import (CommitDoesNotExistError, - RepositoryRequirementError, NodeDoesNotExistError, EmptyRepositoryError) + RepositoryRequirementError, EmptyRepositoryError) from rhodecode.model.changeset_status import ChangesetStatusModel from rhodecode.model.comment import CommentsModel from rhodecode.model.db import (func, or_, PullRequest, PullRequestVersion, @@ -201,10 +202,16 @@ class RepoPullRequestsView(RepoAppView, DataGridAppView): return data + def _is_diff_cache_enabled(self, target_repo): + caching_enabled = self._get_general_setting( + target_repo, 'rhodecode_diff_cache') + log.debug('Diff caching enabled: %s', caching_enabled) + return caching_enabled + def _get_diffset(self, source_repo_name, source_repo, source_ref_id, target_ref_id, - target_commit, source_commit, diff_limit, fulldiff, - file_limit, display_inline_comments): + target_commit, source_commit, diff_limit, file_limit, + fulldiff): vcs_diff = PullRequestModel().get_diff( source_repo, source_ref_id, target_ref_id) @@ -215,24 +222,14 @@ class RepoPullRequestsView(RepoAppView, DataGridAppView): _parsed = diff_processor.prepare() - def _node_getter(commit): - def get_node(fname): - try: - return commit.get_node(fname) - except NodeDoesNotExistError: - return None - - return get_node - diffset = codeblocks.DiffSet( repo_name=self.db_repo_name, source_repo_name=source_repo_name, - source_node_getter=_node_getter(target_commit), - target_node_getter=_node_getter(source_commit), - comments=display_inline_comments + source_node_getter=codeblocks.diffset_node_getter(target_commit), + target_node_getter=codeblocks.diffset_node_getter(source_commit), ) - diffset = diffset.render_patchset( - _parsed, target_commit.raw_id, source_commit.raw_id) + diffset = self.path_filter.render_patchset_filtered( + diffset, _parsed, target_commit.raw_id, source_commit.raw_id) return diffset @@ -443,42 +440,54 @@ class RepoPullRequestsView(RepoAppView, DataGridAppView): commits_source_repo = source_scm c.commits_source_repo = commits_source_repo - commit_cache = {} - try: - pre_load = ["author", "branch", "date", "message"] - show_revs = pull_request_at_ver.revisions - for rev in show_revs: - comm = commits_source_repo.get_commit( - commit_id=rev, pre_load=pre_load) - c.commit_ranges.append(comm) - commit_cache[comm.raw_id] = comm - - # Order here matters, we first need to get target, and then - # the source - target_commit = commits_source_repo.get_commit( - commit_id=safe_str(target_ref_id)) - - source_commit = commits_source_repo.get_commit( - commit_id=safe_str(source_ref_id)) - - except CommitDoesNotExistError: - log.warning( - 'Failed to get commit from `{}` repo'.format( - commits_source_repo), exc_info=True) - except RepositoryRequirementError: - log.warning( - 'Failed to get all required data from repo', exc_info=True) - c.missing_requirements = True - c.ancestor = None # set it to None, to hide it from PR view - try: - ancestor_id = source_scm.get_common_ancestor( - source_commit.raw_id, target_commit.raw_id, target_scm) - c.ancestor_commit = source_scm.get_commit(ancestor_id) - except Exception: - c.ancestor_commit = None + # empty version means latest, so we keep this to prevent + # double caching + version_normalized = version or 'latest' + from_version_normalized = from_version or 'latest' + cache_path = self.rhodecode_vcs_repo.get_create_shadow_cache_pr_path( + target_repo) + cache_file_path = diff_cache_exist( + cache_path, 'pull_request', pull_request_id, version_normalized, + from_version_normalized, source_ref_id, target_ref_id, c.fulldiff) + + caching_enabled = self._is_diff_cache_enabled(c.target_repo) + force_recache = str2bool(self.request.GET.get('force_recache')) + + cached_diff = None + if caching_enabled: + cached_diff = load_cached_diff(cache_file_path) + + has_proper_commit_cache = ( + cached_diff and cached_diff.get('commits') + and len(cached_diff.get('commits', [])) == 5 + and cached_diff.get('commits')[0] + and cached_diff.get('commits')[3]) + if not force_recache and has_proper_commit_cache: + diff_commit_cache = \ + (ancestor_commit, commit_cache, missing_requirements, + source_commit, target_commit) = cached_diff['commits'] + else: + diff_commit_cache = \ + (ancestor_commit, commit_cache, missing_requirements, + source_commit, target_commit) = self.get_commits( + commits_source_repo, + pull_request_at_ver, + source_commit, + source_ref_id, + source_scm, + target_commit, + target_ref_id, + target_scm) + + # register our commit range + for comm in commit_cache.values(): + c.commit_ranges.append(comm) + + c.missing_requirements = missing_requirements + c.ancestor_commit = ancestor_commit c.statuses = source_repo.statuses( [x.raw_id for x in c.commit_ranges]) @@ -500,12 +509,23 @@ class RepoPullRequestsView(RepoAppView, DataGridAppView): c.missing_commits = True else: + c.inline_comments = display_inline_comments - c.diffset = self._get_diffset( - c.source_repo.repo_name, commits_source_repo, - source_ref_id, target_ref_id, - target_commit, source_commit, - diff_limit, c.fulldiff, file_limit, display_inline_comments) + has_proper_diff_cache = cached_diff and cached_diff.get('commits') + if not force_recache and has_proper_diff_cache: + c.diffset = cached_diff['diff'] + (ancestor_commit, commit_cache, missing_requirements, + source_commit, target_commit) = cached_diff['commits'] + else: + c.diffset = self._get_diffset( + c.source_repo.repo_name, commits_source_repo, + source_ref_id, target_ref_id, + target_commit, source_commit, + diff_limit, file_limit, c.fulldiff) + + # save cached diff + if caching_enabled: + cache_diff(cache_file_path, c.diffset, diff_commit_cache) c.limited_diff = c.diffset.limited_diff @@ -568,7 +588,6 @@ class RepoPullRequestsView(RepoAppView, DataGridAppView): if self._rhodecode_user.user_id in allowed_reviewers: for co in general_comments: if co.author.user_id == self._rhodecode_user.user_id: - # each comment has a status change status = co.status_change if status: _ver_pr = status[0].comment.pull_request_version_id @@ -576,6 +595,43 @@ class RepoPullRequestsView(RepoAppView, DataGridAppView): return self._get_template_context(c) + def get_commits( + self, commits_source_repo, pull_request_at_ver, source_commit, + source_ref_id, source_scm, target_commit, target_ref_id, target_scm): + commit_cache = collections.OrderedDict() + missing_requirements = False + try: + pre_load = ["author", "branch", "date", "message"] + show_revs = pull_request_at_ver.revisions + for rev in show_revs: + comm = commits_source_repo.get_commit( + commit_id=rev, pre_load=pre_load) + commit_cache[comm.raw_id] = comm + + # Order here matters, we first need to get target, and then + # the source + target_commit = commits_source_repo.get_commit( + commit_id=safe_str(target_ref_id)) + + source_commit = commits_source_repo.get_commit( + commit_id=safe_str(source_ref_id)) + except CommitDoesNotExistError: + log.warning( + 'Failed to get commit from `{}` repo'.format( + commits_source_repo), exc_info=True) + except RepositoryRequirementError: + log.warning( + 'Failed to get all required data from repo', exc_info=True) + missing_requirements = True + ancestor_commit = None + try: + ancestor_id = source_scm.get_common_ancestor( + source_commit.raw_id, target_commit.raw_id, target_scm) + ancestor_commit = source_scm.get_commit(ancestor_id) + except Exception: + ancestor_commit = None + return ancestor_commit, commit_cache, missing_requirements, source_commit, target_commit + def assure_not_empty_repo(self): _ = self.request.translate diff --git a/rhodecode/apps/repository/views/repo_settings.py b/rhodecode/apps/repository/views/repo_settings.py index 7dd314dd..b21fe5e7 100644 --- a/rhodecode/apps/repository/views/repo_settings.py +++ b/rhodecode/apps/repository/views/repo_settings.py @@ -138,15 +138,19 @@ class RepoSettingsView(RepoAppView): repo_description=schema_data['repo_description'], repo_private=schema_data['repo_private'], clone_uri=schema_data['repo_clone_uri'], + push_uri=schema_data['repo_push_uri'], repo_landing_rev=schema_data['repo_landing_commit_ref'], repo_enable_statistics=schema_data['repo_enable_statistics'], repo_enable_locking=schema_data['repo_enable_locking'], repo_enable_downloads=schema_data['repo_enable_downloads'], ) - # detect if CLONE URI changed, if we get OLD means we keep old values + # detect if SYNC URI changed, if we get OLD means we keep old values if schema_data['repo_clone_uri_change'] == 'OLD': validated_updates['clone_uri'] = self.db_repo.clone_uri + if schema_data['repo_push_uri_change'] == 'OLD': + validated_updates['push_uri'] = self.db_repo.push_uri + # use the new full name for redirect new_repo_name = schema_data['repo_group']['repo_name_with_group'] diff --git a/rhodecode/apps/repository/views/repo_settings_advanced.py b/rhodecode/apps/repository/views/repo_settings_advanced.py index d64526dd..bf020719 100644 --- a/rhodecode/apps/repository/views/repo_settings_advanced.py +++ b/rhodecode/apps/repository/views/repo_settings_advanced.py @@ -43,8 +43,6 @@ class RepoSettingsView(RepoAppView): def load_default_context(self): c = self._get_local_tmpl_context() - - return c @LoginRequired() @@ -231,3 +229,19 @@ class RepoSettingsView(RepoAppView): raise HTTPFound( h.route_path('edit_repo_advanced', repo_name=self.db_repo_name)) + + @LoginRequired() + @HasRepoPermissionAnyDecorator('repository.admin') + @view_config( + route_name='edit_repo_advanced_hooks', request_method='GET', + renderer='rhodecode:templates/admin/repos/repo_edit.mako') + def edit_advanced_install_hooks(self): + """ + Install Hooks for repository + """ + _ = self.request.translate + self.load_default_context() + self.rhodecode_vcs_repo.install_hooks(force=True) + h.flash(_('installed hooks repository'), category='success') + raise HTTPFound( + h.route_path('edit_repo_advanced', repo_name=self.db_repo_name)) diff --git a/rhodecode/apps/repository/views/repo_settings_remote.py b/rhodecode/apps/repository/views/repo_settings_remote.py index d1ce1131..12f13889 100644 --- a/rhodecode/apps/repository/views/repo_settings_remote.py +++ b/rhodecode/apps/repository/views/repo_settings_remote.py @@ -35,8 +35,6 @@ log = logging.getLogger(__name__) class RepoSettingsRemoteView(RepoAppView): def load_default_context(self): c = self._get_local_tmpl_context() - - return c @LoginRequired() diff --git a/rhodecode/apps/ssh_support/lib/backends/svn.py b/rhodecode/apps/ssh_support/lib/backends/svn.py index dd048b15..93c731c9 100644 --- a/rhodecode/apps/ssh_support/lib/backends/svn.py +++ b/rhodecode/apps/ssh_support/lib/backends/svn.py @@ -117,6 +117,7 @@ class SubversionTunnelWrapper(object): message=self._svn_string(message))) self.remove_configs() self.process.kill() + return 1 def interrupt(self, signum, frame): self.fail("Exited by timeout") @@ -171,7 +172,7 @@ class SubversionTunnelWrapper(object): first_response = self.get_first_client_response() if not first_response: - self.fail("Repository name cannot be extracted") + return self.fail("Repository name cannot be extracted") url_parts = urlparse.urlparse(first_response['url']) self.server.repo_name = url_parts.path.strip('/') diff --git a/rhodecode/apps/ssh_support/lib/ssh_wrapper.py b/rhodecode/apps/ssh_support/lib/ssh_wrapper.py index db5920a7..4215a944 100644 --- a/rhodecode/apps/ssh_support/lib/ssh_wrapper.py +++ b/rhodecode/apps/ssh_support/lib/ssh_wrapper.py @@ -68,7 +68,7 @@ def main(ini_path, mode, user, user_id, key_id, shell, debug): 'of this script.') connection_info = os.environ.get('SSH_CONNECTION', '') - with bootstrap(ini_path) as env: + with bootstrap(ini_path, env={'RC_CMD_SSH_WRAPPER': '1'}) as env: try: ssh_wrapper = SshWrapper( command, connection_info, mode, diff --git a/rhodecode/apps/ssh_support/tests/test_server_svn.py b/rhodecode/apps/ssh_support/tests/test_server_svn.py index e88850a3..10e835f1 100644 --- a/rhodecode/apps/ssh_support/tests/test_server_svn.py +++ b/rhodecode/apps/ssh_support/tests/test_server_svn.py @@ -103,7 +103,7 @@ class TestSubversionServer(object): return_value=0): with mock.patch.object( SubversionTunnelWrapper, 'command', - return_value='date'): + return_value=['date']): exit_code = server.run() # SVN has this differently configured, and we get in our mock env @@ -115,7 +115,7 @@ class TestSubversionServer(object): from rhodecode.apps.ssh_support.lib.backends.svn import SubversionTunnelWrapper with mock.patch.object( SubversionTunnelWrapper, 'command', - return_value='date'): + return_value=['date']): with mock.patch.object( SubversionTunnelWrapper, 'get_first_client_response', return_value=None): diff --git a/rhodecode/apps/svn_support/templates/mod-dav-svn.conf.mako b/rhodecode/apps/svn_support/templates/mod-dav-svn.conf.mako index f62ad6ca..0561aa7f 100644 --- a/rhodecode/apps/svn_support/templates/mod-dav-svn.conf.mako +++ b/rhodecode/apps/svn_support/templates/mod-dav-svn.conf.mako @@ -64,6 +64,9 @@ RequestHeader edit Destination ^https: http: early SVNParentPath "${parent_path_root|n}" SVNListParentPath ${"On" if svn_list_parent_path else "Off"|n} + # use specific SVN conf/authz file for each repository + #AuthzSVNReposRelativeAccessFile authz + Allow from all Order allow,deny @@ -82,6 +85,9 @@ RequestHeader edit Destination ^https: http: early SVNParentPath "${parent_path|n}" SVNListParentPath ${"On" if svn_list_parent_path else "Off"|n} + # use specific SVN conf/authz file for each repository + #AuthzSVNReposRelativeAccessFile authz + Allow from all Order allow,deny diff --git a/rhodecode/apps/user_group_profile/__init__.py b/rhodecode/apps/user_group_profile/__init__.py new file mode 100644 index 00000000..458ef58c --- /dev/null +++ b/rhodecode/apps/user_group_profile/__init__.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2016-2018 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + + +def includeme(config): + config.add_route( + name='user_group_profile', + pattern='/_profile_user_group/{user_group_name}') + # Scan module for configuration decorators. + config.scan('.views', ignore='.tests') diff --git a/rhodecode/apps/user_group_profile/tests/__init__.py b/rhodecode/apps/user_group_profile/tests/__init__.py new file mode 100644 index 00000000..d45eb3e0 --- /dev/null +++ b/rhodecode/apps/user_group_profile/tests/__init__.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2016-2018 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ diff --git a/rhodecode/apps/user_group_profile/tests/test_user_group.py b/rhodecode/apps/user_group_profile/tests/test_user_group.py new file mode 100644 index 00000000..fe8be403 --- /dev/null +++ b/rhodecode/apps/user_group_profile/tests/test_user_group.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2010-2018 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ +from rhodecode.model.user_group import UserGroupModel +from rhodecode.tests import ( + TestController, TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS) +from rhodecode.tests.fixture import Fixture +from rhodecode.tests.utils import AssertResponse + +fixture = Fixture() + + +def route_path(name, **kwargs): + return '/_profile_user_group/{user_group_name}'.format(**kwargs) + + +class TestUsersController(TestController): + + def test_user_group_profile(self, user_util): + self.log_user(TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS) + user, usergroup = user_util.create_user_with_group() + + response = self.app.get(route_path('profile_user_group', user_group_name=usergroup.users_group_name)) + response.mustcontain(usergroup.users_group_name) + response.mustcontain(user.username) + + def test_user_can_check_own_group(self, user_util): + user = user_util.create_user( + TEST_USER_REGULAR_LOGIN, password=TEST_USER_REGULAR_PASS, email='testme@rhodecode.org') + self.log_user(TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS) + usergroup = user_util.create_user_group(owner=user) + response = self.app.get(route_path('profile_user_group', user_group_name=usergroup.users_group_name)) + response.mustcontain(usergroup.users_group_name) + response.mustcontain(user.username) + + def test_user_can_not_check_other_group(self, user_util): + self.log_user(TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS) + user_group = user_util.create_user_group() + UserGroupModel().grant_user_permission(user_group, self._get_logged_user(), 'usergroup.none') + response = self.app.get(route_path('profile_user_group', user_group_name=user_group.users_group_name), status=404) + assert response.status_code == 404 + + def test_another_user_can_check_if_he_is_in_group(self, user_util): + self.log_user(TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS) + user = user_util.create_user( + 'test-my-user', password='qweqwe', email='testme@rhodecode.org') + user_group = user_util.create_user_group() + UserGroupModel().add_user_to_group(user_group, user) + UserGroupModel().grant_user_permission(user_group, self._get_logged_user(), 'usergroup.read') + response = self.app.get(route_path('profile_user_group', user_group_name=user_group.users_group_name)) + response.mustcontain(user_group.users_group_name) + response.mustcontain(user.username) + + def test_with_anonymous_user(self, user_util): + user = user_util.create_user( + 'test-my-user', password='qweqwe', email='testme@rhodecode.org') + user_group = user_util.create_user_group() + UserGroupModel().add_user_to_group(user_group, user) + response = self.app.get(route_path('profile_user_group', user_group_name=user_group.users_group_name), status=302) + assert response.status_code == 302 \ No newline at end of file diff --git a/rhodecode/apps/user_group_profile/views.py b/rhodecode/apps/user_group_profile/views.py new file mode 100644 index 00000000..6a6aa150 --- /dev/null +++ b/rhodecode/apps/user_group_profile/views.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2016-2018 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +import logging + +from pyramid.httpexceptions import HTTPNotFound +from pyramid.view import view_config + +from rhodecode.apps._base import BaseAppView +from rhodecode.lib.auth import HasUserGroupPermissionAnyDecorator, LoginRequired, NotAnonymous +from rhodecode.model.db import UserGroup, User + + +log = logging.getLogger(__name__) + + +class UserGroupProfileView(BaseAppView): + + @LoginRequired() + @NotAnonymous() + @HasUserGroupPermissionAnyDecorator('usergroup.read', 'usergroup.write', 'usergroup.admin',) + @view_config( + route_name='user_group_profile', request_method='GET', + renderer='rhodecode:templates/user_group/user_group.mako') + def user_group_profile(self): + c = self._get_local_tmpl_context() + c.active = 'profile' + self.db_user_group_name = self.request.matchdict.get('user_group_name') + c.user_group = UserGroup().get_by_group_name(self.db_user_group_name) + if not c.user_group: + raise HTTPNotFound() + group_members_obj = sorted((x.user for x in c.user_group.members), + key=lambda u: u.username.lower()) + c.group_members = group_members_obj + c.anonymous = self._rhodecode_user.username == User.DEFAULT_USER + return self._get_template_context(c) diff --git a/rhodecode/authentication/__init__.py b/rhodecode/authentication/__init__.py index e8f932a0..0aa51b48 100644 --- a/rhodecode/authentication/__init__.py +++ b/rhodecode/authentication/__init__.py @@ -18,6 +18,7 @@ # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ +import os import logging import importlib @@ -71,9 +72,12 @@ def _discover_legacy_plugins(config, prefix=legacy_plugin_prefix): setting in database which are using the specified prefix. Normally 'py:' is used for the legacy plugins. """ - auth_plugins = SettingsModel().get_setting_by_name('auth_plugins') - enabled_plugins = auth_plugins.app_settings_value - legacy_plugins = [id_ for id_ in enabled_plugins if id_.startswith(prefix)] + try: + auth_plugins = SettingsModel().get_setting_by_name('auth_plugins') + enabled_plugins = auth_plugins.app_settings_value + legacy_plugins = [id_ for id_ in enabled_plugins if id_.startswith(prefix)] + except Exception: + legacy_plugins = [] for plugin_id in legacy_plugins: log.debug('Legacy plugin discovered: "%s"', plugin_id) @@ -117,6 +121,11 @@ def includeme(config): route_name='auth_home', context=AuthnRootResource) + for key in ['RC_CMD_SETUP_RC', 'RC_CMD_UPGRADE_DB', 'RC_CMD_SSH_WRAPPER']: + if os.environ.get(key): + # skip this heavy step below on certain CLI commands + return + # Auto discover authentication plugins and include their configuration. _discover_plugins(config) _discover_legacy_plugins(config) diff --git a/rhodecode/authentication/base.py b/rhodecode/authentication/base.py index de1dd030..a6b1ba9a 100644 --- a/rhodecode/authentication/base.py +++ b/rhodecode/authentication/base.py @@ -171,11 +171,6 @@ class RhodeCodeAuthPluginBase(object): db_type = '{}.encrypted'.format(db_type) return db_type - @LazyProperty - def plugin_settings(self): - settings = SettingsModel().get_all_settings() - return settings - def is_enabled(self): """ Returns true if this plugin is enabled. An enabled plugin can be @@ -185,12 +180,13 @@ class RhodeCodeAuthPluginBase(object): auth_plugins = SettingsModel().get_auth_plugins() return self.get_id() in auth_plugins - def is_active(self): + def is_active(self, plugin_cached_settings=None): """ Returns true if the plugin is activated. An activated plugin is consulted during authentication, assumed it is also enabled. """ - return self.get_setting_by_name('enabled') + return self.get_setting_by_name( + 'enabled', plugin_cached_settings=plugin_cached_settings) def get_id(self): """ @@ -210,13 +206,24 @@ class RhodeCodeAuthPluginBase(object): """ return AuthnPluginSettingsSchemaBase() - def get_setting_by_name(self, name, default=None, cache=True): + def get_settings(self): + """ + Returns the plugin settings as dictionary. + """ + settings = {} + raw_settings = SettingsModel().get_all_settings() + for node in self.get_settings_schema(): + settings[node.name] = self.get_setting_by_name( + node.name, plugin_cached_settings=raw_settings) + return settings + + def get_setting_by_name(self, name, default=None, plugin_cached_settings=None): """ Returns a plugin setting by name. """ full_name = 'rhodecode_{}'.format(self._get_setting_full_name(name)) - if cache: - plugin_settings = self.plugin_settings + if plugin_cached_settings: + plugin_settings = plugin_cached_settings else: plugin_settings = SettingsModel().get_all_settings() @@ -235,15 +242,6 @@ class RhodeCodeAuthPluginBase(object): full_name, value, type_) return db_setting.app_settings_value - def get_settings(self): - """ - Returns the plugin settings as dictionary. - """ - settings = {} - for node in self.get_settings_schema(): - settings[node.name] = self.get_setting_by_name(node.name) - return settings - def log_safe_settings(self, settings): """ returns a log safe representation of settings, without any secrets @@ -625,12 +623,13 @@ def authenticate(username, password, environ=None, auth_type=None, 'headers plugin, skipping...', plugin.get_id()) continue + log.debug('Trying authentication using ** %s **', plugin.get_id()) + # load plugin settings from RhodeCode database plugin_settings = plugin.get_settings() plugin_sanitized_settings = plugin.log_safe_settings(plugin_settings) - log.debug('Plugin settings:%s', plugin_sanitized_settings) + log.debug('Plugin `%s` settings:%s', plugin.get_id(), plugin_sanitized_settings) - log.debug('Trying authentication using ** %s **', plugin.get_id()) # use plugin's method of user extraction. user = plugin.get_user(username, environ=environ, settings=plugin_settings) @@ -684,7 +683,8 @@ def authenticate(username, password, environ=None, auth_type=None, environ=environ or {}) if plugin_cache_active: - log.debug('Trying to fetch cached auth by `...%s`', _password_hash[:6]) + log.debug('Trying to fetch cached auth by pwd hash `...%s`', + _password_hash[:6]) plugin_user = cache_manager.get( _password_hash, createfunc=auth_func) else: diff --git a/rhodecode/authentication/plugins/auth_crowd.py b/rhodecode/authentication/plugins/auth_crowd.py index 5c75574f..74bab21e 100644 --- a/rhodecode/authentication/plugins/auth_crowd.py +++ b/rhodecode/authentication/plugins/auth_crowd.py @@ -281,5 +281,5 @@ class RhodeCodeAuthPlugin(RhodeCodeExternalAuthPlugin): if group in user_attrs["groups"]: user_attrs["admin"] = True log.debug("Final crowd user object: \n%s" % (formatted_json(user_attrs))) - log.info('user %s authenticated correctly' % user_attrs['username']) + log.info('user `%s` authenticated correctly' % user_attrs['username']) return user_attrs diff --git a/rhodecode/authentication/plugins/auth_jasig_cas.py b/rhodecode/authentication/plugins/auth_jasig_cas.py index c664ecf8..b1c36fa9 100644 --- a/rhodecode/authentication/plugins/auth_jasig_cas.py +++ b/rhodecode/authentication/plugins/auth_jasig_cas.py @@ -163,5 +163,5 @@ class RhodeCodeAuthPlugin(RhodeCodeExternalAuthPlugin): 'extern_type': extern_type, } - log.info('user %s authenticated correctly' % user_attrs['username']) + log.info('user `%s` authenticated correctly' % user_attrs['username']) return user_attrs diff --git a/rhodecode/authentication/plugins/auth_ldap.py b/rhodecode/authentication/plugins/auth_ldap.py index b095e569..3d32c719 100644 --- a/rhodecode/authentication/plugins/auth_ldap.py +++ b/rhodecode/authentication/plugins/auth_ldap.py @@ -22,10 +22,12 @@ RhodeCode authentication plugin for LDAP """ - +import socket +import re import colander import logging import traceback +import string from rhodecode.translation import _ from rhodecode.authentication.base import ( @@ -50,6 +52,9 @@ except ImportError: ldap = Missing +class LdapError(Exception): + pass + def plugin_factory(plugin_id, *args, **kwds): """ Factory function that is called during plugin discovery. @@ -86,6 +91,16 @@ class LdapSettingsSchema(AuthnPluginSettingsSchemaBase): title=_('Port'), validator=colander.Range(min=0, max=65536), widget='int') + + timeout = colander.SchemaNode( + colander.Int(), + default=60 * 5, + description=_('Timeout for LDAP connection'), + preparer=strip_whitespace, + title=_('Connection timeout'), + validator=colander.Range(min=1), + widget='int') + dn_user = colander.SchemaNode( colander.String(), default='', @@ -187,19 +202,47 @@ class LdapSettingsSchema(AuthnPluginSettingsSchemaBase): class AuthLdap(object): def _build_servers(self): + def host_resolver(host, port): + """ + Main work for this function is to prevent ldap connection issues, + and detect them early using a "greenified" sockets + """ + host = host.strip() + + log.info('Resolving LDAP host %s', host) + try: + ip = socket.gethostbyname(host) + log.info('Got LDAP server %s ip %s', host, ip) + except Exception: + raise LdapConnectionError( + 'Failed to resolve host: `{}`'.format(host)) + + log.info('Checking LDAP IP access %s', ip) + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + s.connect((ip, int(port))) + s.shutdown(socket.SHUT_RD) + except Exception: + raise LdapConnectionError( + 'Failed to connect to host: `{}:{}`'.format(host, port)) + + return '{}:{}'.format(host, port) + + port = self.LDAP_SERVER_PORT return ', '.join( - ["{}://{}:{}".format( - self.ldap_server_type, host.strip(), self.LDAP_SERVER_PORT) + ["{}://{}".format( + self.ldap_server_type, host_resolver(host, port)) for host in self.SERVER_ADDRESSES]) def __init__(self, server, base_dn, port=389, bind_dn='', bind_pass='', tls_kind='PLAIN', tls_reqcert='DEMAND', ldap_version=3, search_scope='SUBTREE', attr_login='uid', - ldap_filter=''): + ldap_filter='', timeout=None): if ldap == Missing: raise LdapImportError("Missing or incompatible ldap library") self.debug = False + self.timeout = timeout or 60 * 5 self.ldap_version = ldap_version self.ldap_server_type = 'ldap' @@ -226,34 +269,40 @@ class AuthLdap(object): self.BASE_DN = safe_str(base_dn) self.LDAP_FILTER = safe_str(ldap_filter) - def _get_ldap_server(self): + def _get_ldap_conn(self): + log.debug('initializing LDAP connection to:%s', self.LDAP_SERVER) + if self.debug: ldap.set_option(ldap.OPT_DEBUG_LEVEL, 255) - if hasattr(ldap, 'OPT_X_TLS_CACERTDIR'): - ldap.set_option(ldap.OPT_X_TLS_CACERTDIR, - '/etc/openldap/cacerts') - ldap.set_option(ldap.OPT_REFERRALS, ldap.OPT_OFF) - ldap.set_option(ldap.OPT_RESTART, ldap.OPT_ON) - ldap.set_option(ldap.OPT_NETWORK_TIMEOUT, 60 * 10) - ldap.set_option(ldap.OPT_TIMEOUT, 60 * 10) + if hasattr(ldap, 'OPT_X_TLS_CACERTDIR'): + ldap.set_option(ldap.OPT_X_TLS_CACERTDIR, '/etc/openldap/cacerts') if self.TLS_KIND != 'PLAIN': ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, self.TLS_REQCERT) - server = ldap.initialize(self.LDAP_SERVER) + + ldap.set_option(ldap.OPT_REFERRALS, ldap.OPT_OFF) + ldap.set_option(ldap.OPT_RESTART, ldap.OPT_ON) + + # init connection now + ldap_conn = ldap.initialize(self.LDAP_SERVER) + ldap_conn.set_option(ldap.OPT_NETWORK_TIMEOUT, self.timeout) + ldap_conn.set_option(ldap.OPT_TIMEOUT, self.timeout) + ldap_conn.timeout = self.timeout + if self.ldap_version == 2: - server.protocol = ldap.VERSION2 + ldap_conn.protocol = ldap.VERSION2 else: - server.protocol = ldap.VERSION3 + ldap_conn.protocol = ldap.VERSION3 if self.TLS_KIND == 'START_TLS': - server.start_tls_s() + ldap_conn.start_tls_s() if self.LDAP_BIND_DN and self.LDAP_BIND_PASS: log.debug('Trying simple_bind with password and given login DN: %s', self.LDAP_BIND_DN) - server.simple_bind_s(self.LDAP_BIND_DN, self.LDAP_BIND_PASS) + ldap_conn.simple_bind_s(self.LDAP_BIND_DN, self.LDAP_BIND_PASS) - return server + return ldap_conn def get_uid(self, username): uid = username @@ -295,13 +344,14 @@ class AuthLdap(object): if "," in username: raise LdapUsernameError( "invalid character `,` in username: `{}`".format(username)) + ldap_conn = None try: - server = self._get_ldap_server() + ldap_conn = self._get_ldap_conn() filter_ = '(&%s(%s=%s))' % ( self.LDAP_FILTER, self.attr_login, username) log.debug("Authenticating %r filter %s at %s", self.BASE_DN, filter_, self.LDAP_SERVER) - lobjects = server.search_ext_s( + lobjects = ldap_conn.search_ext_s( self.BASE_DN, self.SEARCH_SCOPE, filter_) if not lobjects: @@ -315,7 +365,7 @@ class AuthLdap(object): continue user_attrs = self.fetch_attrs_from_simple_bind( - server, dn, username, password) + ldap_conn, dn, username, password) if user_attrs: break @@ -333,6 +383,15 @@ class AuthLdap(object): raise LdapConnectionError( "LDAP can't access authentication " "server, org_exc:%s" % org_exc) + finally: + if ldap_conn: + log.debug('ldap: connection release') + try: + ldap_conn.unbind_s() + except Exception: + # for any reason this can raise exception we must catch it + # to not crush the server + pass return dn, user_attrs @@ -429,6 +488,7 @@ class RhodeCodeAuthPlugin(RhodeCodeExternalAuthPlugin): 'attr_login': settings.get('attr_login'), 'ldap_version': 3, 'ldap_filter': settings.get('filter'), + 'timeout': settings.get('timeout') } ldap_attrs = self.try_dynamic_binding(username, password, ldap_args) @@ -469,7 +529,7 @@ class RhodeCodeAuthPlugin(RhodeCodeExternalAuthPlugin): 'extern_type': extern_type, } log.debug('ldap user: %s', user_attrs) - log.info('user %s authenticated correctly', user_attrs['username']) + log.info('user `%s` authenticated correctly', user_attrs['username']) return user_attrs @@ -479,3 +539,4 @@ class RhodeCodeAuthPlugin(RhodeCodeExternalAuthPlugin): except (Exception,): log.exception("Other exception") return None + diff --git a/rhodecode/authentication/plugins/auth_pam.py b/rhodecode/authentication/plugins/auth_pam.py index f851e2bc..e91330c0 100644 --- a/rhodecode/authentication/plugins/auth_pam.py +++ b/rhodecode/authentication/plugins/auth_pam.py @@ -157,5 +157,5 @@ class RhodeCodeAuthPlugin(RhodeCodeExternalAuthPlugin): pass log.debug("pamuser: %s", user_attrs) - log.info('user %s authenticated correctly' % user_attrs['username']) + log.info('user `%s` authenticated correctly' % user_attrs['username']) return user_attrs diff --git a/rhodecode/authentication/plugins/auth_rhodecode.py b/rhodecode/authentication/plugins/auth_rhodecode.py index 54a77bbb..83659307 100644 --- a/rhodecode/authentication/plugins/auth_rhodecode.py +++ b/rhodecode/authentication/plugins/auth_rhodecode.py @@ -127,14 +127,14 @@ class RhodeCodeAuthPlugin(RhodeCodeAuthPluginBase): if userobj.username == User.DEFAULT_USER and userobj.active: log.info( - 'user %s authenticated correctly as anonymous user', userobj) + 'user `%s` authenticated correctly as anonymous user', userobj.username) return user_attrs elif userobj.username == username and password_match: - log.info('user %s authenticated correctly', userobj) + log.info('user `%s` authenticated correctly', userobj.username) return user_attrs - log.info("user %s had a bad password when " - "authenticating on this plugin", userobj) + log.warn("user `%s` used a wrong password when " + "authenticating on this plugin", userobj.username) return None else: log.warning( diff --git a/rhodecode/authentication/plugins/auth_token.py b/rhodecode/authentication/plugins/auth_token.py index 1c0fdf35..995e9c1c 100644 --- a/rhodecode/authentication/plugins/auth_token.py +++ b/rhodecode/authentication/plugins/auth_token.py @@ -137,7 +137,7 @@ class RhodeCodeAuthPlugin(RhodeCodeAuthPluginBase): 'user `%s` successfully authenticated via %s', user_attrs['username'], self.name) return user_attrs - log.error( + log.warn( 'user `%s` failed to authenticate via %s, reason: bad or ' 'inactive token.', username, self.name) else: diff --git a/rhodecode/authentication/registry.py b/rhodecode/authentication/registry.py index 1b77ff1b..11900d43 100644 --- a/rhodecode/authentication/registry.py +++ b/rhodecode/authentication/registry.py @@ -70,9 +70,11 @@ class AuthenticationPluginRegistry(object): # Add all enabled and active plugins to the list. We iterate over the # auth_plugins setting from DB because it also represents the ordering. enabled_plugins = SettingsModel().get_auth_plugins() + raw_settings = SettingsModel().get_all_settings() for plugin_id in enabled_plugins: plugin = self.get_plugin(plugin_id) - if plugin is not None and plugin.is_active(): + if plugin is not None and plugin.is_active( + plugin_cached_settings=raw_settings): plugins.append(plugin) # Add the fallback plugin from ini file. diff --git a/rhodecode/authentication/views.py b/rhodecode/authentication/views.py index 620af0be..570cd06c 100644 --- a/rhodecode/authentication/views.py +++ b/rhodecode/authentication/views.py @@ -63,7 +63,7 @@ class AuthnPluginViewBase(BaseAppView): for node in schema: if node.name not in defaults: defaults[node.name] = self.plugin.get_setting_by_name( - node.name, node.default, cache=False) + node.name, node.default) template_context = { 'defaults': defaults, @@ -145,7 +145,7 @@ class AuthSettingsView(BaseAppView): # Create form default values and fill the form. form_defaults = { - 'auth_plugins': ','.join(enabled_plugins) + 'auth_plugins': ',\n'.join(enabled_plugins) } form_defaults.update(defaults) html = formencode.htmlfill.render( diff --git a/rhodecode/config/hook_templates/git_post_receive.py.tmpl b/rhodecode/config/hook_templates/git_post_receive.py.tmpl deleted file mode 100644 index ed13d65b..00000000 --- a/rhodecode/config/hook_templates/git_post_receive.py.tmpl +++ /dev/null @@ -1,46 +0,0 @@ -#!_ENV_ -import os -import sys - -try: - from vcsserver import hooks -except ImportError: - if os.environ.get('RC_DEBUG_GIT_HOOK'): - import traceback - print traceback.format_exc() - hooks = None - - -RC_HOOK_VER = '_TMPL_' - - -def main(): - if hooks is None: - # exit with success if we cannot import vcsserver.hooks !! - # this allows simply push to this repo even without rhodecode - sys.exit(0) - - if os.environ.get('RC_SKIP_HOOKS'): - sys.exit(0) - - repo_path = os.getcwd() - push_data = sys.stdin.readlines() - os.environ['RC_HOOK_VER'] = RC_HOOK_VER - # os.environ is modified here by a subprocess call that - # runs git and later git executes this hook. - # Environ gets some additional info from rhodecode system - # like IP or username from basic-auth - try: - result = hooks.git_post_receive(repo_path, push_data, os.environ) - sys.exit(result) - except Exception as error: - # TODO: johbo: Improve handling of this special case - if not getattr(error, '_vcs_kind', None) == 'repo_locked': - raise - print 'ERROR:', error - sys.exit(1) - sys.exit(0) - - -if __name__ == '__main__': - main() diff --git a/rhodecode/config/hook_templates/git_pre_receive.py.tmpl b/rhodecode/config/hook_templates/git_pre_receive.py.tmpl deleted file mode 100644 index 29596eea..00000000 --- a/rhodecode/config/hook_templates/git_pre_receive.py.tmpl +++ /dev/null @@ -1,46 +0,0 @@ -#!_ENV_ -import os -import sys - -try: - from vcsserver import hooks -except ImportError: - if os.environ.get('RC_DEBUG_GIT_HOOK'): - import traceback - print traceback.format_exc() - hooks = None - - -RC_HOOK_VER = '_TMPL_' - - -def main(): - if hooks is None: - # exit with success if we cannot import vcsserver.hooks !! - # this allows simply push to this repo even without rhodecode - sys.exit(0) - - if os.environ.get('RC_SKIP_HOOKS'): - sys.exit(0) - - repo_path = os.getcwd() - push_data = sys.stdin.readlines() - os.environ['RC_HOOK_VER'] = RC_HOOK_VER - # os.environ is modified here by a subprocess call that - # runs git and later git executes this hook. - # Environ gets some additional info from rhodecode system - # like IP or username from basic-auth - try: - result = hooks.git_pre_receive(repo_path, push_data, os.environ) - sys.exit(result) - except Exception as error: - # TODO: johbo: Improve handling of this special case - if not getattr(error, '_vcs_kind', None) == 'repo_locked': - raise - print 'ERROR:', error - sys.exit(1) - sys.exit(0) - - -if __name__ == '__main__': - main() diff --git a/rhodecode/config/hook_templates/svn_post_commit_hook.py.tmpl b/rhodecode/config/hook_templates/svn_post_commit_hook.py.tmpl deleted file mode 100644 index 06a370fa..00000000 --- a/rhodecode/config/hook_templates/svn_post_commit_hook.py.tmpl +++ /dev/null @@ -1,14 +0,0 @@ -#!_ENV_ - -import sys - - -RC_HOOK_VER = '_TMPL_' - - -def main(): - sys.exit(0) - - -if __name__ == '__main__': - main() diff --git a/rhodecode/config/hook_templates/svn_pre_commit_hook.py.tmpl b/rhodecode/config/hook_templates/svn_pre_commit_hook.py.tmpl deleted file mode 100644 index 9e325e2c..00000000 --- a/rhodecode/config/hook_templates/svn_pre_commit_hook.py.tmpl +++ /dev/null @@ -1,19 +0,0 @@ -#!_ENV_ - -import os -import sys - - -RC_HOOK_VER = '_TMPL_' - - -def main(): - if os.environ.get('SSH_READ_ONLY') == '1': - sys.stderr.write('Only read-only access is allowed') - sys.exit(1) - - sys.exit(0) - - -if __name__ == '__main__': - main() diff --git a/rhodecode/config/middleware.py b/rhodecode/config/middleware.py index 433eda16..5f9a9ece 100644 --- a/rhodecode/config/middleware.py +++ b/rhodecode/config/middleware.py @@ -226,6 +226,7 @@ def includeme(config): config.include('rhodecode.apps.user_group') config.include('rhodecode.apps.search') config.include('rhodecode.apps.user_profile') + config.include('rhodecode.apps.user_group_profile') config.include('rhodecode.apps.my_account') config.include('rhodecode.apps.svn_support') config.include('rhodecode.apps.ssh_support') diff --git a/rhodecode/events/pullrequest.py b/rhodecode/events/pullrequest.py index f26993db..e473c749 100644 --- a/rhodecode/events/pullrequest.py +++ b/rhodecode/events/pullrequest.py @@ -37,6 +37,7 @@ class PullRequestEvent(RepoEvent): self.pullrequest = pullrequest def as_dict(self): + from rhodecode.lib.utils2 import md5_safe from rhodecode.model.pull_request import PullRequestModel data = super(PullRequestEvent, self).as_dict() @@ -46,6 +47,9 @@ class PullRequestEvent(RepoEvent): repos=[self.pullrequest.source_repo] ) issues = _issues_as_dict(commits) + # calculate hashes of all commits for unique identifier of commits + # inside that pull request + commits_hash = md5_safe(':'.join(x.get('raw_id', '') for x in commits)) data.update({ 'pullrequest': { @@ -56,7 +60,10 @@ class PullRequestEvent(RepoEvent): self.pullrequest, request=self.request), 'permalink_url': PullRequestModel().get_url( self.pullrequest, request=self.request, permalink=True), + 'shadow_url': PullRequestModel().get_shadow_clone_url( + self.pullrequest, request=self.request), 'status': self.pullrequest.calculated_review_status(), + 'commits_uid': commits_hash, 'commits': commits, } }) diff --git a/rhodecode/events/repo.py b/rhodecode/events/repo.py index 0751d23b..096be277 100644 --- a/rhodecode/events/repo.py +++ b/rhodecode/events/repo.py @@ -132,7 +132,8 @@ def _commits_as_dict(event, commit_ids, repos): missing_commits = set(commit_ids) - set(c['raw_id'] for c in commits) if missing_commits: - log.error('missing commits: %s' % ', '.join(missing_commits)) + log.error('Inconsistent repository state. ' + 'Missing commits: %s' % ', '.join(missing_commits)) return commits diff --git a/rhodecode/integrations/__init__.py b/rhodecode/integrations/__init__.py index a4f84efa..191e76ab 100644 --- a/rhodecode/integrations/__init__.py +++ b/rhodecode/integrations/__init__.py @@ -45,6 +45,8 @@ integration_type_registry.register_integration_type( base.EEIntegration('Jira Issues integration', 'jira')) integration_type_registry.register_integration_type( base.EEIntegration('Redmine Tracker integration', 'redmine')) +integration_type_registry.register_integration_type( + base.EEIntegration('Jenkins CI integration', 'jenkins')) def integrations_event_handler(event): diff --git a/rhodecode/integrations/types/base.py b/rhodecode/integrations/types/base.py index e2882389..4eb68e55 100644 --- a/rhodecode/integrations/types/base.py +++ b/rhodecode/integrations/types/base.py @@ -19,74 +19,86 @@ # and proprietary license terms, please see https://rhodecode.com/licenses/ import colander +import string +import collections +import logging + +from mako import exceptions + from rhodecode.translation import _ +log = logging.getLogger(__name__) + + class IntegrationTypeBase(object): """ Base class for IntegrationType plugins """ is_dummy = False description = '' - icon = ''' - - - - - - image/svg+xml - - - - - - - - - - -''' + + @classmethod + def icon(cls): + return ''' + + + + + + image/svg+xml + + + + + + + + + + + ''' def __init__(self, settings): """ @@ -108,3 +120,192 @@ class EEIntegration(IntegrationTypeBase): def __init__(self, name, key, settings=None): self.display_name = name self.key = key + super(EEIntegration, self).__init__(settings) + + +# Helpers # +WEBHOOK_URL_VARS = [ + ('event_name', 'Unique name of the event type, e.g pullrequest-update'), + ('repo_name', 'Full name of the repository'), + ('repo_type', 'VCS type of repository'), + ('repo_id', 'Unique id of repository'), + ('repo_url', 'Repository url'), + # extra repo fields + ('extra:', 'Extra repo variables, read from its settings.'), + + # special attrs below that we handle, using multi-call + ('branch', 'Name of each brach submitted, if any.'), + ('commit_id', 'Id of each commit submitted, if any.'), + + # pr events vars + ('pull_request_id', 'Unique ID of the pull request.'), + ('pull_request_title', 'Title of the pull request.'), + ('pull_request_url', 'Pull request url.'), + ('pull_request_shadow_url', 'Pull request shadow repo clone url.'), + ('pull_request_commits_uid', 'Calculated UID of all commits inside the PR. ' + 'Changes after PR update'), + + # user who triggers the call + ('username', 'User who triggered the call.'), + ('user_id', 'User id who triggered the call.'), +] + +# common vars for url template used for CI plugins. Shared with webhook +CI_URL_VARS = WEBHOOK_URL_VARS + + +class CommitParsingDataHandler(object): + + def aggregate_branch_data(self, branches, commits): + branch_data = collections.OrderedDict() + for obj in branches: + branch_data[obj['name']] = obj + + branches_commits = collections.OrderedDict() + for commit in commits: + if commit.get('git_ref_change'): + # special case for GIT that allows creating tags, + # deleting branches without associated commit + continue + commit_branch = commit['branch'] + + if commit_branch not in branches_commits: + _branch = branch_data[commit_branch] \ + if commit_branch else commit_branch + branch_commits = {'branch': _branch, + 'commits': []} + branches_commits[commit_branch] = branch_commits + + branch_commits = branches_commits[commit_branch] + branch_commits['commits'].append(commit) + return branches_commits + + +class WebhookDataHandler(CommitParsingDataHandler): + name = 'webhook' + + def __init__(self, template_url, headers): + self.template_url = template_url + self.headers = headers + + def get_base_parsed_template(self, data): + """ + initially parses the passed in template with some common variables + available on ALL calls + """ + # note: make sure to update the `WEBHOOK_URL_VARS` if this changes + common_vars = { + 'repo_name': data['repo']['repo_name'], + 'repo_type': data['repo']['repo_type'], + 'repo_id': data['repo']['repo_id'], + 'repo_url': data['repo']['url'], + 'username': data['actor']['username'], + 'user_id': data['actor']['user_id'], + 'event_name': data['name'] + } + + extra_vars = {} + for extra_key, extra_val in data['repo']['extra_fields'].items(): + extra_vars['extra__{}'.format(extra_key)] = extra_val + common_vars.update(extra_vars) + + template_url = self.template_url.replace('${extra:', '${extra__') + return string.Template(template_url).safe_substitute(**common_vars) + + def repo_push_event_handler(self, event, data): + url = self.get_base_parsed_template(data) + url_cals = [] + + branches_commits = self.aggregate_branch_data( + data['push']['branches'], data['push']['commits']) + if '${branch}' in url: + # call it multiple times, for each branch if used in variables + for branch, commit_ids in branches_commits.items(): + branch_url = string.Template(url).safe_substitute(branch=branch) + # call further down for each commit if used + if '${commit_id}' in branch_url: + for commit_data in commit_ids['commits']: + commit_id = commit_data['raw_id'] + commit_url = string.Template(branch_url).safe_substitute( + commit_id=commit_id) + # register per-commit call + log.debug( + 'register %s call(%s) to url %s', + self.name, event, commit_url) + url_cals.append( + (commit_url, self.headers, data)) + + else: + # register per-branch call + log.debug( + 'register %s call(%s) to url %s', + self.name, event, branch_url) + url_cals.append( + (branch_url, self.headers, data)) + + else: + log.debug( + 'register %s call(%s) to url %s', self.name, event, url) + url_cals.append((url, self.headers, data)) + + return url_cals + + def repo_create_event_handler(self, event, data): + url = self.get_base_parsed_template(data) + log.debug( + 'register %s call(%s) to url %s', self.name, event, url) + return [(url, self.headers, data)] + + def pull_request_event_handler(self, event, data): + url = self.get_base_parsed_template(data) + log.debug( + 'register %s call(%s) to url %s', self.name, event, url) + url = string.Template(url).safe_substitute( + pull_request_id=data['pullrequest']['pull_request_id'], + pull_request_title=data['pullrequest']['title'], + pull_request_url=data['pullrequest']['url'], + pull_request_shadow_url=data['pullrequest']['shadow_url'], + pull_request_commits_uid=data['pullrequest']['commits_uid'], + ) + return [(url, self.headers, data)] + + def __call__(self, event, data): + from rhodecode import events + + if isinstance(event, events.RepoPushEvent): + return self.repo_push_event_handler(event, data) + elif isinstance(event, events.RepoCreateEvent): + return self.repo_create_event_handler(event, data) + elif isinstance(event, events.PullRequestEvent): + return self.pull_request_event_handler(event, data) + else: + raise ValueError( + 'event type `%s` not in supported list: %s' % ( + event.__class__, events)) + + +def get_auth(settings): + from requests.auth import HTTPBasicAuth + username = settings.get('username') + password = settings.get('password') + if username and password: + return HTTPBasicAuth(username, password) + return None + + +def get_web_token(settings): + return settings['secret_token'] + + +def get_url_vars(url_vars): + return '\n'.join( + '{} - {}'.format('${' + key + '}', explanation) + for key, explanation in url_vars) + + +def render_with_traceback(template, *args, **kwargs): + try: + return template.render(*args, **kwargs) + except Exception: + log.error(exceptions.text_error_template().render()) + raise diff --git a/rhodecode/integrations/types/email.py b/rhodecode/integrations/types/email.py index 1c363d4b..e0ad17ba 100644 --- a/rhodecode/integrations/types/email.py +++ b/rhodecode/integrations/types/email.py @@ -29,7 +29,8 @@ from rhodecode import events from rhodecode.translation import _ from rhodecode.lib.celerylib import run_task from rhodecode.lib.celerylib import tasks -from rhodecode.integrations.types.base import IntegrationTypeBase +from rhodecode.integrations.types.base import ( + IntegrationTypeBase, render_with_traceback) log = logging.getLogger(__name__) @@ -127,11 +128,15 @@ repo_push_template_html = Template(''' - % for commit in data['push']['commits']: - ${commit['short_id']} by ${commit['author']} at ${commit['date']}
- ${commit['message_html']}
-
- % endfor + % if data['push']['commits']: + % for commit in data['push']['commits']: + ${commit['short_id']} by ${commit['author']} at ${commit['date']}
+ ${commit['message_html']}
+
+ % endfor + % else: + No commit data + % endif @@ -146,7 +151,34 @@ repo_push_template_html = Template(''' ''') -email_icon = ''' + +class EmailSettingsSchema(colander.Schema): + @colander.instantiate(validator=colander.Length(min=1)) + class recipients(colander.SequenceSchema): + title = _('Recipients') + description = _('Email addresses to send push events to') + widget = deform.widget.SequenceWidget(min_len=1) + + recipient = colander.SchemaNode( + colander.String(), + title=_('Email address'), + description=_('Email address'), + default='', + validator=colander.Email(), + widget=deform.widget.TextInputWidget( + placeholder='user@domain.com', + ), + ) + + +class EmailIntegrationType(IntegrationTypeBase): + key = 'email' + display_name = _('Email') + description = _('Send repo push summaries to a list of recipients via email') + + @classmethod + def icon(cls): + return ''' ''' - -class EmailSettingsSchema(colander.Schema): - @colander.instantiate(validator=colander.Length(min=1)) - class recipients(colander.SequenceSchema): - title = _('Recipients') - description = _('Email addresses to send push events to') - widget = deform.widget.SequenceWidget(min_len=1) - - recipient = colander.SchemaNode( - colander.String(), - title=_('Email address'), - description=_('Email address'), - default='', - validator=colander.Email(), - widget=deform.widget.TextInputWidget( - placeholder='user@domain.com', - ), - ) - - -class EmailIntegrationType(IntegrationTypeBase): - key = 'email' - display_name = _('Email') - description = _('Send repo push summaries to a list of recipients via email') - icon = email_icon - def settings_schema(self): schema = EmailSettingsSchema() return schema @@ -276,12 +282,14 @@ def repo_push_handler(data, settings): branches=', '.join( branch['name'] for branch in data['push']['branches'])) - email_body_plaintext = repo_push_template_plaintext.render( + email_body_plaintext = render_with_traceback( + repo_push_template_plaintext, data=data, subject=subject, instance_url=server_url) - email_body_html = repo_push_template_html.render( + email_body_html = render_with_traceback( + repo_push_template_html, data=data, subject=subject, instance_url=server_url) diff --git a/rhodecode/integrations/types/hipchat.py b/rhodecode/integrations/types/hipchat.py index d6e09cf0..0a0412e0 100644 --- a/rhodecode/integrations/types/hipchat.py +++ b/rhodecode/integrations/types/hipchat.py @@ -24,14 +24,14 @@ import logging import requests import colander import textwrap -from collections import OrderedDict from mako.template import Template from rhodecode import events from rhodecode.translation import _ from rhodecode.lib import helpers as h from rhodecode.lib.celerylib import run_task, async_task, RequestContextTask from rhodecode.lib.colander_utils import strip_whitespace -from rhodecode.integrations.types.base import IntegrationTypeBase +from rhodecode.integrations.types.base import ( + IntegrationTypeBase, CommitParsingDataHandler, render_with_traceback) log = logging.getLogger(__name__) @@ -81,23 +81,31 @@ repo_push_template = Template('''
    %for branch, branch_commits in branches_commits.items():
  • + % if branch: branch: ${branch_commits['branch']['name']} + % else: + to trunk + % endif
      - %for commit in branch_commits['commits']: + % for commit in branch_commits['commits']:
    • ${commit['short_id']} - ${commit['message_html']}
      • - %endfor + % endfor
    • %endfor ''') -class HipchatIntegrationType(IntegrationTypeBase): +class HipchatIntegrationType(IntegrationTypeBase, CommitParsingDataHandler): key = 'hipchat' display_name = _('Hipchat') description = _('Send events such as repo pushes and pull requests to ' 'your hipchat channel.') - icon = '''''' + + @classmethod + def icon(cls): + return '''''' + valid_events = [ events.PullRequestCloseEvent, events.PullRequestMergeEvent, @@ -213,20 +221,11 @@ class HipchatIntegrationType(IntegrationTypeBase): ) def format_repo_push_event(self, data): - branch_data = {branch['name']: branch - for branch in data['push']['branches']} + branches_commits = self.aggregate_branch_data( + data['push']['branches'], data['push']['commits']) - branches_commits = OrderedDict() - for commit in data['push']['commits']: - if commit['branch'] not in branches_commits: - branch_commits = {'branch': branch_data[commit['branch']], - 'commits': []} - branches_commits[commit['branch']] = branch_commits - - branch_commits = branches_commits[commit['branch']] - branch_commits['commits'].append(commit) - - result = repo_push_template.render( + result = render_with_traceback( + repo_push_template, data=data, branches_commits=branches_commits, ) diff --git a/rhodecode/integrations/types/slack.py b/rhodecode/integrations/types/slack.py index 50dd0508..2aa72efb 100644 --- a/rhodecode/integrations/types/slack.py +++ b/rhodecode/integrations/types/slack.py @@ -28,14 +28,14 @@ import deform import requests import colander from mako.template import Template -from collections import OrderedDict from rhodecode import events from rhodecode.translation import _ from rhodecode.lib import helpers as h from rhodecode.lib.celerylib import run_task, async_task, RequestContextTask from rhodecode.lib.colander_utils import strip_whitespace -from rhodecode.integrations.types.base import IntegrationTypeBase +from rhodecode.integrations.types.base import ( + IntegrationTypeBase, CommitParsingDataHandler, render_with_traceback) log = logging.getLogger(__name__) @@ -87,12 +87,16 @@ class SlackSettingsSchema(colander.Schema): ) -class SlackIntegrationType(IntegrationTypeBase): +class SlackIntegrationType(IntegrationTypeBase, CommitParsingDataHandler): key = 'slack' display_name = _('Slack') description = _('Send events such as repo pushes and pull requests to ' 'your slack channel.') - icon = '''''' + + @classmethod + def icon(cls): + return '''''' + valid_events = [ events.PullRequestCloseEvent, events.PullRequestMergeEvent, @@ -190,32 +194,39 @@ class SlackIntegrationType(IntegrationTypeBase): } ] - title = Template(textwrap.dedent(r''' + template = Template(textwrap.dedent(r''' *${data['actor']['username']}* left ${data['comment']['type']} on pull request <${data['pullrequest']['url']}|#${data['pullrequest']['pull_request_id']}>: - ''')).render(data=data, comment=event.comment) + ''')) + title = render_with_traceback( + template, data=data, comment=event.comment) - text = Template(textwrap.dedent(r''' + template = Template(textwrap.dedent(r''' *pull request title*: ${pr_title} % if status_text: *submitted status*: `${status_text}` % endif >>> ${comment_text} - ''')).render(comment_text=comment_text, - pr_title=data['pullrequest']['title'], - status_text=status_text) + ''')) + text = render_with_traceback( + template, + comment_text=comment_text, + pr_title=data['pullrequest']['title'], + status_text=status_text) return title, text, fields, overrides def format_pull_request_review_event(self, event, data): - title = Template(textwrap.dedent(r''' + template = Template(textwrap.dedent(r''' *${data['actor']['username']}* changed status of pull request <${data['pullrequest']['url']}|#${data['pullrequest']['pull_request_id']} to `${data['pullrequest']['status']}`>: - ''')).render(data=data) + ''')) + title = render_with_traceback(template, data=data) - text = Template(textwrap.dedent(r''' + template = Template(textwrap.dedent(r''' *pull request title*: ${pr_title} - ''')).render( - pr_title=data['pullrequest']['title'], - ) + ''')) + text = render_with_traceback( + template, + pr_title=data['pullrequest']['title']) return title, text @@ -227,50 +238,53 @@ class SlackIntegrationType(IntegrationTypeBase): events.PullRequestCreateEvent: 'created', }.get(event.__class__, str(event.__class__)) - title = Template(textwrap.dedent(r''' + template = Template(textwrap.dedent(r''' *${data['actor']['username']}* `${action}` pull request <${data['pullrequest']['url']}|#${data['pullrequest']['pull_request_id']}>: - ''')).render(data=data, action=action) + ''')) + title = render_with_traceback(template, data=data, action=action) - text = Template(textwrap.dedent(r''' + template = Template(textwrap.dedent(r''' *pull request title*: ${pr_title} %if data['pullrequest']['commits']: *commits*: ${len(data['pullrequest']['commits'])} %endif - ''')).render( + ''')) + text = render_with_traceback( + template, pr_title=data['pullrequest']['title'], - data=data - ) + data=data) return title, text def format_repo_push_event(self, data): - branch_data = {branch['name']: branch - for branch in data['push']['branches']} - branches_commits = OrderedDict() - for commit in data['push']['commits']: - if commit['branch'] not in branches_commits: - branch_commits = {'branch': branch_data[commit['branch']], - 'commits': []} - branches_commits[commit['branch']] = branch_commits + branches_commits = self.aggregate_branch_data( + data['push']['branches'], data['push']['commits']) - branch_commits = branches_commits[commit['branch']] - branch_commits['commits'].append(commit) - - title = Template(r''' + template = Template(r''' *${data['actor']['username']}* pushed to repo <${data['repo']['url']}|${data['repo']['repo_name']}>: - ''').render(data=data) + ''') + title = render_with_traceback(template, data=data) repo_push_template = Template(textwrap.dedent(r''' - %for branch, branch_commits in branches_commits.items(): - ${len(branch_commits['commits'])} ${'commit' if len(branch_commits['commits']) == 1 else 'commits'} on branch: <${branch_commits['branch']['url']}|${branch_commits['branch']['name']}> - %for commit in branch_commits['commits']: + <% + def branch_text(branch): + if branch: + return 'on branch: <{}|{}>'.format(branch_commits['branch']['url'], branch_commits['branch']['name']) + else: + ## case for SVN no branch push... + return 'to trunk' + %> \ + % for branch, branch_commits in branches_commits.items(): + ${len(branch_commits['commits'])} ${'commit' if len(branch_commits['commits']) == 1 else 'commits'} ${branch_text(branch)} + % for commit in branch_commits['commits']: `<${commit['url']}|${commit['short_id']}>` - ${commit['message_html']|html_to_slack_links} - %endfor - %endfor + % endfor + % endfor ''')) - text = repo_push_template.render( + text = render_with_traceback( + repo_push_template, data=data, branches_commits=branches_commits, html_to_slack_links=html_to_slack_links, @@ -279,14 +293,16 @@ class SlackIntegrationType(IntegrationTypeBase): return title, text def format_repo_create_event(self, data): - title = Template(r''' + template = Template(r''' *${data['actor']['username']}* created new repository ${data['repo']['repo_name']}: - ''').render(data=data) + ''') + title = render_with_traceback(template, data=data) - text = Template(textwrap.dedent(r''' + template = Template(textwrap.dedent(r''' repo_url: ${data['repo']['url']} repo_type: ${data['repo']['repo_type']} - ''')).render(data=data) + ''')) + text = render_with_traceback(template, data=data) return title, text diff --git a/rhodecode/integrations/types/webhook.py b/rhodecode/integrations/types/webhook.py index 6d385e3c..f2c5e400 100644 --- a/rhodecode/integrations/types/webhook.py +++ b/rhodecode/integrations/types/webhook.py @@ -19,8 +19,6 @@ # and proprietary license terms, please see https://rhodecode.com/licenses/ from __future__ import unicode_literals -import string -from collections import OrderedDict import deform import deform.widget @@ -33,149 +31,18 @@ from requests.packages.urllib3.util.retry import Retry import rhodecode from rhodecode import events from rhodecode.translation import _ -from rhodecode.integrations.types.base import IntegrationTypeBase +from rhodecode.integrations.types.base import ( + IntegrationTypeBase, get_auth, get_web_token, get_url_vars, + WebhookDataHandler, WEBHOOK_URL_VARS) from rhodecode.lib.celerylib import run_task, async_task, RequestContextTask +from rhodecode.model.validation_schema import widgets log = logging.getLogger(__name__) # updating this required to update the `common_vars` passed in url calling func -WEBHOOK_URL_VARS = [ - 'repo_name', - 'repo_type', - 'repo_id', - 'repo_url', - # extra repo fields - 'extra:', - # special attrs below that we handle, using multi-call - 'branch', - 'commit_id', - - # pr events vars - 'pull_request_id', - 'pull_request_url', - - # user who triggers the call - 'username', - 'user_id', - -] -URL_VARS = ', '.join('${' + x + '}' for x in WEBHOOK_URL_VARS) - - -def get_auth(settings): - from requests.auth import HTTPBasicAuth - username = settings.get('username') - password = settings.get('password') - if username and password: - return HTTPBasicAuth(username, password) - return None - - -class WebhookHandler(object): - def __init__(self, template_url, secret_token, headers): - self.template_url = template_url - self.secret_token = secret_token - self.headers = headers - - def get_base_parsed_template(self, data): - """ - initially parses the passed in template with some common variables - available on ALL calls - """ - # note: make sure to update the `WEBHOOK_URL_VARS` if this changes - common_vars = { - 'repo_name': data['repo']['repo_name'], - 'repo_type': data['repo']['repo_type'], - 'repo_id': data['repo']['repo_id'], - 'repo_url': data['repo']['url'], - 'username': data['actor']['username'], - 'user_id': data['actor']['user_id'] - } - - extra_vars = {} - for extra_key, extra_val in data['repo']['extra_fields'].items(): - extra_vars['extra__{}'.format(extra_key)] = extra_val - common_vars.update(extra_vars) - - template_url = self.template_url.replace('${extra:', '${extra__') - return string.Template(template_url).safe_substitute(**common_vars) - - def repo_push_event_handler(self, event, data): - url = self.get_base_parsed_template(data) - url_cals = [] - branch_data = OrderedDict() - for obj in data['push']['branches']: - branch_data[obj['name']] = obj - - branches_commits = OrderedDict() - for commit in data['push']['commits']: - if commit.get('git_ref_change'): - # special case for GIT that allows creating tags, - # deleting branches without associated commit - continue - - if commit['branch'] not in branches_commits: - branch_commits = {'branch': branch_data[commit['branch']], - 'commits': []} - branches_commits[commit['branch']] = branch_commits - - branch_commits = branches_commits[commit['branch']] - branch_commits['commits'].append(commit) - - if '${branch}' in url: - # call it multiple times, for each branch if used in variables - for branch, commit_ids in branches_commits.items(): - branch_url = string.Template(url).safe_substitute(branch=branch) - # call further down for each commit if used - if '${commit_id}' in branch_url: - for commit_data in commit_ids['commits']: - commit_id = commit_data['raw_id'] - commit_url = string.Template(branch_url).safe_substitute( - commit_id=commit_id) - # register per-commit call - log.debug( - 'register webhook call(%s) to url %s', event, commit_url) - url_cals.append((commit_url, self.secret_token, self.headers, data)) - - else: - # register per-branch call - log.debug( - 'register webhook call(%s) to url %s', event, branch_url) - url_cals.append((branch_url, self.secret_token, self.headers, data)) - - else: - log.debug( - 'register webhook call(%s) to url %s', event, url) - url_cals.append((url, self.secret_token, self.headers, data)) - - return url_cals - - def repo_create_event_handler(self, event, data): - url = self.get_base_parsed_template(data) - log.debug( - 'register webhook call(%s) to url %s', event, url) - return [(url, self.secret_token, self.headers, data)] - - def pull_request_event_handler(self, event, data): - url = self.get_base_parsed_template(data) - log.debug( - 'register webhook call(%s) to url %s', event, url) - url = string.Template(url).safe_substitute( - pull_request_id=data['pullrequest']['pull_request_id'], - pull_request_url=data['pullrequest']['url']) - return [(url, self.secret_token, self.headers, data)] - - def __call__(self, event, data): - if isinstance(event, events.RepoPushEvent): - return self.repo_push_event_handler(event, data) - elif isinstance(event, events.RepoCreateEvent): - return self.repo_create_event_handler(event, data) - elif isinstance(event, events.PullRequestEvent): - return self.pull_request_event_handler(event, data) - else: - raise ValueError('event type not supported: %s' % events) +URL_VARS = get_url_vars(WEBHOOK_URL_VARS) class WebhookSettingsSchema(colander.Schema): @@ -183,17 +50,21 @@ class WebhookSettingsSchema(colander.Schema): colander.String(), title=_('Webhook URL'), description= - _('URL to which Webhook should submit data. Following variables ' - 'are allowed to be used: {vars}. Some of the variables would ' - 'trigger multiple calls, like ${{branch}} or ${{commit_id}}. ' - 'Webhook will be called as many times as unique objects in ' - 'data in such cases.').format(vars=URL_VARS), + _('URL to which Webhook should submit data. If used some of the ' + 'variables would trigger multiple calls, like ${branch} or ' + '${commit_id}. Webhook will be called as many times as unique ' + 'objects in data in such cases.'), missing=colander.required, required=True, validator=colander.url, - widget=deform.widget.TextInputWidget( - placeholder='https://www.example.com/webhook' - ), + widget=widgets.CodeMirrorWidget( + help_block_collapsable_name='Show url variables', + help_block_collapsable=( + 'E.g http://my-serv/trigger_job/${{event_name}}' + '?PR_ID=${{pull_request_id}}' + '\nFull list of vars:\n{}'.format(URL_VARS)), + codemirror_mode='text', + codemirror_options='{"lineNumbers": false, "lineWrapping": true}'), ) secret_token = colander.SchemaNode( colander.String(), @@ -234,7 +105,7 @@ class WebhookSettingsSchema(colander.Schema): default='', missing='', widget=deform.widget.TextInputWidget( - placeholder='e.g.Authorization' + placeholder='e.g: Authorization' ), ) custom_header_val = colander.SchemaNode( @@ -244,7 +115,7 @@ class WebhookSettingsSchema(colander.Schema): default='', missing='', widget=deform.widget.TextInputWidget( - placeholder='e.g. RcLogin auth=xxxx' + placeholder='e.g. Basic XxXxXx' ), ) method_type = colander.SchemaNode( @@ -264,8 +135,11 @@ class WebhookSettingsSchema(colander.Schema): class WebhookIntegrationType(IntegrationTypeBase): key = 'webhook' display_name = _('Webhook') - description = _('Post json events to a Webhook endpoint') - icon = '''''' + description = _('send JSON data to a url endpoint') + + @classmethod + def icon(cls): + return '''''' valid_events = [ events.PullRequestCloseEvent, @@ -293,8 +167,8 @@ class WebhookIntegrationType(IntegrationTypeBase): return schema def send_event(self, event): - log.debug('handling event %s with Webhook integration %s', - event.name, self) + log.debug( + 'handling event %s with Webhook integration %s', event.name, self) if event.__class__ not in self.valid_events: log.debug('event not valid: %r' % event) @@ -313,8 +187,7 @@ class WebhookIntegrationType(IntegrationTypeBase): if head_key and head_val: headers = {head_key: head_val} - handler = WebhookHandler( - template_url, self.settings['secret_token'], headers) + handler = WebhookDataHandler(template_url, headers) url_calls = handler(event, data) log.debug('webhook: calling following urls: %s', @@ -370,7 +243,10 @@ def post_to_webhook(url_calls, settings): rhodecode.__version__) } # updated below with custom ones, allows override - for url, token, headers, data in url_calls: + auth = get_auth(settings) + token = get_web_token(settings) + + for url, headers, data in url_calls: req_session = requests.Session() req_session.mount( # retry max N times 'http://', requests.adapters.HTTPAdapter(max_retries=retries)) @@ -380,7 +256,6 @@ def post_to_webhook(url_calls, settings): headers = headers or {} call_headers.update(headers) - auth = get_auth(settings) log.debug('calling Webhook with method: %s, and auth:%s', call_method, auth) @@ -392,4 +267,8 @@ def post_to_webhook(url_calls, settings): }, headers=call_headers, auth=auth) log.debug('Got Webhook response: %s', resp) - resp.raise_for_status() # raise exception on a failed request + try: + resp.raise_for_status() # raise exception on a failed request + except Exception: + log.error(resp.text) + raise diff --git a/rhodecode/lib/caching_query.py b/rhodecode/lib/caching_query.py index a7b2cad7..4993522a 100644 --- a/rhodecode/lib/caching_query.py +++ b/rhodecode/lib/caching_query.py @@ -99,8 +99,11 @@ class CachingQuery(Query): """ if hasattr(self, '_cache_parameters'): - return self.get_value(createfunc=lambda: - list(Query.__iter__(self))) + + def caching_query(): + return list(Query.__iter__(self)) + + return self.get_value(createfunc=caching_query) else: return Query.__iter__(self) diff --git a/rhodecode/lib/codeblocks.py b/rhodecode/lib/codeblocks.py index ad5c4129..ae82a208 100644 --- a/rhodecode/lib/codeblocks.py +++ b/rhodecode/lib/codeblocks.py @@ -28,8 +28,9 @@ from pygments.lexers.special import TextLexer, Token from rhodecode.lib.helpers import ( get_lexer_for_filenode, html_escape, get_custom_lexer) -from rhodecode.lib.utils2 import AttributeDict +from rhodecode.lib.utils2 import AttributeDict, StrictAttributeDict from rhodecode.lib.vcs.nodes import FileNode +from rhodecode.lib.vcs.exceptions import VCSError, NodeDoesNotExistError from rhodecode.lib.diff_match_patch import diff_match_patch from rhodecode.lib.diffs import LimitedDiffContainer from pygments.lexers import get_lexer_by_name @@ -38,7 +39,7 @@ plain_text_lexer = get_lexer_by_name( 'text', stripall=False, stripnl=False, ensurenl=False) -log = logging.getLogger() +log = logging.getLogger(__name__) def filenode_as_lines_tokens(filenode, lexer=None): @@ -351,6 +352,16 @@ def tokens_diff(old_tokens, new_tokens, use_diff_match_patch=True): return old_tokens_result, new_tokens_result, similarity +def diffset_node_getter(commit): + def get_node(fname): + try: + return commit.get_node(fname) + except NodeDoesNotExistError: + return None + + return get_node + + class DiffSet(object): """ An object for parsing the diff result from diffs.DiffProcessor and @@ -400,7 +411,12 @@ class DiffSet(object): for patch in patchset: diffset.file_stats[patch['filename']] = patch['stats'] filediff = self.render_patch(patch) - filediff.diffset = diffset + filediff.diffset = StrictAttributeDict(dict( + source_ref=diffset.source_ref, + target_ref=diffset.target_ref, + repo_name=diffset.repo_name, + source_repo_name=diffset.source_repo_name, + )) diffset.files.append(filediff) diffset.changed_files += 1 if not patch['stats']['binary']: @@ -510,6 +526,7 @@ class DiffSet(object): if target_file_path in self.comments_store: for lineno, comments in self.comments_store[target_file_path].items(): left_comments[lineno] = comments + # left comments are one that we couldn't place in diff lines. # could be outdated, or the diff changed and this line is no # longer available @@ -546,7 +563,7 @@ class DiffSet(object): result.lines.extend( self.parse_lines(before, after, source_file, target_file)) - result.unified = self.as_unified(result.lines) + result.unified = list(self.as_unified(result.lines)) result.sideside = result.lines return result @@ -601,8 +618,9 @@ class DiffSet(object): original.lineno = before['old_lineno'] original.content = before['line'] original.action = self.action_to_op(before['action']) - original.comments = self.get_comments_for('old', - source_file, before['old_lineno']) + + original.get_comment_args = ( + source_file, 'o', before['old_lineno']) if after: if after['action'] == 'new-no-nl': @@ -614,8 +632,9 @@ class DiffSet(object): modified.lineno = after['new_lineno'] modified.content = after['line'] modified.action = self.action_to_op(after['action']) - modified.comments = self.get_comments_for('new', - target_file, after['new_lineno']) + + modified.get_comment_args = ( + target_file, 'n', after['new_lineno']) # diff the lines if before_tokens and after_tokens: @@ -644,23 +663,6 @@ class DiffSet(object): return lines - def get_comments_for(self, version, filename, line_number): - if hasattr(filename, 'unicode_path'): - filename = filename.unicode_path - - if not isinstance(filename, basestring): - return None - - line_key = { - 'old': 'o', - 'new': 'n', - }[version] + str(line_number) - - if filename in self.comments_store: - file_comments = self.comments_store[filename] - if line_key in file_comments: - return file_comments.pop(line_key) - def get_line_tokens(self, line_text, line_number, file=None): filenode = None filename = None @@ -717,25 +719,25 @@ class DiffSet(object): if line.original.action == ' ': yield (line.original.lineno, line.modified.lineno, line.original.action, line.original.content, - line.original.comments) + line.original.get_comment_args) continue if line.original.action == '-': yield (line.original.lineno, None, line.original.action, line.original.content, - line.original.comments) + line.original.get_comment_args) if line.modified.action == '+': buf.append(( None, line.modified.lineno, line.modified.action, line.modified.content, - line.modified.comments)) + line.modified.get_comment_args)) continue if line.modified: yield (None, line.modified.lineno, line.modified.action, line.modified.content, - line.modified.comments) + line.modified.get_comment_args) for b in buf: yield b diff --git a/rhodecode/lib/db_manage.py b/rhodecode/lib/db_manage.py index ff0d98a1..30ff0a85 100644 --- a/rhodecode/lib/db_manage.py +++ b/rhodecode/lib/db_manage.py @@ -126,7 +126,7 @@ class DbManage(object): log.info("Deleting (%s) cache keys now...", total) CacheKey.delete_all_cache() - def upgrade(self): + def upgrade(self, version=None): """ Upgrades given database schema to given revision following all needed steps, to perform the upgrade @@ -157,9 +157,9 @@ class DbManage(object): db_uri = self.dburi try: - curr_version = api.db_version(db_uri, repository_path) - msg = ('Found current database under version ' - 'control with version %s' % curr_version) + curr_version = version or api.db_version(db_uri, repository_path) + msg = ('Found current database db_uri under version ' + 'control with version {}'.format(curr_version)) except (RuntimeError, DatabaseNotControlledError): curr_version = 1 diff --git a/rhodecode/lib/dbmigrate/versions/086_version_4_12_0.py b/rhodecode/lib/dbmigrate/versions/086_version_4_12_0.py new file mode 100644 index 00000000..22470bb1 --- /dev/null +++ b/rhodecode/lib/dbmigrate/versions/086_version_4_12_0.py @@ -0,0 +1,39 @@ +import logging + +from sqlalchemy import * + +from rhodecode.model import meta +from rhodecode.lib.dbmigrate.versions import _reset_base, notify + +log = logging.getLogger(__name__) + + +def upgrade(migrate_engine): + """ + Upgrade operations go here. + Don't create your own engine; bind migrate_engine to your metadata + """ + _reset_base(migrate_engine) + from rhodecode.lib.dbmigrate.schema import db_4_11_0_0 as db + + repository_table = db.Repository.__table__ + + push_uri = Column( + "push_uri", db.EncryptedTextValue(), nullable=True, unique=False, + default=None) + + push_uri.create(table=repository_table) + + # issue fixups + fixups(db, meta.Session) + + +def downgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + + +def fixups(models, _SESSION): + pass + + diff --git a/rhodecode/lib/diffs.py b/rhodecode/lib/diffs.py index f7cc998f..e2b0467c 100644 --- a/rhodecode/lib/diffs.py +++ b/rhodecode/lib/diffs.py @@ -23,16 +23,19 @@ Set of diffing helpers, previously part of vcs """ +import os import re +import bz2 + import collections import difflib import logging - +import cPickle as pickle from itertools import tee, imap from rhodecode.lib.vcs.exceptions import VCSError from rhodecode.lib.vcs.nodes import FileNode, SubModuleNode -from rhodecode.lib.utils2 import safe_unicode +from rhodecode.lib.utils2 import safe_unicode, safe_str log = logging.getLogger(__name__) @@ -1129,3 +1132,82 @@ class LineNotInDiffException(Exception): class DiffLimitExceeded(Exception): pass + + +def cache_diff(cached_diff_file, diff, commits): + + struct = { + 'version': 'v1', + 'diff': diff, + 'commits': commits + } + + try: + with bz2.BZ2File(cached_diff_file, 'wb') as f: + pickle.dump(struct, f) + log.debug('Saved diff cache under %s', cached_diff_file) + except Exception: + log.warn('Failed to save cache', exc_info=True) + # cleanup file to not store it "damaged" + try: + os.remove(cached_diff_file) + except Exception: + log.exception('Failed to cleanup path %s', cached_diff_file) + + +def load_cached_diff(cached_diff_file): + + default_struct = { + 'version': 'v1', + 'diff': None, + 'commits': None + } + + has_cache = os.path.isfile(cached_diff_file) + if not has_cache: + return default_struct + + data = None + try: + with bz2.BZ2File(cached_diff_file, 'rb') as f: + data = pickle.load(f) + log.debug('Loaded diff cache from %s', cached_diff_file) + except Exception: + log.warn('Failed to read diff cache file', exc_info=True) + + if not data: + data = default_struct + + if not isinstance(data, dict): + # old version of data ? + data = default_struct + + return data + + +def generate_diff_cache_key(*args): + """ + Helper to generate a cache key using arguments + """ + def arg_mapper(input_param): + input_param = safe_str(input_param) + # we cannot allow '/' in arguments since it would allow + # subdirectory usage + input_param.replace('/', '_') + return input_param or None # prevent empty string arguments + + return '_'.join([ + '{}' for i in range(len(args))]).format(*map(arg_mapper, args)) + + +def diff_cache_exist(cache_storage, *args): + """ + Based on all generated arguments check and return a cache path + """ + cache_key = generate_diff_cache_key(*args) + cache_file_path = os.path.join(cache_storage, cache_key) + # prevent path traversal attacks using some param that have e.g '../../' + if not os.path.abspath(cache_file_path).startswith(cache_storage): + raise ValueError('Final path must be within {}'.format(cache_storage)) + + return cache_file_path diff --git a/rhodecode/lib/exceptions.py b/rhodecode/lib/exceptions.py index 914f46f1..3051ce9c 100644 --- a/rhodecode/lib/exceptions.py +++ b/rhodecode/lib/exceptions.py @@ -132,6 +132,7 @@ class VCSServerUnavailable(HTTPBadGateway): 'Incorrect vcs.server=host:port', 'Incorrect vcs.server.protocol', ] + def __init__(self, message=''): self.explanation = 'Could not connect to VCS Server' if message: diff --git a/rhodecode/lib/helpers.py b/rhodecode/lib/helpers.py index 8925d053..034c9fa2 100644 --- a/rhodecode/lib/helpers.py +++ b/rhodecode/lib/helpers.py @@ -25,6 +25,7 @@ Consists of functions to typically be used within templates, but also available to Controllers. This module is available to both as 'h'. """ +import os import random import hashlib import StringIO @@ -742,18 +743,23 @@ short_id = lambda x: x[:12] hide_credentials = lambda x: ''.join(credentials_filter(x)) +import pytz +import tzlocal +local_timezone = tzlocal.get_localzone() + + def age_component(datetime_iso, value=None, time_is_local=False): title = value or format_date(datetime_iso) tzinfo = '+00:00' # detect if we have a timezone info, otherwise, add it - if isinstance(datetime_iso, datetime) and not datetime_iso.tzinfo: - if time_is_local: - tzinfo = time.strftime("+%H:%M", - time.gmtime( - (datetime.now() - datetime.utcnow()).seconds + 1 - ) - ) + if time_is_local and isinstance(datetime_iso, datetime) and not datetime_iso.tzinfo: + force_timezone = os.environ.get('RC_TIMEZONE', '') + if force_timezone: + force_timezone = pytz.timezone(force_timezone) + timezone = force_timezone or local_timezone + offset = timezone.localize(datetime_iso).strftime('%z') + tzinfo = '{}:{}'.format(offset[:-2], offset[-2:]) return literal( '