engineering/unrhodecode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

pytest: restructure fixtures/plugins to fix problems with pkg_resources and Can't perform this operation for unregistered loader type errors.

Browse source 
- seems the latest pytest breaks older packages relying on pkg_resources when defining a entrypoints plugins
- change the way we register plugins to be local only, this is what we need anyway
This commit is contained in:
RhodeCode Admin 2023-03-21 12:24:16 +01:00
parent 38d873cd45
commit e1d1da53b9
8 changed files with 209 additions and 268 deletions
Download patch file Download diff file Expand all files Collapse all files

2
rhodecode/api/tests/test_update_repo.py
View file

@ -27,7 +27,7 @@ from rhodecode.tests import TEST_USER_ADMIN_LOGIN, TEST_USER_REGULAR_LOGIN
from rhodecode.api.tests.utils import (
build_data, api_call, assert_error, assert_ok, crash, jsonify)
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.plugin import plain_http_host_only_stub
from rhodecode.tests.fixture_mods.fixture_utils import plain_http_host_only_stub
fixture = Fixture()

199
rhodecode/tests/conftest.py Normal file
View file

@ -0,0 +1,199 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2010-2020 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import pytest
from rhodecode.lib import ext_json
pytest_plugins = [
"rhodecode.tests.fixture_mods.fixture_pyramid",
"rhodecode.tests.fixture_mods.fixture_utils",
]
def pytest_configure(config):
from rhodecode.config import patches
def pytest_addoption(parser):
def _parse_json(value):
return ext_json.str_json(value) if value else None
def _split_comma(value):
return value.split(',')
parser.addoption(
'--keep-tmp-path', action='store_true',
help="Keep the test temporary directories")
parser.addoption(
'--backends', action='store', type=_split_comma,
default=['git', 'hg', 'svn'],
help="Select which backends to test for backend specific tests.")
parser.addoption(
'--dbs', action='store', type=_split_comma,
default=['sqlite'],
help="Select which database to test for database specific tests. "
"Possible options are sqlite,postgres,mysql")
parser.addoption(
'--appenlight', '--ae', action='store_true',
help="Track statistics in appenlight.")
parser.addoption(
'--appenlight-api-key', '--ae-key',
help="API key for Appenlight.")
parser.addoption(
'--appenlight-url', '--ae-url',
default="https://ae.rhodecode.com",
help="Appenlight service URL, defaults to https://ae.rhodecode.com")
parser.addoption(
'--sqlite-connection-string', action='store',
default='', help="Connection string for the dbs tests with SQLite")
parser.addoption(
'--postgres-connection-string', action='store',
default='', help="Connection string for the dbs tests with Postgres")
parser.addoption(
'--mysql-connection-string', action='store',
default='', help="Connection string for the dbs tests with MySQL")
parser.addoption(
'--repeat', type=int, default=100,
help="Number of repetitions in performance tests.")
parser.addoption(
'--test-loglevel', dest='test_loglevel',
help="Set default Logging level for tests, critical(default), error, warn , info, debug")
group = parser.getgroup('pylons')
group.addoption(
'--with-pylons', dest='pyramid_config',
help="Set up a Pylons environment with the specified config file.")
group.addoption(
'--ini-config-override', action='store', type=_parse_json,
default=None, dest='pyramid_config_override', help=(
"Overrides the .ini file settings. Should be specified in JSON"
" format, e.g. '{\"section\": {\"parameter\": \"value\", ...}}'"
)
)
parser.addini(
'pyramid_config',
"Set up a Pyramid environment with the specified config file.")
vcsgroup = parser.getgroup('vcs')
vcsgroup.addoption(
'--without-vcsserver', dest='with_vcsserver', action='store_false',
help="Do not start the VCSServer in a background process.")
vcsgroup.addoption(
'--with-vcsserver-http', dest='vcsserver_config_http',
help="Start the HTTP VCSServer with the specified config file.")
vcsgroup.addoption(
'--vcsserver-protocol', dest='vcsserver_protocol',
help="Start the VCSServer with HTTP protocol support.")
vcsgroup.addoption(
'--vcsserver-config-override', action='store', type=_parse_json,
default=None, dest='vcsserver_config_override', help=(
"Overrides the .ini file settings for the VCSServer. "
"Should be specified in JSON "
"format, e.g. '{\"section\": {\"parameter\": \"value\", ...}}'"
)
)
vcsgroup.addoption(
'--vcsserver-port', action='store', type=int,
default=None, help=(
"Allows to set the port of the vcsserver. Useful when testing "
"against an already running server and random ports cause "
"trouble."))
parser.addini(
'vcsserver_config_http',
"Start the HTTP VCSServer with the specified config file.")
parser.addini(
'vcsserver_protocol',
"Start the VCSServer with HTTP protocol support.")
@pytest.hookimpl(tryfirst=True, hookwrapper=True)
def pytest_runtest_makereport(item, call):
"""
Adding the remote traceback if the exception has this information.
VCSServer attaches this information as the attribute `_vcs_server_traceback`
to the exception instance.
"""
outcome = yield
report = outcome.get_result()
if call.excinfo:
exc = call.excinfo.value
vcsserver_traceback = getattr(exc, '_vcs_server_traceback', None)
if vcsserver_traceback:
section = 'VCSServer remote traceback ' + report.when
report.sections.append((section, vcsserver_traceback))
def pytest_collection_modifyitems(session, config, items):
# nottest marked, compare nose, used for transition from nose to pytest
remaining = [
i for i in items if getattr(i.obj, '__test__', True)]
items[:] = remaining
# NOTE(marcink): custom test ordering, db tests and vcstests are slowes and should
# be executed at the end for faster test feedback
def sorter(item):
pos = 0
key = item._nodeid
if key.startswith('rhodecode/tests/database'):
pos = 1
elif key.startswith('rhodecode/tests/vcs_operations'):
pos = 2
return pos
items.sort(key=sorter)
def get_backends_from_metafunc(metafunc):
requested_backends = set(metafunc.config.getoption('--backends'))
backend_mark = metafunc.definition.get_closest_marker('backends')
if backend_mark:
# Supported backends by this test function, created from
# pytest.mark.backends
backends = backend_mark.args
elif hasattr(metafunc.cls, 'backend_alias'):
# Support class attribute "backend_alias", this is mainly
# for legacy reasons for tests not yet using pytest.mark.backends
backends = [metafunc.cls.backend_alias]
else:
backends = metafunc.config.getoption('--backends')
return requested_backends.intersection(backends)
def pytest_generate_tests(metafunc):
# Support test generation based on --backend parameter
if 'backend_alias' in metafunc.fixturenames:
backends = get_backends_from_metafunc(metafunc)
scope = None
if not backends:
pytest.skip("Not enabled for any of selected backends")
metafunc.parametrize('backend_alias', backends, scope=scope)
backend_mark = metafunc.definition.get_closest_marker('backends')
if backend_mark:
backends = get_backends_from_metafunc(metafunc)
if not backends:
pytest.skip("Not enabled for any of selected backends")

0
rhodecode/tests/fixture_mods/__init__.py Normal file
View file

63
rhodecode/tests/pylons_plugin.py → rhodecode/tests/fixture_mods/fixture_pyramid.py
View file

@ -18,10 +18,6 @@
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import json
import platform
import socket
import random
import pytest
from rhodecode.lib.pyramid_utils import get_app_config
@ -29,61 +25,6 @@ from rhodecode.tests.fixture import TestINI
from rhodecode.tests.server_utils import RcVCSServer
def _parse_json(value):
return json.loads(value) if value else None
def pytest_addoption(parser):
parser.addoption(
'--test-loglevel', dest='test_loglevel',
help="Set default Logging level for tests, critical(default), error, warn , info, debug")
group = parser.getgroup('pylons')
group.addoption(
'--with-pylons', dest='pyramid_config',
help="Set up a Pylons environment with the specified config file.")
group.addoption(
'--ini-config-override', action='store', type=_parse_json,
default=None, dest='pyramid_config_override', help=(
"Overrides the .ini file settings. Should be specified in JSON"
" format, e.g. '{\"section\": {\"parameter\": \"value\", ...}}'"
)
)
parser.addini(
'pyramid_config',
"Set up a Pyramid environment with the specified config file.")
vcsgroup = parser.getgroup('vcs')
vcsgroup.addoption(
'--without-vcsserver', dest='with_vcsserver', action='store_false',
help="Do not start the VCSServer in a background process.")
vcsgroup.addoption(
'--with-vcsserver-http', dest='vcsserver_config_http',
help="Start the HTTP VCSServer with the specified config file.")
vcsgroup.addoption(
'--vcsserver-protocol', dest='vcsserver_protocol',
help="Start the VCSServer with HTTP protocol support.")
vcsgroup.addoption(
'--vcsserver-config-override', action='store', type=_parse_json,
default=None, dest='vcsserver_config_override', help=(
"Overrides the .ini file settings for the VCSServer. "
"Should be specified in JSON "
"format, e.g. '{\"section\": {\"parameter\": \"value\", ...}}'"
)
)
vcsgroup.addoption(
'--vcsserver-port', action='store', type=int,
default=None, help=(
"Allows to set the port of the vcsserver. Useful when testing "
"against an already running server and random ports cause "
"trouble."))
parser.addini(
'vcsserver_config_http',
"Start the HTTP VCSServer with the specified config file.")
parser.addini(
'vcsserver_protocol',
"Start the VCSServer with HTTP protocol support.")
@pytest.fixture(scope='session')
def vcsserver(request, vcsserver_port, vcsserver_factory):
"""
@ -146,10 +87,6 @@ def vcsserver_factory(tmpdir_factory):
return factory
def is_cygwin():
return 'cygwin' in platform.system().lower()
def _use_log_level(config):
level = config.getoption('test_loglevel') or 'critical'
return level.upper()

126
rhodecode/tests/plugin.py → rhodecode/tests/fixture_mods/fixture_utils.py
View file

@ -74,107 +74,6 @@ def cmp(a, b):
# backport cmp from python2 so we can still use it in the custom code in this module
return (a > b) - (a < b)
def _split_comma(value):
return value.split(',')
def pytest_addoption(parser):
parser.addoption(
'--keep-tmp-path', action='store_true',
help="Keep the test temporary directories")
parser.addoption(
'--backends', action='store', type=_split_comma,
default=['git', 'hg', 'svn'],
help="Select which backends to test for backend specific tests.")
parser.addoption(
'--dbs', action='store', type=_split_comma,
default=['sqlite'],
help="Select which database to test for database specific tests. "
"Possible options are sqlite,postgres,mysql")
parser.addoption(
'--appenlight', '--ae', action='store_true',
help="Track statistics in appenlight.")
parser.addoption(
'--appenlight-api-key', '--ae-key',
help="API key for Appenlight.")
parser.addoption(
'--appenlight-url', '--ae-url',
default="https://ae.rhodecode.com",
help="Appenlight service URL, defaults to https://ae.rhodecode.com")
parser.addoption(
'--sqlite-connection-string', action='store',
default='', help="Connection string for the dbs tests with SQLite")
parser.addoption(
'--postgres-connection-string', action='store',
default='', help="Connection string for the dbs tests with Postgres")
parser.addoption(
'--mysql-connection-string', action='store',
default='', help="Connection string for the dbs tests with MySQL")
parser.addoption(
'--repeat', type=int, default=100,
help="Number of repetitions in performance tests.")
def pytest_configure(config):
from rhodecode.config import patches
def pytest_collection_modifyitems(session, config, items):
# nottest marked, compare nose, used for transition from nose to pytest
remaining = [
i for i in items if getattr(i.obj, '__test__', True)]
items[:] = remaining
# NOTE(marcink): custom test ordering, db tests and vcstests are slowes and should
# be executed at the end for faster test feedback
def sorter(item):
pos = 0
key = item._nodeid
if key.startswith('rhodecode/tests/database'):
pos = 1
elif key.startswith('rhodecode/tests/vcs_operations'):
pos = 2
return pos
items.sort(key=sorter)
def pytest_generate_tests(metafunc):
# Support test generation based on --backend parameter
if 'backend_alias' in metafunc.fixturenames:
backends = get_backends_from_metafunc(metafunc)
scope = None
if not backends:
pytest.skip("Not enabled for any of selected backends")
metafunc.parametrize('backend_alias', backends, scope=scope)
backend_mark = metafunc.definition.get_closest_marker('backends')
if backend_mark:
backends = get_backends_from_metafunc(metafunc)
if not backends:
pytest.skip("Not enabled for any of selected backends")
def get_backends_from_metafunc(metafunc):
requested_backends = set(metafunc.config.getoption('--backends'))
backend_mark = metafunc.definition.get_closest_marker('backends')
if backend_mark:
# Supported backends by this test function, created from
# pytest.mark.backends
backends = backend_mark.args
elif hasattr(metafunc.cls, 'backend_alias'):
# Support class attribute "backend_alias", this is mainly
# for legacy reasons for tests not yet using pytest.mark.backends
backends = [metafunc.cls.backend_alias]
else:
backends = metafunc.config.getoption('--backends')
return requested_backends.intersection(backends)
@pytest.fixture(scope='session', autouse=True)
def activate_example_rcextensions(request):
"""
@ -1400,30 +1299,6 @@ class UserUtility(object):
self.fixture.destroy_user(user_id)
# TODO: Think about moving this into a pytest-pyro package and make it a
# pytest plugin
@pytest.hookimpl(tryfirst=True, hookwrapper=True)
def pytest_runtest_makereport(item, call):
"""
Adding the remote traceback if the exception has this information.
VCSServer attaches this information as the attribute `_vcs_server_traceback`
to the exception instance.
"""
outcome = yield
report = outcome.get_result()
if call.excinfo:
_add_vcsserver_remote_traceback(report, call.excinfo.value)
def _add_vcsserver_remote_traceback(report, exc):
vcsserver_traceback = getattr(exc, '_vcs_server_traceback', None)
if vcsserver_traceback:
section = 'VCSServer remote traceback ' + report.when
report.sections.append((section, vcsserver_traceback))
@pytest.fixture(scope='session')
def testrun():
return {
@ -1722,6 +1597,7 @@ def StubIntegrationType():
integration_type_registry.register_integration_type(_StubIntegrationType)
return _StubIntegrationType
@pytest.fixture()
def stub_integration_settings():
return {

11
rhodecode/tests/utils.py
View file

@ -34,7 +34,12 @@ from webtest.app import TestResponse, TestApp
from webtest.compat import print_stderr
import pytest
import rc_testdata
try:
import rc_testdata
except ImportError:
raise ImportError('Failed to import rc_testdata, '
'please make sure this package is installed from requirements_test.txt')
from rhodecode.model.db import User, Repository
from rhodecode.model.meta import Session
@ -424,13 +429,13 @@ def commit_change(
message=message,
nodes=nodes,
parent_commit=_commit,
author='{} <admin@rhodecode.com>'.format(TEST_USER_ADMIN_LOGIN),
author=f'{TEST_USER_ADMIN_LOGIN} <admin@rhodecode.com>',
)
else:
commit = ScmModel().commit_change(
repo=repo.scm_instance(), repo_name=repo.repo_name,
commit=parent, user=TEST_USER_ADMIN_LOGIN,
author='{} <admin@rhodecode.com>'.format(TEST_USER_ADMIN_LOGIN),
author=f'{TEST_USER_ADMIN_LOGIN} <admin@rhodecode.com>',
message=message,
content=content,
f_path=filename

4
setup.py
View file

@ -189,10 +189,6 @@ setup(
'pyramid.pshell_runner': [
'ipython = rhodecode.lib.pyramid_shell:ipython_shell_runner',
],
'pytest11': [
'pylons=rhodecode.tests.pylons_plugin',
'enterprise=rhodecode.tests.plugin',
],
'console_scripts': [
'rc-setup-app=rhodecode.lib.rc_commands.setup_rc:main',
'rc-upgrade-db=rhodecode.lib.rc_commands.upgrade_db:main',

72
test_special_dict.py
View file

@ -1,72 +0,0 @@
import collections
class PermOriginDict(dict):
"""
A special dict used for tracking permissions along with their origins.
`__setitem__` has been overridden to expect a tuple(perm, origin)
`__getitem__` will return only the perm
`.perm_origin_stack` will return the stack of (perm, origin) set per key
>>> perms = PermOriginDict()
>>> perms['resource'] = 'read', 'default', 1
>>> perms['resource']
'read'
>>> perms['resource'] = 'write', 'admin', 2
>>> perms['resource']
'write'
>>> perms.perm_origin_stack
{'resource': [('read', 'default', 1), ('write', 'admin', 2)]}
"""
def __init__(self, *args, **kw):
dict.__init__(self, *args, **kw)
super(PermOriginDict, self).__setattr__('perm_origin_stack', collections.OrderedDict())
def __setitem__(self, key, perm_origin_obj_id):
print(f"setting: {perm_origin_obj_id}")
if isinstance(perm_origin_obj_id, tuple):
(perm, origin, obj_id) = perm_origin_obj_id
self.perm_origin_stack.setdefault(key, []).append((perm, origin, obj_id))
else:
perm = perm_origin_obj_id
dict.__setitem__(self, key, perm)
# def __getstate__(self):
# dict_state = self.__dict__.copy()
# perm_origin_stack = self.perm_origin_stack.__dict__.copy()
# print(dict_state, perm_origin_stack)
# return dict_state, perm_origin_stack
#
# def __setstate__(self, state):
# print('setstate called')
# print(f'setstate os {state}')
# self.perm_origin_stack = collections.OrderedDict()
dd = PermOriginDict()
dd['key1'] = 'perm1', 'from_where1', 121
dd['key2'] = 'perm2', 'from_where2', 122
dd['key3'] = 'perm3', 'from_where3', 123
dd['key3'] = 'perm4', 'from_where4', 125
print(list(dd.keys()))
print(f"key:{dd['key1']}")
print(dd.perm_origin_stack)
print ('-- before pickle --')
import pickle
data = pickle.dumps(dd, protocol=pickle.HIGHEST_PROTOCOL)
print(data)
new_dd = pickle.loads(data)
print ('-- after pickle --')
print(list(dd.keys()))
print(f"key:{dd['key1']}")
print(dd.perm_origin_stack)