tests: use gunicorn for testing. This is close to production testing

and actually will reproduce any chunked encoding problems as we hit on production
servers.
This commit is contained in:
Marcin Kuzminski 2017-12-18 12:29:33 +01:00
parent fad0fcf1c3
commit aa84b1a0e4
19 changed files with 126 additions and 189 deletions

View file

@ -171,7 +171,7 @@ def rc_web_server(
host_url = 'http://' + get_host_url(rc_web_server_config)
assert_no_running_instance(host_url)
command = ['pserve', rc_web_server_config]
command = ['gunicorn', '--worker-class', 'gevent', '--paste', rc_web_server_config]
print('Starting rhodecode server: {}'.format(host_url))
print('Command: {}'.format(command))

View file

@ -22,6 +22,8 @@ import os
import json
import platform
import socket
import tempfile
import subprocess32
import time
from urllib2 import urlopen, URLError
@ -34,6 +36,9 @@ import pyramid.paster
from rhodecode.lib.pyramid_utils import get_app_config
from rhodecode.tests.fixture import TestINI
import rhodecode
from rhodecode.tests.other.vcs_operations.conftest import get_host_url, get_port
VCSSERVER_LOG = os.path.join(tempfile.gettempdir(), 'rc-vcsserver.log')
def _parse_json(value):
@ -204,10 +209,11 @@ class HttpVCSServer(VCSServer):
Represents a running VCSServer instance.
"""
def __init__(self, config_file):
self.config_file = config_file
config_data = configobj.ConfigObj(config_file)
self._config = config_data['server:main']
args = ['pserve', config_file]
args = ['gunicorn', '--workers', '1', '--paste', config_file]
self._args = args
@property
@ -216,7 +222,21 @@ class HttpVCSServer(VCSServer):
return template.format(**self._config)
def start(self):
self.process = subprocess32.Popen(self._args)
env = os.environ.copy()
host_url = 'http://' + get_host_url(self.config_file)
rc_log = list(VCSSERVER_LOG.partition('.log'))
rc_log.insert(1, get_port(self.config_file))
rc_log = ''.join(rc_log)
server_out = open(rc_log, 'w')
command = ' '.join(self._args)
print('Starting rhodecode-vcsserver: {}'.format(host_url))
print('Command: {}'.format(command))
print('Logfile: {}'.format(rc_log))
self.process = subprocess32.Popen(
self._args, bufsize=0, env=env, stdout=server_out, stderr=server_out)
def wait_until_ready(self, timeout=30):
host = self._config['host']

View file

@ -46,27 +46,12 @@ debug = true
host = 0.0.0.0
port = 5000
##################################
## WAITRESS WSGI SERVER ##
## Recommended for Development ##
##################################
use = egg:waitress#main
## number of worker threads
threads = 5
## MAX BODY SIZE 100GB
max_request_body_size = 107374182400
## Use poll instead of select, fixes file descriptors limits problems.
## May not work on old windows systems.
asyncore_use_poll = true
##########################
## GUNICORN WSGI SERVER ##
##########################
## run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini
#use = egg:gunicorn#main
use = egg:gunicorn#main
## Sets the number of process workers. You must set `instance_id = *`
## when this option is set to more than one worker, recommended
## value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers
@ -90,66 +75,6 @@ asyncore_use_poll = true
## gets killed and restarted. Set to 6hrs
#timeout = 21600
## UWSGI ##
## run with uwsgi --ini-paste-logged <inifile.ini>
#[uwsgi]
#socket = /tmp/uwsgi.sock
#master = true
#http = 127.0.0.1:5000
## set as deamon and redirect all output to file
#daemonize = ./uwsgi_rhodecode.log
## master process PID
#pidfile = ./uwsgi_rhodecode.pid
## stats server with workers statistics, use uwsgitop
## for monitoring, `uwsgitop 127.0.0.1:1717`
#stats = 127.0.0.1:1717
#memory-report = true
## log 5XX errors
#log-5xx = true
## Set the socket listen queue size.
#listen = 256
## Gracefully Reload workers after the specified amount of managed requests
## (avoid memory leaks).
#max-requests = 1000
## enable large buffers
#buffer-size=65535
## socket and http timeouts ##
#http-timeout=3600
#socket-timeout=3600
## Log requests slower than the specified number of milliseconds.
#log-slow = 10
## Exit if no app can be loaded.
#need-app = true
## Set lazy mode (load apps in workers instead of master).
#lazy = true
## scaling ##
## set cheaper algorithm to use, if not set default will be used
#cheaper-algo = spare
## minimum number of workers to keep at all times
#cheaper = 1
## number of workers to spawn at startup
#cheaper-initial = 1
## maximum number of workers that can be spawned
#workers = 4
## how many workers should be spawned at a time
#cheaper-step = 1
## prefix middleware for RhodeCode.
## recommended when using proxy setup.
## allows to set RhodeCode under a prefix in server.

View file

@ -1,77 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2010-2017 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
"""
Module providing backend independent mixin class. It requires that
InMemoryCommit class is working properly at backend class.
"""
import datetime
import pytest
from rhodecode.lib.vcs.nodes import FileNode
# TODO: Needs clean py.test integration, knowledge is spread around
# conftest.py and this class, that's not good.
@pytest.mark.usefixtures("vcs_repository_support")
class BackendTestMixin(object):
"""
This is a backend independent test case class which should be created
with ``type`` method.
It is required to set following attributes at subclass:
- ``backend_alias``: alias of used backend (see ``vcs.BACKENDS``)
- ``repo_path``: path to the repository which would be created for set of
tests
- ``recreate_repo_per_test``: If set to ``False``, repo would NOT be
created
before every single test. Defaults to ``True``.
"""
recreate_repo_per_test = True
@classmethod
def _get_commits(cls):
commits = [
{
'message': u'Initial commit',
'author': u'Joe Doe <joe.doe@example.com>',
'date': datetime.datetime(2010, 1, 1, 20),
'added': [
FileNode('foobar', content='Foobar'),
FileNode('foobar2', content='Foobar II'),
FileNode('foo/bar/baz', content='baz here!'),
],
},
{
'message': u'Changes...',
'author': u'Jane Doe <jane.doe@example.com>',
'date': datetime.datetime(2010, 1, 1, 21),
'added': [
FileNode('some/new.txt', content='news...'),
],
'changed': [
FileNode('foobar', 'Foobar I'),
],
'removed': [],
},
]
return commits

View file

@ -18,8 +18,8 @@
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import shutil
import time
import shutil
import datetime
import pytest
@ -29,7 +29,6 @@ from rhodecode.lib.vcs.backends.base import Config
from rhodecode.lib.vcs.nodes import FileNode
from rhodecode.tests import get_new_dir
from rhodecode.tests.utils import check_skip_backends, check_xfail_backends
from rhodecode.tests.vcs.base import BackendTestMixin
@pytest.fixture()
@ -61,7 +60,7 @@ def vcs_repository_support(
cls.Backend = cls.backend_class = repo.__class__
cls.imc = repo.in_memory_commit
return (backend_alias, repo)
return backend_alias, repo
@pytest.fixture(scope='class')
@ -88,6 +87,7 @@ class VcsRepoContainer(object):
def get_repo(self, test_class, backend_alias):
if backend_alias not in self._repos:
repo = _create_empty_repository(test_class, backend_alias)
self._cleanup_paths.append(repo.path)
self._repos[backend_alias] = repo
return self._repos[backend_alias]
@ -106,7 +106,8 @@ def _create_empty_repository(cls, backend_alias=None):
repo_path = get_new_dir(str(time.time()))
repo = Backend(repo_path, create=True)
if hasattr(cls, '_get_commits'):
cls.tip = _add_commits_to_repo(repo, cls._get_commits())
commits = cls._get_commits()
cls.tip = _add_commits_to_repo(repo, commits)
return repo
@ -132,7 +133,7 @@ def config():
def _add_commits_to_repo(repo, commits):
imc = repo.in_memory_commit
commit = None
tip = None
for commit in commits:
for node in commit.get('added', []):
@ -142,13 +143,13 @@ def _add_commits_to_repo(repo, commits):
for node in commit.get('removed', []):
imc.remove(FileNode(node.path))
commit = imc.commit(
tip = imc.commit(
message=unicode(commit['message']),
author=unicode(commit['author']),
date=commit['date'],
branch=commit.get('branch'))
return commit
return tip
@pytest.fixture
@ -198,7 +199,7 @@ def generate_repo_with_commits(vcs_repo):
def hg_repo(request, vcs_repo):
repo = vcs_repo
commits = BackendTestMixin._get_commits()
commits = repo._get_commits()
_add_commits_to_repo(repo, commits)
return repo
@ -207,3 +208,50 @@ def hg_repo(request, vcs_repo):
@pytest.fixture
def hg_commit(hg_repo):
return hg_repo.get_commit()
class BackendTestMixin(object):
"""
This is a backend independent test case class which should be created
with ``type`` method.
It is required to set following attributes at subclass:
- ``backend_alias``: alias of used backend (see ``vcs.BACKENDS``)
- ``repo_path``: path to the repository which would be created for set of
tests
- ``recreate_repo_per_test``: If set to ``False``, repo would NOT be
created
before every single test. Defaults to ``True``.
"""
recreate_repo_per_test = True
@classmethod
def _get_commits(cls):
commits = [
{
'message': u'Initial commit',
'author': u'Joe Doe <joe.doe@example.com>',
'date': datetime.datetime(2010, 1, 1, 20),
'added': [
FileNode('foobar', content='Foobar'),
FileNode('foobar2', content='Foobar II'),
FileNode('foo/bar/baz', content='baz here!'),
],
},
{
'message': u'Changes...',
'author': u'Jane Doe <jane.doe@example.com>',
'date': datetime.datetime(2010, 1, 1, 21),
'added': [
FileNode('some/new.txt', content='news...'),
],
'changed': [
FileNode('foobar', 'Foobar I'),
],
'removed': [],
},
]
return commits

View file

@ -32,9 +32,10 @@ import pytest
from rhodecode.lib.vcs.backends import base
from rhodecode.lib.vcs.exceptions import ImproperArchiveTypeError, VCSError
from rhodecode.lib.vcs.nodes import FileNode
from rhodecode.tests.vcs.base import BackendTestMixin
from rhodecode.tests.vcs.conftest import BackendTestMixin
@pytest.mark.usefixtures("vcs_repository_support")
class TestArchives(BackendTestMixin):
@pytest.fixture(autouse=True)
@ -48,7 +49,7 @@ class TestArchives(BackendTestMixin):
@classmethod
def _get_commits(cls):
start_date = datetime.datetime(2010, 1, 1, 20)
for x in xrange(5):
for x in range(5):
yield {
'message': 'Commit %d' % x,
'author': 'Joe Doe <joe.doe@example.com>',
@ -68,7 +69,7 @@ class TestArchives(BackendTestMixin):
out_file.extractall(out_dir)
out_file.close()
for x in xrange(5):
for x in range(5):
node_path = '%d/file_%d.txt' % (x, x)
with open(os.path.join(out_dir, 'repo/' + node_path)) as f:
file_content = f.read()
@ -80,7 +81,7 @@ class TestArchives(BackendTestMixin):
self.tip.archive_repo(self.temp_file, kind='zip', prefix='repo')
out = zipfile.ZipFile(self.temp_file)
for x in xrange(5):
for x in range(5):
node_path = '%d/file_%d.txt' % (x, x)
decompressed = StringIO.StringIO()
decompressed.write(out.read('repo/' + node_path))
@ -98,7 +99,7 @@ class TestArchives(BackendTestMixin):
raw_id = self.tip.raw_id
assert 'rev:%s' % raw_id in metafile
for x in xrange(5):
for x in range(5):
node_path = '%d/file_%d.txt' % (x, x)
decompressed = StringIO.StringIO()
decompressed.write(out.read('repo/' + node_path))

View file

@ -23,9 +23,10 @@ import datetime
import pytest
from rhodecode.lib.vcs.nodes import FileNode
from rhodecode.tests.vcs.base import BackendTestMixin
from rhodecode.tests.vcs.conftest import BackendTestMixin
@pytest.mark.usefixtures("vcs_repository_support")
class TestBranches(BackendTestMixin):
def test_empty_repository_has_no_branches(self, vcsbackend):

View file

@ -32,7 +32,7 @@ from rhodecode.lib.vcs.nodes import (
FileNode, AddedFileNodesGenerator,
ChangedFileNodesGenerator, RemovedFileNodesGenerator)
from rhodecode.tests import get_new_dir
from rhodecode.tests.vcs.base import BackendTestMixin
from rhodecode.tests.vcs.conftest import BackendTestMixin
class TestBaseChangeset:
@ -42,13 +42,14 @@ class TestBaseChangeset:
pytest.deprecated_call(BaseChangeset)
class TestEmptyCommit:
class TestEmptyCommit(object):
def test_branch_without_alias_returns_none(self):
commit = EmptyCommit()
assert commit.branch is None
@pytest.mark.usefixtures("vcs_repository_support")
class TestCommitsInNonEmptyRepo(BackendTestMixin):
recreate_repo_per_test = True
@ -197,6 +198,7 @@ class TestCommitsInNonEmptyRepo(BackendTestMixin):
assert [commit] == repo.get_commit(commit_idx=test_idx).children
@pytest.mark.usefixtures("vcs_repository_support")
class TestCommits(BackendTestMixin):
recreate_repo_per_test = False
@ -501,6 +503,7 @@ def test_commit_is_link(vcsbackend, filename, expected):
assert link_status is expected
@pytest.mark.usefixtures("vcs_repository_support")
class TestCommitsChanges(BackendTestMixin):
recreate_repo_per_test = False

View file

@ -22,7 +22,7 @@ import datetime
import pytest
from rhodecode.lib.vcs.nodes import FileNode
from rhodecode.tests.vcs.base import BackendTestMixin
from rhodecode.tests.vcs.conftest import BackendTestMixin
class TestGetDiffValidation:
@ -68,6 +68,7 @@ class TestGetDiffValidation:
path='trunk/example.py', path1='branches/argparse/example.py')
@pytest.mark.usefixtures("vcs_repository_support")
class TestRepositoryGetDiff(BackendTestMixin):
recreate_repo_per_test = False
@ -405,6 +406,7 @@ new file mode 10644
assert diff.raw == expected_diff
@pytest.mark.usefixtures("vcs_repository_support")
class TestGetDiffBinary(BackendTestMixin):
recreate_repo_per_test = False

View file

@ -19,10 +19,13 @@
# and proprietary license terms, please see https://rhodecode.com/licenses/
import datetime
import pytest
from rhodecode.lib.vcs.nodes import FileNode
from rhodecode.tests.vcs.base import BackendTestMixin
from rhodecode.tests.vcs.conftest import BackendTestMixin
@pytest.mark.usefixtures("vcs_repository_support")
class TestFileNodeUnicodePath(BackendTestMixin):
fname = 'ąśðąęłąć.txt'

View file

@ -24,9 +24,10 @@ import pytest
from rhodecode.lib.vcs.exceptions import CommitDoesNotExistError
from rhodecode.lib.vcs.nodes import FileNode
from rhodecode.tests.vcs.base import BackendTestMixin
from rhodecode.tests.vcs.conftest import BackendTestMixin
@pytest.mark.usefixtures("vcs_repository_support")
class TestGetitem(BackendTestMixin):
@classmethod

View file

@ -19,10 +19,12 @@
# and proprietary license terms, please see https://rhodecode.com/licenses/
import datetime
import pytest
from rhodecode.lib.vcs.nodes import FileNode
from rhodecode.tests.vcs.base import BackendTestMixin
from rhodecode.tests.vcs.conftest import BackendTestMixin
@pytest.mark.usefixtures("vcs_repository_support")
class TestGetslice(BackendTestMixin):
@classmethod

View file

@ -35,7 +35,7 @@ from rhodecode.lib.vcs.exceptions import (
from rhodecode.lib.vcs.nodes import (
NodeKind, FileNode, DirNode, NodeState, SubModuleNode)
from rhodecode.tests import TEST_GIT_REPO, TEST_GIT_REPO_CLONE, get_new_dir
from rhodecode.tests.vcs.base import BackendTestMixin
from rhodecode.tests.vcs.conftest import BackendTestMixin
pytestmark = pytest.mark.backends("git")
@ -1030,6 +1030,7 @@ class TestLargeFileRepo(object):
assert lf_node.name == '1MB.zip'
@pytest.mark.usefixtures("vcs_repository_support")
class TestGitSpecificWithRepo(BackendTestMixin):
@classmethod
@ -1092,6 +1093,7 @@ class TestGitSpecificWithRepo(BackendTestMixin):
self.repo._get_commit_id(1), '--', 'foo'])
@pytest.mark.usefixtures("vcs_repository_support")
class TestGitRegression(BackendTestMixin):
@classmethod

View file

@ -31,7 +31,7 @@ from rhodecode.lib.vcs.exceptions import (
NodeAlreadyRemovedError, NodeAlreadyChangedError, NodeDoesNotExistError,
NodeNotChangedError)
from rhodecode.lib.vcs.nodes import DirNode, FileNode
from rhodecode.tests.vcs.base import BackendTestMixin
from rhodecode.tests.vcs.conftest import BackendTestMixin
@pytest.fixture
@ -57,6 +57,7 @@ def nodes():
return nodes
@pytest.mark.usefixtures("vcs_repository_support")
class TestInMemoryCommit(BackendTestMixin):
"""
This is a backend independent test case class which should be created

View file

@ -27,7 +27,7 @@ from rhodecode.lib.vcs.nodes import FileNode
from rhodecode.lib.vcs.nodes import Node
from rhodecode.lib.vcs.nodes import NodeError
from rhodecode.lib.vcs.nodes import NodeKind
from rhodecode.tests.vcs.base import BackendTestMixin
from rhodecode.tests.vcs.conftest import BackendTestMixin
@pytest.fixture()
@ -248,7 +248,7 @@ class TestNodeBasics:
assert (1, 1) == py_node.lines(count_empty=True)
class TestNodeContent:
class TestNodeContent(object):
def test_if_binary(self, binary_filenode):
filenode = binary_filenode('calendar.jpg')
@ -263,6 +263,7 @@ class TestNodeContent:
assert tar_node.mimetype == 'application/x-tar'
@pytest.mark.usefixtures("vcs_repository_support")
class TestNodesCommits(BackendTestMixin):
def test_node_last_commit(self, generate_repo_with_commits):

View file

@ -29,9 +29,10 @@ from rhodecode.lib.vcs.backends.base import (
Config, BaseInMemoryCommit, Reference, MergeResponse, MergeFailureReason)
from rhodecode.lib.vcs.exceptions import VCSError, RepositoryError
from rhodecode.lib.vcs.nodes import FileNode
from rhodecode.tests.vcs.base import BackendTestMixin
from rhodecode.tests.vcs.conftest import BackendTestMixin
@pytest.mark.usefixtures("vcs_repository_support")
class TestRepositoryBase(BackendTestMixin):
recreate_repo_per_test = False
@ -148,6 +149,7 @@ class TestRepositoryBase(BackendTestMixin):
assert self.repo._get_url(url)
@pytest.mark.usefixtures("vcs_repository_support")
class TestDeprecatedRepositoryAPI(BackendTestMixin):
recreate_repo_per_test = False
@ -450,6 +452,7 @@ class TestRepositoryMerge:
ref, self, ref, 'workspace_id', 'user name', 'user@email.com')
@pytest.mark.usefixtures("vcs_repository_support")
class TestRepositoryStrip(BackendTestMixin):
recreate_repo_per_test = True

View file

@ -20,7 +20,7 @@
import pytest
from rhodecode.tests.vcs.base import BackendTestMixin
from rhodecode.tests.vcs.conftest import BackendTestMixin
from rhodecode.lib.vcs.exceptions import (
TagAlreadyExistError, TagDoesNotExistError)
@ -28,6 +28,7 @@ from rhodecode.lib.vcs.exceptions import (
pytestmark = pytest.mark.backends("git", "hg")
@pytest.mark.usefixtures("vcs_repository_support")
class TestTags(BackendTestMixin):
def test_new_tag(self):

View file

@ -18,8 +18,8 @@
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import datetime
import os
import datetime
import subprocess32
import pytest
@ -33,7 +33,7 @@ from rhodecode.lib.vcs.utils.paths import get_dirs_for_path
@pytest.mark.usefixtures("baseapp")
class TestPaths:
class TestPaths(object):
def _test_get_dirs_for_path(self, path, expected):
"""
@ -69,7 +69,7 @@ class TestPaths:
assert set(get_scms_for_path(new)) == set(['git', 'hg'])
class TestGetScm:
class TestGetScm(object):
def test_existing_repository(self, vcs_repository_support):
alias, repo = vcs_repository_support
@ -98,7 +98,7 @@ class TestGetScm:
get_scm(tmpdir.strpath)
class TestParseDatetime:
class TestParseDatetime(object):
def test_datetime_text(self):
assert parse_datetime('2010-04-07 21:29:41') == \
@ -186,7 +186,7 @@ class TestParseDatetime:
('Mr Double Name withemail@email.com ',
'Mr Double Name', 'withemail@email.com'),
])
class TestAuthorExtractors:
class TestAuthorExtractors(object):
def test_author_email(self, test_str, name, email):
assert email == author_email(test_str)

View file

@ -8,7 +8,6 @@ use = egg:rhodecode-vcsserver
pyramid.default_locale_name = en
pyramid.includes =
pyramid.reload_templates = true
# default locale used by VCS systems
locale = en_US.UTF-8
@ -22,10 +21,11 @@ beaker.cache.repo_object.expire = 300
beaker.cache.repo_object.enabled = true
[server:main]
use = egg:waitress#main
host = 127.0.0.1
port = 9900
use = egg:gunicorn#main
################################
### LOGGING CONFIGURATION ####
################################