From bf83236cf820798af8d6da1adad2c8b0fca02ecd Mon Sep 17 00:00:00 2001 From: RhodeCode Admin Date: Wed, 8 Jan 2025 13:41:15 +0100 Subject: [PATCH] fix: removed old scripts --- rhodecode/tests/scripts/create_rc.sh | 17 -- rhodecode/tests/scripts/mem_watch | 1 - rhodecode/tests/scripts/test_concurency.py | 201 --------------------- rhodecode/tests/scripts/test_crawler.py | 188 ------------------- 4 files changed, 407 deletions(-) delete mode 100755 rhodecode/tests/scripts/create_rc.sh delete mode 100644 rhodecode/tests/scripts/mem_watch delete mode 100644 rhodecode/tests/scripts/test_concurency.py delete mode 100644 rhodecode/tests/scripts/test_crawler.py diff --git a/rhodecode/tests/scripts/create_rc.sh b/rhodecode/tests/scripts/create_rc.sh deleted file mode 100755 index 90930ca8..00000000 --- a/rhodecode/tests/scripts/create_rc.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/sh -psql -U postgres -h localhost -c 'drop database if exists rhodecode;' -psql -U postgres -h localhost -c 'create database rhodecode;' -rc-setup-app rc.ini --force-yes --user=marcink --password=qweqwe --email=marcin@python-blog.com --repos=/home/marcink/repos --no-public-access -API_KEY=`psql -R " " -A -U postgres -h localhost -c "select api_key from users where admin=TRUE" -d rhodecode | awk '{print $2}'` -echo "run those after running server" -paster serve rc.ini --pid-file=rc.pid --daemon -sleep 3 -rhodecode-api --apikey=$API_KEY --apihost=http://127.0.0.1:5001 create_user username:demo1 password:qweqwe email:demo1@rhodecode.org -rhodecode-api --apikey=$API_KEY --apihost=http://127.0.0.1:5001 create_user username:demo2 password:qweqwe email:demo2@rhodecode.org -rhodecode-api --apikey=$API_KEY --apihost=http://127.0.0.1:5001 create_user username:demo3 password:qweqwe email:demo3@rhodecode.org -rhodecode-api --apikey=$API_KEY --apihost=http://127.0.0.1:5001 create_user_group group_name:demo12 -rhodecode-api --apikey=$API_KEY --apihost=http://127.0.0.1:5001 add_user_to_user_group usergroupid:demo12 userid:demo1 -rhodecode-api --apikey=$API_KEY --apihost=http://127.0.0.1:5001 add_user_to_user_group usergroupid:demo12 userid:demo2 -echo "killing server" -kill `cat rc.pid` -rm rc.pid diff --git a/rhodecode/tests/scripts/mem_watch b/rhodecode/tests/scripts/mem_watch deleted file mode 100644 index dda5dbc2..00000000 --- a/rhodecode/tests/scripts/mem_watch +++ /dev/null @@ -1 +0,0 @@ -ps -eo size,pid,user,command --sort -size | awk '{ hr=$1/1024 ; printf("%13.2f Mb ",hr) } { for ( x=4 ; x<=NF ; x++ ) { printf("%s ",$x) } print "" }' diff --git a/rhodecode/tests/scripts/test_concurency.py b/rhodecode/tests/scripts/test_concurency.py deleted file mode 100644 index 45cb6072..00000000 --- a/rhodecode/tests/scripts/test_concurency.py +++ /dev/null @@ -1,201 +0,0 @@ -# Copyright (C) 2010-2024 RhodeCode GmbH -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License, version 3 -# (only), as published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -# This program is dual-licensed. If you wish to learn more about the -# RhodeCode Enterprise Edition, including its added features, Support services, -# and proprietary license terms, please see https://rhodecode.com/licenses/ - -""" -Test suite for making push/pull operations -""" - -import os -import sys -import shutil -import logging -from os.path import join as jn -from os.path import dirname as dn - -from tempfile import _RandomNameSequence -from subprocess import Popen, PIPE - -from rhodecode.lib.utils2 import engine_from_config -from rhodecode.lib.auth import get_crypt_password -from rhodecode.model import init_model -from rhodecode.model import meta -from rhodecode.model.db import User, Repository - -from rhodecode.tests import TESTS_TMP_PATH, HG_REPO - -rel_path = dn(dn(dn(dn(os.path.abspath(__file__))))) - - -USER = 'test_admin' -PASS = 'test12' -HOST = 'rc.local' -METHOD = 'pull' -DEBUG = True -log = logging.getLogger(__name__) - - -class Command(object): - - def __init__(self, cwd): - self.cwd = cwd - - def execute(self, cmd, *args): - """Runs command on the system with given ``args``. - """ - - command = cmd + ' ' + ' '.join(args) - log.debug('Executing %s', command) - if DEBUG: - print(command) - p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE, cwd=self.cwd) - stdout, stderr = p.communicate() - if DEBUG: - print('{} {}'.format(stdout, stderr)) - return stdout, stderr - - -def get_session(): - conf = {} - engine = engine_from_config(conf, 'sqlalchemy.db1.') - init_model(engine) - sa = meta.Session - return sa - - -def create_test_user(force=True): - print('creating test user') - sa = get_session() - - user = sa.query(User).filter(User.username == USER).scalar() - - if force and user is not None: - print('removing current user') - for repo in sa.query(Repository).filter(Repository.user == user).all(): - sa.delete(repo) - sa.delete(user) - sa.commit() - - if user is None or force: - print('creating new one') - new_usr = User() - new_usr.username = USER - new_usr.password = get_crypt_password(PASS) - new_usr.email = 'mail@mail.com' - new_usr.name = 'test' - new_usr.lastname = 'lasttestname' - new_usr.active = True - new_usr.admin = True - sa.add(new_usr) - sa.commit() - - print('done') - - -def create_test_repo(force=True): - print('creating test repo') - from rhodecode.model.repo import RepoModel - sa = get_session() - - user = sa.query(User).filter(User.username == USER).scalar() - if user is None: - raise Exception('user not found') - - repo = sa.query(Repository).filter(Repository.repo_name == HG_REPO).scalar() - - if repo is None: - print('repo not found creating') - - form_data = {'repo_name': HG_REPO, - 'repo_type': 'hg', - 'private':False, - 'clone_uri': '' } - rm = RepoModel(sa) - rm.base_path = '/home/hg' - rm.create(form_data, user) - - print('done') - - -def get_anonymous_access(): - sa = get_session() - return sa.query(User).filter(User.username == 'default').one().active - - -#============================================================================== -# TESTS -#============================================================================== -def test_clone_with_credentials(repo=HG_REPO, method=METHOD, - seq=None, backend='hg', check_output=True): - cwd = path = jn(TESTS_TMP_PATH, repo) - - if seq is None: - seq = next(_RandomNameSequence()) - - try: - shutil.rmtree(path, ignore_errors=True) - os.makedirs(path) - except OSError: - raise - - clone_url = 'http://%(user)s:%(pass)s@%(host)s/%(cloned_repo)s' % \ - {'user': USER, - 'pass': PASS, - 'host': HOST, - 'cloned_repo': repo, } - - dest = path + seq - if method == 'pull': - stdout, stderr = Command(cwd).execute(backend, method, '--cwd', dest, clone_url) - else: - stdout, stderr = Command(cwd).execute(backend, method, clone_url, dest) - if check_output: - if backend == 'hg': - assert """adding file changes""" in stdout, 'no messages about cloning' - assert """abort""" not in stderr, 'got error from clone' - elif backend == 'git': - assert """Cloning into""" in stdout, 'no messages about cloning' - - -if __name__ == '__main__': - try: - create_test_user(force=False) - seq = None - import time - - try: - METHOD = sys.argv[3] - except Exception: - pass - - try: - backend = sys.argv[4] - except Exception: - backend = 'hg' - - if METHOD == 'pull': - seq = next(_RandomNameSequence()) - test_clone_with_credentials(repo=sys.argv[1], method='clone', - seq=seq, backend=backend) - s = time.time() - for i in range(1, int(sys.argv[2]) + 1): - print('take {}'.format(i)) - test_clone_with_credentials(repo=sys.argv[1], method=METHOD, - seq=seq, backend=backend) - print('time taken %.4f' % (time.time() - s)) - except Exception as e: - sys.exit('stop on %s' % e) diff --git a/rhodecode/tests/scripts/test_crawler.py b/rhodecode/tests/scripts/test_crawler.py deleted file mode 100644 index 4960f4d3..00000000 --- a/rhodecode/tests/scripts/test_crawler.py +++ /dev/null @@ -1,188 +0,0 @@ -# Copyright (C) 2010-2024 RhodeCode GmbH -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License, version 3 -# (only), as published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -# This program is dual-licensed. If you wish to learn more about the -# RhodeCode Enterprise Edition, including its added features, Support services, -# and proprietary license terms, please see https://rhodecode.com/licenses/ - -""" -Test for crawling a project for memory usage -This should be runned just as regular script together -with a watch script that will show memory usage. - -watch -n1 ./rhodecode/tests/mem_watch -""" - - -import cookielib -import urllib.request -import urllib.parse -import urllib.error -import urllib.request -import urllib.error -import urllib.parse -import time -import os -import sys -from os.path import join as jn -from os.path import dirname as dn -from sqlalchemy.util import OrderedSet - -__here__ = os.path.abspath(__file__) -__root__ = dn(dn(dn(__here__))) -sys.path.append(__root__) - -from rhodecode.lib import vcs -from rhodecode.lib.vcs.exceptions import RepositoryError - -PASES = 3 -HOST = 'http://127.0.0.1' -PORT = 5001 -BASE_URI = '%s:%s/' % (HOST, PORT) - -if len(sys.argv) == 2: - BASE_URI = sys.argv[1] - -if not BASE_URI.endswith('/'): - BASE_URI += '/' - -print('Crawling @ %s' % BASE_URI) -BASE_URI += '%s' -PROJECT_PATH = jn('/', 'home', 'marcink', 'repos') -PROJECTS = [ - #'linux-magx-pbranch', - 'CPython', - 'rhodecode_tip', -] - - -cj = cookielib.FileCookieJar('/tmp/rc_test_cookie.txt') -o = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj)) -o.addheaders = [ - ('User-agent', 'rhodecode-crawler'), - ('Accept-Language', 'en - us, en;q = 0.5') -] - -urllib.request.install_opener(o) - - -def _get_repo(proj): - if isinstance(proj, str): - repo = vcs.get_repo(jn(PROJECT_PATH, proj)) - proj = proj - else: - repo = proj - proj = repo.name - - return repo, proj - - -def test_changelog_walk(proj, pages=100): - repo, proj = _get_repo(proj) - - total_time = 0 - for i in range(1, pages): - - page = '/'.join((proj, 'changelog',)) - - full_uri = (BASE_URI % page) + '?' + urllib.parse.urlencode({'page': i}) - s = time.time() - f = o.open(full_uri) - - assert f.url == full_uri, 'URL:%s does not match %s' % (f.url, full_uri) - - size = len(f.read()) - e = time.time() - s - total_time += e - print('visited %s size:%s req:%s ms' % (full_uri, size, e)) - - print('total_time {}'.format(total_time)) - print('average on req {}'.format(total_time / float(pages))) - - -def test_commit_walk(proj, limit=None): - repo, proj = _get_repo(proj) - - print('processing', jn(PROJECT_PATH, proj)) - total_time = 0 - - cnt = 0 - for i in repo: - cnt += 1 - raw_cs = '/'.join((proj, 'changeset', i.raw_id)) - if limit and limit == cnt: - break - - full_uri = (BASE_URI % raw_cs) - print('%s visiting %s\%s' % (cnt, full_uri, i)) - s = time.time() - f = o.open(full_uri) - size = len(f.read()) - e = time.time() - s - total_time += e - print('%s visited %s\%s size:%s req:%s ms' % (cnt, full_uri, i, size, e)) - - print('total_time {}'.format(total_time)) - print('average on req {}'.format(total_time / float(cnt))) - - -def test_files_walk(proj, limit=100): - repo, proj = _get_repo(proj) - - print('processing {}'.format(jn(PROJECT_PATH, proj))) - total_time = 0 - - paths_ = OrderedSet(['']) - try: - tip = repo.get_commit('tip') - for topnode, dirs, files in tip.walk('/'): - - for dir in dirs: - paths_.add(dir.path) - for f in dir: - paths_.add(f.path) - - for f in files: - paths_.add(f.path) - - except RepositoryError as e: - pass - - cnt = 0 - for f in paths_: - cnt += 1 - if limit and limit == cnt: - break - - file_path = '/'.join((proj, 'files', 'tip', f)) - full_uri = (BASE_URI % file_path) - print('%s visiting %s' % (cnt, full_uri)) - s = time.time() - f = o.open(full_uri) - size = len(f.read()) - e = time.time() - s - total_time += e - print('%s visited OK size:%s req:%s ms' % (cnt, size, e)) - - print('total_time {}'.format(total_time)) - print('average on req {}'.format(total_time / float(cnt))) - -if __name__ == '__main__': - for path in PROJECTS: - repo = vcs.get_repo(jn(PROJECT_PATH, path)) - for i in range(PASES): - print('PASS %s/%s' % (i, PASES)) - test_changelog_walk(repo, pages=80) - test_commit_walk(repo, limit=100) - test_files_walk(repo, limit=100)