feat: changed last change of repo group to be commit of a latest repository.

This commit is contained in:
Marcin Kuzminski 2019-05-21 11:47:13 +02:00
parent 2af444c473
commit 11d4863052
28 changed files with 240 additions and 49 deletions

View file

@ -45,7 +45,7 @@ PYRAMID_SETTINGS = {}
EXTENSIONS = {}
__version__ = ('.'.join((str(each) for each in VERSION[:3])))
__dbversion__ = 97 # defines current db version for migrations
__dbversion__ = 98 # defines current db version for migrations
__platform__ = platform.system()
__license__ = 'AGPLv3, and Commercial License'
__author__ = 'RhodeCode GmbH'

View file

@ -703,8 +703,9 @@ class HomeView(BaseAppView):
def repo_group_main_page(self):
c = self.load_default_context()
c.repo_group = self.request.db_repo_group
repo_data, repo_group_data = self._get_groups_and_repos(
c.repo_group.group_id)
repo_data, repo_group_data = self._get_groups_and_repos(c.repo_group.group_id)
c.repo_group.update_commit_cache()
# json used to render the grids
c.repos_data = json.dumps(repo_data)

View file

@ -485,9 +485,6 @@ class DbManage(object):
self.populate_default_permissions()
return fixed
def update_repo_info(self):
RepoModel.update_repoinfo()
def config_prompt(self, test_repo_path='', retries=3):
defaults = self.cli_args
_path = defaults.get('repos_location')

View file

@ -614,7 +614,7 @@ class Repository(Base, BaseModel):
if (cs_cache != self.changeset_cache or not self.changeset_cache):
_default = datetime.datetime.fromtimestamp(0)
last_change = cs_cache.get('date') or _default
log.debug('updated repo %s with new cs cache %s', self.repo_name, cs_cache)
log.debug('updated repo %s with new commit cache %s', self.repo_name, cs_cache)
self.updated_on = last_change
self.changeset_cache = cs_cache
Session().add(self)

View file

@ -2164,7 +2164,7 @@ class Repository(Base, BaseModel):
if is_outdated(cs_cache) or not self.changeset_cache:
_default = datetime.datetime.fromtimestamp(0)
last_change = cs_cache.get('date') or _default
log.debug('updated repo %s with new cs cache %s',
log.debug('updated repo %s with new commit cache %s',
self.repo_name, cs_cache)
self.updated_on = last_change
self.changeset_cache = cs_cache

View file

@ -2230,7 +2230,7 @@ class Repository(Base, BaseModel):
if is_outdated(cs_cache) or not self.changeset_cache:
_default = datetime.datetime.fromtimestamp(0)
last_change = cs_cache.get('date') or _default
log.debug('updated repo %s with new cs cache %s',
log.debug('updated repo %s with new commit cache %s',
self.repo_name, cs_cache)
self.updated_on = last_change
self.changeset_cache = cs_cache

View file

@ -2278,7 +2278,7 @@ class Repository(Base, BaseModel):
# if yes, we use the current timestamp instead. Imagine you get
# old commit pushed 1y ago, we'd set last update 1y to ago.
last_change = _default
log.debug('updated repo %s with new cs cache %s',
log.debug('updated repo %s with new commit cache %s',
self.repo_name, cs_cache)
self.updated_on = last_change
self.changeset_cache = cs_cache

View file

@ -2301,7 +2301,7 @@ class Repository(Base, BaseModel):
# if yes, we use the current timestamp instead. Imagine you get
# old commit pushed 1y ago, we'd set last update 1y to ago.
last_change = _default
log.debug('updated repo %s with new cs cache %s',
log.debug('updated repo %s with new commit cache %s',
self.repo_name, cs_cache)
self.updated_on = last_change
self.changeset_cache = cs_cache

View file

@ -2301,7 +2301,7 @@ class Repository(Base, BaseModel):
# if yes, we use the current timestamp instead. Imagine you get
# old commit pushed 1y ago, we'd set last update 1y to ago.
last_change = _default
log.debug('updated repo %s with new cs cache %s',
log.debug('updated repo %s with new commit cache %s',
self.repo_name, cs_cache)
self.updated_on = last_change
self.changeset_cache = cs_cache

View file

@ -1865,7 +1865,7 @@ class Repository(Base, BaseModel):
if is_outdated(cs_cache) or not self.changeset_cache:
_default = datetime.datetime.fromtimestamp(0)
last_change = cs_cache.get('date') or _default
log.debug('updated repo %s with new cs cache %s',
log.debug('updated repo %s with new commit cache %s',
self.repo_name, cs_cache)
self.updated_on = last_change
self.changeset_cache = cs_cache

View file

@ -1868,7 +1868,7 @@ class Repository(Base, BaseModel):
if is_outdated(cs_cache) or not self.changeset_cache:
_default = datetime.datetime.fromtimestamp(0)
last_change = cs_cache.get('date') or _default
log.debug('updated repo %s with new cs cache %s',
log.debug('updated repo %s with new commit cache %s',
self.repo_name, cs_cache)
self.updated_on = last_change
self.changeset_cache = cs_cache

View file

@ -1867,7 +1867,7 @@ class Repository(Base, BaseModel):
if is_outdated(cs_cache) or not self.changeset_cache:
_default = datetime.datetime.fromtimestamp(0)
last_change = cs_cache.get('date') or _default
log.debug('updated repo %s with new cs cache %s',
log.debug('updated repo %s with new commit cache %s',
self.repo_name, cs_cache)
self.updated_on = last_change
self.changeset_cache = cs_cache

View file

@ -1869,7 +1869,7 @@ class Repository(Base, BaseModel):
if is_outdated(cs_cache) or not self.changeset_cache:
_default = datetime.datetime.fromtimestamp(0)
last_change = cs_cache.get('date') or _default
log.debug('updated repo %s with new cs cache %s',
log.debug('updated repo %s with new commit cache %s',
self.repo_name, cs_cache)
self.updated_on = last_change
self.changeset_cache = cs_cache

View file

@ -1869,7 +1869,7 @@ class Repository(Base, BaseModel):
if is_outdated(cs_cache) or not self.changeset_cache:
_default = datetime.datetime.fromtimestamp(0)
last_change = cs_cache.get('date') or _default
log.debug('updated repo %s with new cs cache %s',
log.debug('updated repo %s with new commit cache %s',
self.repo_name, cs_cache)
self.updated_on = last_change
self.changeset_cache = cs_cache

View file

@ -1912,7 +1912,7 @@ class Repository(Base, BaseModel):
if is_outdated(cs_cache) or not self.changeset_cache:
_default = datetime.datetime.fromtimestamp(0)
last_change = cs_cache.get('date') or _default
log.debug('updated repo %s with new cs cache %s',
log.debug('updated repo %s with new commit cache %s',
self.repo_name, cs_cache)
self.updated_on = last_change
self.changeset_cache = cs_cache

View file

@ -1913,7 +1913,7 @@ class Repository(Base, BaseModel):
if is_outdated(cs_cache) or not self.changeset_cache:
_default = datetime.datetime.fromtimestamp(0)
last_change = cs_cache.get('date') or _default
log.debug('updated repo %s with new cs cache %s',
log.debug('updated repo %s with new commit cache %s',
self.repo_name, cs_cache)
self.updated_on = last_change
self.changeset_cache = cs_cache

View file

@ -2100,7 +2100,7 @@ class Repository(Base, BaseModel):
if is_outdated(cs_cache) or not self.changeset_cache:
_default = datetime.datetime.fromtimestamp(0)
last_change = cs_cache.get('date') or _default
log.debug('updated repo %s with new cs cache %s',
log.debug('updated repo %s with new commit cache %s',
self.repo_name, cs_cache)
self.updated_on = last_change
self.changeset_cache = cs_cache

View file

@ -30,7 +30,7 @@ def upgrade(migrate_engine):
op = Operations(context)
repo_group = db_4_16_0_2.RepoGroup.__table__
with op.batch_alter_table(repo_group.name) as batch_op:
batch_op.add_column(
Column("repo_group_name_hash", String(1024), nullable=True, unique=False))

View file

@ -26,7 +26,7 @@ def upgrade(migrate_engine):
op = Operations(context)
repo_group = db_4_16_0_2.RepoGroup.__table__
with op.batch_alter_table(repo_group.name) as batch_op:
batch_op.alter_column("repo_group_name_hash", nullable=False)

View file

@ -0,0 +1,37 @@
# -*- coding: utf-8 -*-
import logging
from alembic.migration import MigrationContext
from alembic.operations import Operations
from sqlalchemy import Column, LargeBinary
from rhodecode.lib.dbmigrate.versions import _reset_base
from rhodecode.model import init_model_encryption
log = logging.getLogger(__name__)
def upgrade(migrate_engine):
"""
Upgrade operations go here.
Don't create your own engine; bind migrate_engine to your metadata
"""
_reset_base(migrate_engine)
from rhodecode.lib.dbmigrate.schema import db_4_16_0_2
init_model_encryption(db_4_16_0_2)
context = MigrationContext.configure(migrate_engine.connect())
op = Operations(context)
repo_group = db_4_16_0_2.RepoGroup.__table__
with op.batch_alter_table(repo_group.name) as batch_op:
batch_op.add_column(
Column("changeset_cache", LargeBinary(1024), nullable=True))
def downgrade(migrate_engine):
pass

View file

@ -1489,10 +1489,12 @@ def breadcrumb_repo_link(repo):
"""
path = [
link_to(group.name, route_path('repo_group_home', repo_group_name=group.group_name))
link_to(group.name, route_path('repo_group_home', repo_group_name=group.group_name),
title='last change:{}'.format(format_date(group.last_commit_change)))
for group in repo.groups_with_parents
] + [
link_to(repo.just_name, route_path('repo_summary', repo_name=repo.repo_name))
link_to(repo.just_name, route_path('repo_summary', repo_name=repo.repo_name),
title='last change:{}'.format(format_date(repo.last_commit_change)))
]
return literal(' » '.join(path))
@ -1510,11 +1512,13 @@ def breadcrumb_repo_group_link(repo_group):
path = [
link_to(group.name,
route_path('repo_group_home', repo_group_name=group.group_name))
route_path('repo_group_home', repo_group_name=group.group_name),
title='last change:{}'.format(format_date(group.last_commit_change)))
for group in repo_group.parents
] + [
link_to(repo_group.name,
route_path('repo_group_home', repo_group_name=repo_group.group_name))
route_path('repo_group_home', repo_group_name=repo_group.group_name),
title='last change:{}'.format(format_date(repo_group.last_commit_change)))
]
return literal(' » '.join(path))

View file

@ -101,23 +101,34 @@ def parse_datetime(text):
:param text: string of desired date/datetime or something more verbose,
like *yesterday*, *2weeks 3days*, etc.
"""
if not text:
raise ValueError('Wrong date: "%s"' % text)
if isinstance(text, datetime.datetime):
return text
text = text.strip().lower()
INPUT_FORMATS = (
input_formats = (
'%Y-%m-%d %H:%M:%S',
'%Y-%m-%dT%H:%M:%S',
'%Y-%m-%d %H:%M',
'%Y-%m-%dT%H:%M',
'%Y-%m-%d',
'%m/%d/%Y %H:%M:%S',
'%m/%d/%YT%H:%M:%S',
'%m/%d/%Y %H:%M',
'%m/%d/%YT%H:%M',
'%m/%d/%Y',
'%m/%d/%y %H:%M:%S',
'%m/%d/%yT%H:%M:%S',
'%m/%d/%y %H:%M',
'%m/%d/%yT%H:%M',
'%m/%d/%y',
)
for format in INPUT_FORMATS:
for format_def in input_formats:
try:
return datetime.datetime(*time.strptime(text, format)[:6])
return datetime.datetime(*time.strptime(text, format_def)[:6])
except ValueError:
pass

View file

@ -59,7 +59,7 @@ from rhodecode.lib.vcs.backends.base import EmptyCommit, Reference
from rhodecode.lib.utils2 import (
str2bool, safe_str, get_commit_safe, safe_unicode, sha1_safe,
time_to_datetime, aslist, Optional, safe_int, get_clone_url, AttributeDict,
glob2re, StrictAttributeDict, cleaned_uri)
glob2re, StrictAttributeDict, cleaned_uri, datetime_to_time, OrderedDefaultDict)
from rhodecode.lib.jsonalchemy import MutationObj, MutationList, JsonType, \
JsonRaw
from rhodecode.lib.ext_json import json
@ -1730,7 +1730,9 @@ class Repository(Base, BaseModel):
from rhodecode.lib.vcs.backends.base import EmptyCommit
dummy = EmptyCommit().__json__()
if not self._changeset_cache:
return dummy
dummy['source_repo_id'] = self.repo_id
return json.loads(json.dumps(dummy))
try:
return json.loads(self._changeset_cache)
except TypeError:
@ -2182,6 +2184,16 @@ class Repository(Base, BaseModel):
return make_lock, currently_locked, lock_info
@property
def last_commit_change(self):
from rhodecode.lib.vcs.utils.helpers import parse_datetime
empty_date = datetime.datetime.fromtimestamp(0)
date_latest = self.changeset_cache.get('date', empty_date)
try:
return parse_datetime(date_latest)
except Exception:
return empty_date
@property
def last_db_change(self):
return self.updated_on
@ -2275,6 +2287,7 @@ class Repository(Base, BaseModel):
"""
Update cache of last changeset for repository, keys should be::
source_repo_id
short_id
raw_id
revision
@ -2283,7 +2296,6 @@ class Repository(Base, BaseModel):
date
author
:param cs_cache:
"""
from rhodecode.lib.vcs.backends.base import BaseChangeset
if cs_cache is None:
@ -2310,17 +2322,20 @@ class Repository(Base, BaseModel):
if is_outdated(cs_cache) or not self.changeset_cache:
_default = datetime.datetime.utcnow()
last_change = cs_cache.get('date') or _default
if self.updated_on and self.updated_on > last_change:
# we check if last update is newer than the new value
# if yes, we use the current timestamp instead. Imagine you get
# old commit pushed 1y ago, we'd set last update 1y to ago.
last_change = _default
log.debug('updated repo %s with new cs cache %s',
self.repo_name, cs_cache)
self.updated_on = last_change
# we check if last update is newer than the new value
# if yes, we use the current timestamp instead. Imagine you get
# old commit pushed 1y ago, we'd set last update 1y to ago.
last_change_timestamp = datetime_to_time(last_change)
current_timestamp = datetime_to_time(last_change)
if last_change_timestamp > current_timestamp:
cs_cache['date'] = _default
self.changeset_cache = cs_cache
Session().add(self)
Session().commit()
log.debug('updated repo %s with new commit cache %s',
self.repo_name, cs_cache)
else:
log.debug('Skipping update_commit_cache for repo:`%s` '
'commit already with latest changes', self.repo_name)
@ -2489,6 +2504,8 @@ class RepoGroup(Base, BaseModel):
created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
updated_on = Column('updated_on', DateTime(timezone=False), nullable=True, unique=None, default=datetime.datetime.now)
personal = Column('personal', Boolean(), nullable=True, unique=None, default=None)
_changeset_cache = Column(
"changeset_cache", LargeBinary(), nullable=True) # JSON data
repo_group_to_perm = relationship('UserRepoGroupToPerm', cascade='all', order_by='UserRepoGroupToPerm.group_to_perm_id')
users_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all')
@ -2513,6 +2530,29 @@ class RepoGroup(Base, BaseModel):
self._group_name = value
self.group_name_hash = self.hash_repo_group_name(value)
@hybrid_property
def changeset_cache(self):
from rhodecode.lib.vcs.backends.base import EmptyCommit
dummy = EmptyCommit().__json__()
if not self._changeset_cache:
dummy['source_repo_id'] = ''
return json.loads(json.dumps(dummy))
try:
return json.loads(self._changeset_cache)
except TypeError:
return dummy
except Exception:
log.error(traceback.format_exc())
return dummy
@changeset_cache.setter
def changeset_cache(self, val):
try:
self._changeset_cache = json.dumps(val)
except Exception:
log.error(traceback.format_exc())
@validates('group_parent_id')
def validate_group_parent_id(self, key, val):
"""
@ -2608,8 +2648,7 @@ class RepoGroup(Base, BaseModel):
return q.all()
@property
def parents(self):
parents_recursion_limit = 10
def parents(self, parents_recursion_limit = 10):
groups = []
if self.parent_group is None:
return groups
@ -2631,6 +2670,16 @@ class RepoGroup(Base, BaseModel):
groups.insert(0, gr)
return groups
@property
def last_commit_change(self):
from rhodecode.lib.vcs.utils.helpers import parse_datetime
empty_date = datetime.datetime.fromtimestamp(0)
date_latest = self.changeset_cache.get('date', empty_date)
try:
return parse_datetime(date_latest)
except Exception:
return empty_date
@property
def last_db_change(self):
return self.updated_on
@ -2670,7 +2719,7 @@ class RepoGroup(Base, BaseModel):
return cnt + children_count(self)
def _recursive_objects(self, include_repos=True):
def _recursive_objects(self, include_repos=True, include_groups=True):
all_ = []
def _get_members(root_gr):
@ -2680,11 +2729,16 @@ class RepoGroup(Base, BaseModel):
childs = root_gr.children.all()
if childs:
for gr in childs:
all_.append(gr)
if include_groups:
all_.append(gr)
_get_members(gr)
root_group = []
if include_groups:
root_group = [self]
_get_members(self)
return [self] + all_
return root_group + all_
def recursive_groups_and_repos(self):
"""
@ -2698,6 +2752,12 @@ class RepoGroup(Base, BaseModel):
"""
return self._recursive_objects(include_repos=False)
def recursive_repos(self):
"""
Returns all children repositories for this group
"""
return self._recursive_objects(include_groups=False)
def get_new_name(self, group_name):
"""
returns new full group name based on parent and new name
@ -2708,6 +2768,62 @@ class RepoGroup(Base, BaseModel):
self.parent_group else [])
return RepoGroup.url_sep().join(path_prefix + [group_name])
def update_commit_cache(self, config=None):
"""
Update cache of last changeset for newest repository inside this group, keys should be::
source_repo_id
short_id
raw_id
revision
parents
message
date
author
"""
from rhodecode.lib.vcs.utils.helpers import parse_datetime
def repo_groups_and_repos():
all_entries = OrderedDefaultDict(list)
def _get_members(root_gr, pos=0):
for repo in root_gr.repositories:
all_entries[root_gr].append(repo)
# fill in all parent positions
for parent_group in root_gr.parents:
all_entries[parent_group].extend(all_entries[root_gr])
children_groups = root_gr.children.all()
if children_groups:
for cnt, gr in enumerate(children_groups, 1):
_get_members(gr, pos=pos+cnt)
_get_members(root_gr=self)
return all_entries
empty_date = datetime.datetime.fromtimestamp(0)
for repo_group, repos in repo_groups_and_repos().items():
latest_repo_cs_cache = {}
for repo in repos:
repo_cs_cache = repo.changeset_cache
date_latest = latest_repo_cs_cache.get('date', empty_date)
date_current = repo_cs_cache.get('date', empty_date)
current_timestamp = datetime_to_time(parse_datetime(date_latest))
if current_timestamp < datetime_to_time(parse_datetime(date_current)):
latest_repo_cs_cache = repo_cs_cache
latest_repo_cs_cache['source_repo_id'] = repo.repo_id
repo_group.changeset_cache = latest_repo_cs_cache
Session().add(repo_group)
Session().commit()
log.debug('updated repo group %s with new commit cache %s',
repo_group.group_name, latest_repo_cs_cache)
def permissions(self, with_admins=True, with_owner=True,
expand_from_user_groups=False):
"""

View file

@ -192,7 +192,7 @@ class RepoModel(BaseModel):
return repo_log
@classmethod
def update_repoinfo(cls, repositories=None):
def update_commit_cache(cls, repositories=None):
if not repositories:
repositories = Repository.getAll()
for repo in repositories:

View file

@ -309,6 +309,10 @@ class RepoGroupModel(BaseModel):
# trigger the post hook
from rhodecode.lib.hooks_base import log_create_repository_group
repo_group = RepoGroup.get_by_group_name(group_name)
# update repo group commit caches initially
repo_group.update_commit_cache()
log_create_repository_group(
created_by=user.username, **repo_group.get_dict())
@ -686,6 +690,13 @@ class RepoGroupModel(BaseModel):
'revoked permission from usergroup: {} on repogroup: {}'.format(
group_name, repo_group), namespace='security.repogroup')
@classmethod
def update_commit_cache(cls, repo_groups=None):
if not repo_groups:
repo_groups = RepoGroup.getAll()
for repo_group in repo_groups:
repo_group.update_commit_cache()
def get_repo_groups_as_dict(self, repo_group_list=None, admin=False,
super_user_actions=False):
@ -707,6 +718,11 @@ class RepoGroupModel(BaseModel):
(datetime.datetime.now() - datetime.datetime.utcnow()).seconds)
return _render("last_change", last_change)
def last_rev(repo_name, cs_cache):
return _render('revision', repo_name, cs_cache.get('revision'),
cs_cache.get('raw_id'), cs_cache.get('author'),
cs_cache.get('message'), cs_cache.get('date'))
def desc(desc, personal):
return _render(
'repo_group_desc', desc, personal, c.visual.stylify_metatags)
@ -723,13 +739,15 @@ class RepoGroupModel(BaseModel):
repo_group_data = []
for group in repo_group_list:
cs_cache = group.changeset_cache
last_repo_name = cs_cache.get('source_repo_name')
row = {
"menu": quick_menu(group.group_name),
"name": repo_group_lnk(group.group_name),
"name_raw": group.group_name,
"last_change": last_change(group.last_db_change),
"last_change_raw": datetime_to_time(group.last_db_change),
"last_change": last_change(group.last_commit_change),
"last_change_raw": datetime_to_time(group.last_commit_change),
"last_changeset": "",
"last_changeset_raw": "",

View file

@ -292,7 +292,7 @@
position: absolute;
top: 100%;
left: 0;
min-width: 150px;
min-width: 180px;
margin: 2px 0 0;
padding: 0;
text-align: left;

View file

@ -1,10 +1,16 @@
<%namespace name="base" file="/base/base.mako"/>
<%
source_repo_id = c.repo_group.changeset_cache.get('source_repo_id')
elems = [
(_('Repository Group ID'), c.repo_group.group_id, '', ''),
(_('Owner'), lambda:base.gravatar_with_user(c.repo_group.user.email), '', ''),
(_('Created on'), h.format_date(c.repo_group.created_on), '', ''),
(_('Updated on'), h.format_date(c.repo_group.updated_on), '', ''),
(_('Cached Commit date'), (c.repo_group.changeset_cache.get('date')), '', ''),
(_('Cached Commit repo_id'), (h.link_to_if(source_repo_id, source_repo_id, h.route_path('repo_summary', repo_name='_{}'.format(source_repo_id)))), '', ''),
(_('Is Personal Group'), c.repo_group.personal or False, '', ''),
(_('Total repositories'), c.repo_group.repositories_recursive_count, '', ''),

View file

@ -7,6 +7,7 @@
(_('Created on'), h.format_date(c.rhodecode_db_repo.created_on), '', ''),
(_('Updated on'), h.format_date(c.rhodecode_db_repo.updated_on), '', ''),
(_('Cached Commit id'), lambda: h.link_to(c.rhodecode_db_repo.changeset_cache.get('short_id'), h.route_path('repo_commit',repo_name=c.repo_name,commit_id=c.rhodecode_db_repo.changeset_cache.get('raw_id'))), '', ''),
(_('Cached Commit date'), c.rhodecode_db_repo.changeset_cache.get('date'), '', ''),
(_('Attached scoped tokens'), len(c.rhodecode_db_repo.scoped_tokens), '', [x.user for x in c.rhodecode_db_repo.scoped_tokens]),
(_('Pull requests source'), len(c.rhodecode_db_repo.pull_requests_source), '', ['pr_id:{}, repo:{}'.format(x.pull_request_id,x.source_repo.repo_name) for x in c.rhodecode_db_repo.pull_requests_source]),
(_('Pull requests target'), len(c.rhodecode_db_repo.pull_requests_target), '', ['pr_id:{}, repo:{}'.format(x.pull_request_id,x.target_repo.repo_name) for x in c.rhodecode_db_repo.pull_requests_target]),