engineering/unrhodecode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

packaging: switched to nix 20.03 packages and latest pip2nix code

Browse source 
- dependencies changed:
  atomicwrites==1.4.0
  ipython==5.10.0
  pytest==4.6.9
  py==1.8.1
  pytest-cov==2.8.1
  pytest-sugar==0.9.3
  pytest-runner==5.2.0
  rhodecode-tools==2.0.0
  ipython==5.10.0
  mysqlclient==1.4.6 (change from mysql-python, which is deprecated)
  mako==1.1.2
  lxml==4.5.0
  click==7.1.2
This commit is contained in:
Marcin Kuzminski 2020-05-11 14:20:30 +02:00
parent 3def14a8c3
commit c4bd55edf6
13 changed files with 2302 additions and 1108 deletions
Download patch file Download diff file Expand all files Collapse all files

1
MANIFEST.in
View file

@ -48,5 +48,6 @@ recursive-include rhodecode/public/js *
recursive-include rhodecode/templates *
# skip any tests files
recursive-exclude result *
recursive-exclude rhodecode/tests *

2
Makefile
View file

@ -54,7 +54,7 @@ web-build:
NODE_PATH=$(NODE_PATH) $(GRUNT)
generate-pkgs:
nix-shell pkgs/shell-generate.nix --command "pip2nix generate --licenses"
nix-shell pkgs/shell-generate.nix --command "pip2nix generate --licenses --no-binary :all:"
pip-packages:
python ${OUTDATED_PACKAGES}

50
default.nix
View file

@ -47,20 +47,22 @@ let
# Evaluates to the last segment of a file system path.
basename = path: with pkgs.lib; last (splitString "/" path);
startsWith = prefix: full: let
actualPrefix = builtins.substring 0 (builtins.stringLength prefix) full;
in actualPrefix == prefix;
# source code filter used as arugment to builtins.filterSource.
src-filter = path: type: with pkgs.lib;
let
ext = last (splitString "." path);
parts = last (splitString "/" path);
in
!builtins.elem (basename path) [
".git" ".hg" "__pycache__" ".eggs" ".idea" ".dev"
"node_modules" "node_binaries"
"build" "data" "result" "tmp"] &&
!builtins.elem ext ["egg-info" "pyc"] &&
# TODO: johbo: This check is wrong, since "path" contains an absolute path,
# it would still be good to restore it since we want to ignore "result-*".
!hasPrefix "result" path;
!startsWith "result" (basename path);
sources =
let
@ -69,6 +71,7 @@ let
all
isString
attrValues;
sourcesConfig = pkgs.config.rc.sources or {};
in
# Ensure that sources are configured as strings. Using a path
@ -153,16 +156,23 @@ let
propagatedBuildInputs =
attrs.propagatedBuildInputs or [] ++ [
];
];
LC_ALL = "en_US.UTF-8";
LOCALE_ARCHIVE =
if pkgs.stdenv.isLinux
then "${pkgs.glibcLocales}/lib/locale/locale-archive"
else "";
preBuild = ''
export NIX_PATH=nixpkgs=${pkgs.path}
export SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt
echo "[BEGIN]: Building frontend assets"
${linkNodePackages}
make web-build
rm -fr node_modules
rm -fr node_binaries
echo "[DONE ]: Building frontend assets"
'';
# Add bin directory to path so that tests can find 'rhodecode'.
preCheck = ''
echo "Expanding PATH with $out/bin directory"
export PATH="$out/bin:$PATH"
'';
@ -178,15 +188,6 @@ let
rm -rf $out/lib/${self.python.libPrefix}/site-packages/rhodecode/tests
'';
preBuild = ''
echo "[BEGIN]: Building frontend assets"
${linkNodePackages}
make web-build
rm -fr node_modules
rm -fr node_binaries
echo "[DONE ]: Building frontend assets"
'';
postInstall = ''
# check required files
STATIC_CHECK="/robots.txt /502.html
@ -247,7 +248,6 @@ let
done
echo "[DONE ]: enterprise-ce binary wrapping"
# rhodecode-tools don't need wrapping
ln -s ${self.rhodecode-tools}/bin/rhodecode-* $out/bin/
@ -259,10 +259,11 @@ let
chmod 755 -R $out/etc/static
'';
});
});
};
basePythonPackages = with builtins;
if isAttrs pythonPackages then
pythonPackages
@ -270,13 +271,8 @@ let
getAttr pythonPackages pkgs;
pythonGeneratedPackages = import ./pkgs/python-packages.nix {
inherit
pkgs;
inherit
(pkgs)
fetchurl
fetchgit
fetchhg;
inherit pkgs;
inherit (pkgs) fetchurl fetchgit fetchhg;
};
pythonCommunityOverrides = import ./pkgs/python-packages-overrides.nix {

2
package.json
View file

@ -1,6 +1,6 @@
{
"name": "rhodecode-enterprise",
"version": "2.0.0",
"version": "4.19.0",
"private": true,
"description" : "RhodeCode JS packaged",
"license": "SEE LICENSE IN LICENSE.txt",

4
pkgs/node-default.nix
View file

@ -1,8 +1,8 @@
# This file has been generated by node2nix 1.6.0. Do not edit!
# This file has been generated by node2nix 1.8.0. Do not edit!
{pkgs ? import <nixpkgs> {
inherit system;
}, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-8_x"}:
}, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-12_x"}:
let
nodeEnv = import ./node-env.nix {

172
pkgs/python-packages-overrides.nix
View file

@ -16,18 +16,18 @@ let
fullName = "Repoze License";
url = http://www.repoze.org/LICENSE.txt;
};
unlicense = {
spdxId = "Unlicense";
fullName = "The Unlicense";
url = http://unlicense.org/;
};
};
in
self: super: {
"appenlight-client" = super."appenlight-client".override (attrs: {
meta = {
license = [ pkgs.lib.licenses.bsdOriginal ];
};
});
"beaker" = super."beaker".override (attrs: {
patches = [
./patches/beaker/patch-beaker-lock-func-debug.diff
@ -36,39 +36,27 @@ self: super: {
});
"cffi" = super."cffi".override (attrs: {
buildInputs = [
buildInputs = with self; attrs.buildInputs ++ [
pkgs.libffi
];
});
"cryptography" = super."cryptography".override (attrs: {
buildInputs = [
buildInputs = with self; attrs.buildInputs ++ [
pkgs.openssl
];
});
"gevent" = super."gevent".override (attrs: {
propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
propagatedBuildInputs = with self; attrs.propagatedBuildInputs ++ [
# NOTE: (marcink) odd requirements from gevent aren't set properly,
# thus we need to inject psutil manually
self."psutil"
];
});
"future" = super."future".override (attrs: {
meta = {
license = [ pkgs.lib.licenses.mit ];
};
});
"testpath" = super."testpath".override (attrs: {
meta = {
license = [ pkgs.lib.licenses.mit ];
};
});
"gnureadline" = super."gnureadline".override (attrs: {
buildInputs = [
buildInputs = with self; attrs.buildInputs ++ [
pkgs.ncurses
];
patchPhase = ''
@ -77,7 +65,7 @@ self: super: {
});
"gunicorn" = super."gunicorn".override (attrs: {
propagatedBuildInputs = [
propagatedBuildInputs = with self; attrs.propagatedBuildInputs ++ [
# johbo: futures is needed as long as we are on Python 2, otherwise
# gunicorn explodes if used with multiple threads per worker.
self."futures"
@ -85,41 +73,48 @@ self: super: {
});
"nbconvert" = super."nbconvert".override (attrs: {
propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
propagatedBuildInputs = with self; attrs.propagatedBuildInputs ++ [
# marcink: plug in jupyter-client for notebook rendering
self."jupyter-client"
];
});
"ipython" = super."ipython".override (attrs: {
propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
propagatedBuildInputs = with self; attrs.propagatedBuildInputs ++ [
self."gnureadline"
];
});
"lxml" = super."lxml".override (attrs: {
buildInputs = [
nativeBuildInputs = with self; attrs.nativeBuildInputs ++ [
pkgs.libxml2.dev
pkgs.libxslt.dev
self.cython
];
buildInputs = with self; attrs.buildInputs ++ [
pkgs.libxml2
pkgs.libxslt
];
propagatedBuildInputs = [
# Needed, so that "setup.py bdist_wheel" does work
self."wheel"
];
});
"mysql-python" = super."mysql-python".override (attrs: {
buildInputs = [
pkgs.openssl
];
propagatedBuildInputs = [
pkgs.libmysql
pkgs.zlib
];
# propagatedBuildInputs = [
# # Needed, so that "setup.py bdist_wheel" does work
# self."wheel"
# ];
});
"mysqlclient" = super."mysqlclient".override (attrs: {
nativeBuildInputs = with self; attrs.nativeBuildInputs ++ [
pkgs.libmysqlclient
pkgs.openssl
pkgs.zlib
];
});
"psycopg2" = super."psycopg2".override (attrs: {
propagatedBuildInputs = [
nativeBuildInputs = with self; attrs.nativeBuildInputs ++ [
pkgs.postgresql
];
meta = {
@ -128,7 +123,12 @@ self: super: {
});
"pycurl" = super."pycurl".override (attrs: {
propagatedBuildInputs = [
nativeBuildInputs = with self; attrs.nativeBuildInputs ++ [
pkgs.curl
pkgs.openssl
];
propagatedBuildInputs = with self; attrs.propagatedBuildInputs ++ [
pkgs.curl
pkgs.openssl
];
@ -143,20 +143,8 @@ self: super: {
};
});
"pyramid" = super."pyramid".override (attrs: {
meta = {
license = localLicenses.repoze;
};
});
"pyramid-debugtoolbar" = super."pyramid-debugtoolbar".override (attrs: {
meta = {
license = [ pkgs.lib.licenses.bsdOriginal localLicenses.repoze ];
};
});
"pysqlite" = super."pysqlite".override (attrs: {
propagatedBuildInputs = [
propagatedBuildInputs = with self; attrs.propagatedBuildInputs ++ [
pkgs.sqlite
];
meta = {
@ -173,7 +161,7 @@ self: super: {
});
"python-pam" = super."python-pam".override (attrs: {
propagatedBuildInputs = [
propagatedBuildInputs = with self; attrs.propagatedBuildInputs ++ [
pkgs.pam
];
@ -187,18 +175,27 @@ self: super: {
});
"python-saml" = super."python-saml".override (attrs: {
buildInputs = [
nativeBuildInputs = with self; attrs.nativeBuildInputs ++ [
pkgs.libxml2
pkgs.libxslt
];
});
"dm.xmlsec.binding" = super."dm.xmlsec.binding".override (attrs: {
buildInputs = [
nativeBuildInputs = with self; attrs.nativeBuildInputs ++ [
pkgs.libxml2.dev
pkgs.libxslt.dev
pkgs.xmlsec
pkgs.libtool
pkgs.zlib
];
buildInputs = with self; attrs.buildInputs ++ [
pkgs.libxml2
pkgs.libxslt
pkgs.xmlsec
pkgs.libtool
pkgs.zlib
];
});
@ -208,13 +205,46 @@ self: super: {
];
});
"supervisor" = super."supervisor".override (attrs: {
patches = [
./patches/supervisor/patch-rlimits-old-kernel.diff
];
});
"py" = super."py".override (attrs: {
buildInputs = with self; attrs.buildInputs ++ [
self."setuptools-scm"
];
});
"importlib-metadata" = super."importlib-metadata".override (attrs: {
buildInputs = with self; attrs.buildInputs ++ [
self."setuptools-scm"
];
});
# License fixes
"appenlight-client" = super."appenlight-client".override (attrs: {
meta = {
license = [ pkgs.lib.licenses.bsdOriginal ];
};
});
"urlobject" = super."urlobject".override (attrs: {
meta = {
license = {
spdxId = "Unlicense";
fullName = "The Unlicense";
url = http://unlicense.org/;
};
license = localLicenses.unlicense;
};
});
"future" = super."future".override (attrs: {
meta = {
license = [ pkgs.lib.licenses.mit ];
};
});
"testpath" = super."testpath".override (attrs: {
meta = {
license = [ pkgs.lib.licenses.mit ];
};
});
@ -242,6 +272,18 @@ self: super: {
};
});
"pyramid" = super."pyramid".override (attrs: {
meta = {
license = localLicenses.repoze;
};
});
"pyramid-debugtoolbar" = super."pyramid-debugtoolbar".override (attrs: {
meta = {
license = [ pkgs.lib.licenses.bsdOriginal localLicenses.repoze ];
};
});
"repoze.lru" = super."repoze.lru".override (attrs: {
meta = {
license = localLicenses.repoze;
@ -266,12 +308,6 @@ self: super: {
};
});
"supervisor" = super."supervisor".override (attrs: {
patches = [
./patches/supervisor/patch-rlimits-old-kernel.diff
];
});
# Avoid that base packages screw up the build process
inherit (basePythonPackages)
setuptools;

3074
pkgs/python-packages.nix
View file

File diff suppressed because it is too large Load diff

27
pkgs/shell-generate.nix
View file

@ -4,7 +4,11 @@
with pkgs.lib;
let _pythonPackages = pythonPackages; in
let
_pythonPackages = pythonPackages;
in
let
pythonPackages = getAttr _pythonPackages pkgs;
@ -18,16 +22,21 @@ in
pkgs.stdenv.mkDerivation {
name = "pip2nix-generated";
buildInputs = [
# Allows to generate python packages
pip2nix.pip2nix
pythonPackages.pip-tools
# compile using ffi
pkgs.libffi
pythonPackages.cython
# Allows to generate node dependencies
pkgs.nodePackages.node2nix
# We need mysql_config to be around
pkgs.mysql
pkgs.libmysqlclient
# We need postgresql to be around
pkgs.postgresql
@ -41,6 +50,12 @@ pkgs.stdenv.mkDerivation {
pkgs.curl
];
LC_ALL = "en_US.UTF-8";
LOCALE_ARCHIVE =
if pkgs.stdenv.isLinux
then "${pkgs.glibcLocales}/lib/locale/locale-archive"
else "";
shellHook = ''
runHook preShellHook
runHook postShellHook
@ -48,8 +63,14 @@ pkgs.stdenv.mkDerivation {
preShellHook = ''
echo "Starting Generate Shell"
# set unpack source date to 1980 to fix ZIP problems that does not support <1980
export SOURCE_DATE_EPOCH=315532800
export TMPDIR=/tmp
# Custom prompt to distinguish from other dev envs.
export PS1="\n\[\033[1;32m\][Generate-shell:\w]$\[\033[0m\] "
export PS1="\n\[\033[1;32m\][pip2nix-generate-shell]$\[\033[0m\] "
export PYCURL_SSL_LIBRARY=openssl
'';
}

14
requirements.txt
View file

@ -6,12 +6,13 @@ beaker==1.9.1
bleach==3.1.3
celery==4.3.0
channelstream==0.5.2
click==7.0
click==7.1.2
colander==1.7.0
# our custom configobj
https://code.rhodecode.com/upstream/configobj/artifacts/download/0-012de99a-b1e1-4f64-a5c0-07a98a41b324.tar.gz?md5=6a513f51fe04b2c18cf84c1395a7c626#egg=configobj==5.0.6
cssselect==1.0.3
cryptography==2.6.1
cython==0.29.17
decorator==4.1.2
deform==2.0.8
docutils==0.16.0
@ -24,8 +25,8 @@ infrae.cache==1.0.1
iso8601==0.1.12
itsdangerous==0.24
kombu==4.6.6
lxml==4.2.5
mako==1.1.0
lxml==4.5.0
mako==1.1.2
markdown==2.6.11
markupsafe==1.1.1
msgpack-python==0.5.6
@ -81,7 +82,7 @@ zope.event==4.4.0
zope.interface==4.6.0
# DB drivers
mysql-python==1.2.5
mysqlclient==1.4.6
pymysql==0.8.1
pysqlite==2.8.3
psycopg2==2.8.4
@ -107,11 +108,10 @@ waitress==1.3.1
## debug
ipdb==0.13.2
ipython==5.1.0
ipython==5.10.0
## rhodecode-tools, special case, use file://PATH.tar.gz#egg=rhodecode-tools==X.Y.Z, to test local version
https://code.rhodecode.com/rhodecode-tools-ce/artifacts/download/0-ed54e749-2ef5-4bc7-ae7f-7900e3c2aa15.tar.gz?sha256=76f024bad3a1e55fdb3d64f13f5b77ff21a12fee699918de2110fe21effd5a3a#egg=rhodecode-tools==1.4.0
https://code.rhodecode.com/rhodecode-tools-ce/artifacts/download/0-0cf09d55-fcb5-4ab0-ad48-e71f65090875.tar.gz?sha256=ae458b6845f278aed1bcb90939ef01cdb581016ba0e7f58602d1ac6513f94707#egg=rhodecode-tools==2.0.0
## appenlight
appenlight-client==0.6.26

10
requirements_pinned.txt
View file

@ -1,6 +1,6 @@
# contains not directly required libraries we want to pin the version.
atomicwrites==1.3.0
atomicwrites==1.4.0
attrs==19.3.0
asn1crypto==0.24.0
billiard==3.6.1.0
@ -21,7 +21,13 @@ pyramid-jinja2==2.7
scandir==1.10.0
setproctitle==1.1.10
tempita==0.5.2
testpath==0.4.4
transaction==2.4.0
vine==1.3.0
wcwidth==0.1.9
# ptyprocess backport, pypi version doesn't support egg/source installs on python 2.X
https://code.rhodecode.com/upstream/ptyprocess/artifacts/download/0-c8b019b1-c4d3-46ac-a0ad-1206ec3fb3cb.tar.gz?sha256=50394f2c5e117fcab4360bf99c8bc40be7211ee1a5860aeb3809b44249550c3e#egg=ptyprocess==0.6.0.rhodecode-upstream1
# testpath backport, pypi version doesn't support egg/source installs on python 2.X
https://code.rhodecode.com/upstream/testpath/artifacts/download/0-618e6b32-6ca5-428a-bda0-494bb347a56d.tar.gz?sha256=fd95bafd89ee2fb2bb0d82be34c9c5bba3a290f52cafc67a12a74ef825527019#egg=testpath==0.4.4.rhodecode-upstream1

10
requirements_test.txt
View file

@ -1,9 +1,9 @@
# test related requirements
pytest==4.6.5
py==1.8.0
pytest-cov==2.7.1
pytest-sugar==0.9.2
pytest-runner==5.1.0
pytest==4.6.9
py==1.8.1
pytest-cov==2.8.1
pytest-sugar==0.9.3
pytest-runner==5.2.0
pytest-profiling==1.7.0
pytest-timeout==1.3.3
gprof2dot==2017.9.19

4
setup.cfg
View file

@ -1,10 +1,6 @@
[aliases]
test = pytest
[egg_info]
tag_build =
tag_svn_revision = false
# Babel configuration
[compile_catalog]
domain = rhodecode

40
shell.nix
View file

@ -27,12 +27,12 @@ let
optionalDevelopInstall = attributeName:
let
path = pkgs.lib.attrByPath [attributeName] null sources;
doIt = doDevelopInstall && path != null;
doDI = doDevelopInstall && path != null;
in
# do develop installation with empty hosts to skip any package duplicates to
# be replaced. This only pushes the package to be locally available
pkgs.lib.optionalString doIt (''
pkgs.lib.optionalString doDI (''
echo "[BEGIN] Develop install of '${attributeName}' from '${path}'"
pushd ${path}
python setup.py develop --prefix $tmp_path --allow-hosts ""
@ -47,8 +47,8 @@ let
optionalDevelopInstallBuildInputs = attributeName:
let
path = pkgs.lib.attrByPath [attributeName] null sources;
doIt = doDevelopInstall && path != null && pkgs.lib.pathExists "${nixFile}";
nixFile = "${path}/default.nix";
doDI = doDevelopInstall && path != null && pkgs.lib.pathExists "${nixFile}";
derivate = import "${nixFile}" {
inherit
@ -56,13 +56,30 @@ let
pythonPackages;
};
in
pkgs.lib.lists.optionals doIt (
pkgs.lib.lists.optionals doDI (
derivate.propagatedBuildInputs
);
optionalBinDeps = attributeName:
let
path = pkgs.lib.attrByPath [attributeName] null sources;
nixFile = "${path}/default.nix";
doDI = doDevelopInstall && path != null && pkgs.lib.pathExists "${nixFile}";
derivate = import "${nixFile}" {
inherit
doCheck
pythonPackages;
};
in
pkgs.lib.optionalString doDI (''
echo "Wrapping PATH with vcsserver vcs binaries"
export PATH="${derivate.vcs_pkgs.subversion}/bin:${derivate.vcs_pkgs.git}/bin:${derivate.pythonPackages.mercurial}/bin:$PATH"
'');
developInstalls = [ "rhodecode-vcsserver" ];
in enterprise-ce.override (attrs: {
in enterprise-ce.override (attrs: rec {
# Avoid that we dump any sources into the store when entering the shell and
# make development a little bit more convenient.
src = null;
@ -90,10 +107,14 @@ in enterprise-ce.override (attrs: {
'';
preShellHook = ''
echo "Entering CE-Shell"
echo "Entering rhodecode-ce"
# Custom prompt to distinguish from other dev envs.
export PS1="\n\[\033[1;32m\][CE-shell:\w]$\[\033[0m\] "
export PS1="\n\[\033[1;32m\][rhodecode-ce-shell:\w]$\[\033[0m\] "
# Set locale
export LOCALE_ARCHIVE="${pkgs.glibcLocales}/lib/locale/locale-archive"
export LC_ALL="en_US.UTF-8"
echo "Building frontend assets"
${enterprise-ce.linkNodePackages}
@ -111,9 +132,8 @@ in enterprise-ce.override (attrs: {
postShellHook = ''
echo "** Additional develop installs **"
'' +
pkgs.lib.strings.concatMapStrings optionalDevelopInstall developInstalls
+ ''
'' + pkgs.lib.strings.concatMapStrings optionalDevelopInstall developInstalls + ''
'' + pkgs.lib.strings.concatMapStrings optionalBinDeps [ "rhodecode-vcsserver" ] + ''
'';
})