packaging: switched to nix 20.03 packages and latest pip2nix code

- dependencies changed:
  atomicwrites==1.4.0
  ipython==5.10.0
  pytest==4.6.9
  py==1.8.1
  pytest-cov==2.8.1
  pytest-sugar==0.9.3
  pytest-runner==5.2.0
  rhodecode-tools==2.0.0
  ipython==5.10.0
  mysqlclient==1.4.6 (change from mysql-python, which is deprecated)
  mako==1.1.2
  lxml==4.5.0
  click==7.1.2

MANIFEST.in

@@ -48,5 +48,6 @@ recursive-include rhodecode/public/js *
 recursive-include rhodecode/templates *
 
 # skip any tests files
+recursive-exclude result *
 recursive-exclude rhodecode/tests *
 

Makefile

@@ -54,7 +54,7 @@ web-build:
 	NODE_PATH=$(NODE_PATH) $(GRUNT)
 
 generate-pkgs:
-	nix-shell pkgs/shell-generate.nix --command "pip2nix generate --licenses"
+	nix-shell pkgs/shell-generate.nix --command "pip2nix generate --licenses --no-binary :all:"
 
 pip-packages:
 	python ${OUTDATED_PACKAGES}

default.nix

@@ -47,20 +47,22 @@ let
 
   # Evaluates to the last segment of a file system path.
   basename = path: with pkgs.lib; last (splitString "/" path);
+  startsWith = prefix: full: let
+    actualPrefix = builtins.substring 0 (builtins.stringLength prefix) full;
+  in actualPrefix == prefix;
 
   # source code filter used as arugment to builtins.filterSource.
   src-filter = path: type: with pkgs.lib;
     let
       ext = last (splitString "." path);
+      parts = last (splitString "/" path);
     in
       !builtins.elem (basename path) [
         ".git" ".hg" "__pycache__" ".eggs" ".idea" ".dev"
         "node_modules" "node_binaries"
         "build" "data" "result" "tmp"] &&
       !builtins.elem ext ["egg-info" "pyc"] &&
-      # TODO: johbo: This check is wrong, since "path" contains an absolute path,
+      !startsWith "result" (basename path);
-      # it would still be good to restore it since we want to ignore "result-*".
-      !hasPrefix "result" path;
 
   sources =
     let
@@ -69,6 +71,7 @@ let
         all
         isString
         attrValues;
+
       sourcesConfig = pkgs.config.rc.sources or {};
     in
       # Ensure that sources are configured as strings. Using a path
@@ -155,14 +158,21 @@ let
 
       ];
 
-      LC_ALL = "en_US.UTF-8";
+      preBuild = ''
-      LOCALE_ARCHIVE =
+        export NIX_PATH=nixpkgs=${pkgs.path}
-        if pkgs.stdenv.isLinux
+        export SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt
-        then "${pkgs.glibcLocales}/lib/locale/locale-archive"
+
-        else "";
+        echo "[BEGIN]: Building frontend assets"
+        ${linkNodePackages}
+        make web-build
+        rm -fr node_modules
+        rm -fr node_binaries
+        echo "[DONE ]: Building frontend assets"
+      '';
 
       # Add bin directory to path so that tests can find 'rhodecode'.
       preCheck = ''
+        echo "Expanding PATH with $out/bin directory"
         export PATH="$out/bin:$PATH"
       '';
 
@@ -178,15 +188,6 @@ let
         rm -rf $out/lib/${self.python.libPrefix}/site-packages/rhodecode/tests
       '';
 
-      preBuild = ''
-        echo "[BEGIN]: Building frontend assets"
-        ${linkNodePackages}
-        make web-build
-        rm -fr node_modules
-        rm -fr node_binaries
-        echo "[DONE ]: Building frontend assets"
-      '';
-
       postInstall = ''
         # check required files
         STATIC_CHECK="/robots.txt /502.html
@@ -247,7 +248,6 @@ let
         done
 
         echo "[DONE ]: enterprise-ce binary wrapping"
-
         # rhodecode-tools don't need wrapping
         ln -s ${self.rhodecode-tools}/bin/rhodecode-* $out/bin/
 
@@ -259,10 +259,11 @@ let
         chmod 755 -R $out/etc/static
 
       '';
-    });
 
+    });
   };
 
+
   basePythonPackages = with builtins;
     if isAttrs pythonPackages then
       pythonPackages
@@ -270,13 +271,8 @@ let
       getAttr pythonPackages pkgs;
 
   pythonGeneratedPackages = import ./pkgs/python-packages.nix {
-    inherit
+    inherit pkgs;
-      pkgs;
+    inherit (pkgs) fetchurl fetchgit fetchhg;
-    inherit
-      (pkgs)
-      fetchurl
-      fetchgit
-      fetchhg;
   };
 
   pythonCommunityOverrides = import ./pkgs/python-packages-overrides.nix {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rhodecode-enterprise",
-  "version": "2.0.0",
+  "version": "4.19.0",
   "private": true,
   "description" : "RhodeCode JS packaged",
   "license": "SEE LICENSE IN LICENSE.txt",

pkgs/node-default.nix

@@ -1,8 +1,8 @@
-# This file has been generated by node2nix 1.6.0. Do not edit!
+# This file has been generated by node2nix 1.8.0. Do not edit!
 
 {pkgs ? import <nixpkgs> {
     inherit system;
-  }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-8_x"}:
+  }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-12_x"}:
 
 let
   nodeEnv = import ./node-env.nix {

pkgs/python-packages-overrides.nix

@@ -16,18 +16,18 @@ let
       fullName = "Repoze License";
       url =  http://www.repoze.org/LICENSE.txt;
     };
+
+    unlicense = {
+      spdxId = "Unlicense";
+      fullName = "The Unlicense";
+      url = http://unlicense.org/;
+    };
   };
 
 in
 
 self: super: {
 
-  "appenlight-client" = super."appenlight-client".override (attrs: {
-    meta = {
-      license = [ pkgs.lib.licenses.bsdOriginal ];
-    };
-  });
-
   "beaker" = super."beaker".override (attrs: {
     patches = [
       ./patches/beaker/patch-beaker-lock-func-debug.diff
@@ -36,39 +36,27 @@ self: super: {
   });
 
   "cffi" = super."cffi".override (attrs: {
-    buildInputs = [
+    buildInputs = with self; attrs.buildInputs ++ [
       pkgs.libffi
     ];
   });
 
   "cryptography" = super."cryptography".override (attrs: {
-    buildInputs = [
+    buildInputs = with self; attrs.buildInputs ++ [
       pkgs.openssl
     ];
   });
 
   "gevent" = super."gevent".override (attrs: {
-    propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
+    propagatedBuildInputs = with self; attrs.propagatedBuildInputs ++ [
       # NOTE: (marcink) odd requirements from gevent aren't set properly,
       # thus we need to inject psutil manually
       self."psutil"
     ];
   });
 
-  "future" = super."future".override (attrs: {
-    meta = {
-      license = [ pkgs.lib.licenses.mit ];
-    };
-  });
-
-  "testpath" = super."testpath".override (attrs: {
-    meta = {
-      license = [ pkgs.lib.licenses.mit ];
-    };
-  });
-
   "gnureadline" = super."gnureadline".override (attrs: {
-    buildInputs = [
+    buildInputs = with self; attrs.buildInputs ++ [
       pkgs.ncurses
     ];
     patchPhase = ''
@@ -77,7 +65,7 @@ self: super: {
   });
 
   "gunicorn" = super."gunicorn".override (attrs: {
-    propagatedBuildInputs = [
+    propagatedBuildInputs = with self; attrs.propagatedBuildInputs ++ [
       # johbo: futures is needed as long as we are on Python 2, otherwise
       # gunicorn explodes if used with multiple threads per worker.
       self."futures"
@@ -85,41 +73,48 @@ self: super: {
   });
 
   "nbconvert" = super."nbconvert".override (attrs: {
-    propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
+    propagatedBuildInputs = with self; attrs.propagatedBuildInputs ++ [
       # marcink: plug in jupyter-client for notebook rendering
       self."jupyter-client"
     ];
   });
 
   "ipython" = super."ipython".override (attrs: {
-    propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
+    propagatedBuildInputs = with self; attrs.propagatedBuildInputs ++ [
       self."gnureadline"
     ];
   });
 
   "lxml" = super."lxml".override (attrs: {
-    buildInputs = [
+    nativeBuildInputs = with self; attrs.nativeBuildInputs ++ [
+      pkgs.libxml2.dev
+      pkgs.libxslt.dev
+      self.cython
+    ];
+
+    buildInputs = with self; attrs.buildInputs ++ [
       pkgs.libxml2
       pkgs.libxslt
-    ];
-    propagatedBuildInputs = [
-      # Needed, so that "setup.py bdist_wheel" does work
-      self."wheel"
-    ];
-  });
-
-  "mysql-python" = super."mysql-python".override (attrs: {
-    buildInputs = [
-      pkgs.openssl
-    ];
-    propagatedBuildInputs = [
-      pkgs.libmysql
       pkgs.zlib
     ];
+
+#    propagatedBuildInputs = [
+#      # Needed, so that "setup.py bdist_wheel" does work
+#      self."wheel"
+#    ];
+  });
+
+  "mysqlclient" = super."mysqlclient".override (attrs: {
+    nativeBuildInputs = with self; attrs.nativeBuildInputs ++ [
+      pkgs.libmysqlclient
+      pkgs.openssl
+      pkgs.zlib
+    ];
+
   });
 
   "psycopg2" = super."psycopg2".override (attrs: {
-    propagatedBuildInputs = [
+    nativeBuildInputs = with self; attrs.nativeBuildInputs ++ [
       pkgs.postgresql
     ];
     meta = {
@@ -128,7 +123,12 @@ self: super: {
   });
 
   "pycurl" = super."pycurl".override (attrs: {
-    propagatedBuildInputs = [
+    nativeBuildInputs = with self; attrs.nativeBuildInputs ++ [
+      pkgs.curl
+      pkgs.openssl
+    ];
+
+    propagatedBuildInputs = with self; attrs.propagatedBuildInputs ++ [
       pkgs.curl
       pkgs.openssl
     ];
@@ -143,20 +143,8 @@ self: super: {
     };
   });
 
-  "pyramid" = super."pyramid".override (attrs: {
-    meta = {
-      license = localLicenses.repoze;
-    };
-  });
-
-  "pyramid-debugtoolbar" = super."pyramid-debugtoolbar".override (attrs: {
-    meta = {
-      license = [ pkgs.lib.licenses.bsdOriginal localLicenses.repoze ];
-    };
-  });
-
   "pysqlite" = super."pysqlite".override (attrs: {
-    propagatedBuildInputs = [
+    propagatedBuildInputs = with self; attrs.propagatedBuildInputs ++ [
       pkgs.sqlite
     ];
     meta = {
@@ -173,7 +161,7 @@ self: super: {
   });
 
   "python-pam" = super."python-pam".override (attrs: {
-    propagatedBuildInputs = [
+    propagatedBuildInputs = with self; attrs.propagatedBuildInputs ++ [
         pkgs.pam
     ];
 
@@ -187,18 +175,27 @@ self: super: {
     });
 
   "python-saml" = super."python-saml".override (attrs: {
-    buildInputs = [
+    nativeBuildInputs = with self; attrs.nativeBuildInputs ++ [
       pkgs.libxml2
       pkgs.libxslt
     ];
   });
 
   "dm.xmlsec.binding" = super."dm.xmlsec.binding".override (attrs: {
-    buildInputs = [
+    nativeBuildInputs = with self; attrs.nativeBuildInputs ++ [
+      pkgs.libxml2.dev
+      pkgs.libxslt.dev
+      pkgs.xmlsec
+      pkgs.libtool
+      pkgs.zlib
+    ];
+
+    buildInputs = with self; attrs.buildInputs ++ [
       pkgs.libxml2
       pkgs.libxslt
       pkgs.xmlsec
       pkgs.libtool
+      pkgs.zlib
     ];
   });
 
@@ -208,13 +205,46 @@ self: super: {
     ];
   });
 
+  "supervisor" = super."supervisor".override (attrs: {
+    patches = [
+      ./patches/supervisor/patch-rlimits-old-kernel.diff
+    ];
+  });
+
+  "py" = super."py".override (attrs: {
+    buildInputs = with self; attrs.buildInputs ++ [
+      self."setuptools-scm"
+    ];
+  });
+
+  "importlib-metadata" = super."importlib-metadata".override (attrs: {
+    buildInputs = with self; attrs.buildInputs ++ [
+      self."setuptools-scm"
+    ];
+  });
+
+  # License fixes
+  "appenlight-client" = super."appenlight-client".override (attrs: {
+    meta = {
+      license = [ pkgs.lib.licenses.bsdOriginal ];
+    };
+  });
+
   "urlobject" = super."urlobject".override (attrs: {
     meta = {
-      license = {
+      license = localLicenses.unlicense;
-        spdxId = "Unlicense";
-        fullName = "The Unlicense";
-        url = http://unlicense.org/;
     };
+  });
+
+  "future" = super."future".override (attrs: {
+    meta = {
+      license = [ pkgs.lib.licenses.mit ];
+    };
+  });
+
+  "testpath" = super."testpath".override (attrs: {
+    meta = {
+      license = [ pkgs.lib.licenses.mit ];
     };
   });
 
@@ -242,6 +272,18 @@ self: super: {
     };
   });
 
+  "pyramid" = super."pyramid".override (attrs: {
+    meta = {
+      license = localLicenses.repoze;
+    };
+  });
+
+  "pyramid-debugtoolbar" = super."pyramid-debugtoolbar".override (attrs: {
+    meta = {
+      license = [ pkgs.lib.licenses.bsdOriginal localLicenses.repoze ];
+    };
+  });
+
   "repoze.lru" = super."repoze.lru".override (attrs: {
     meta = {
       license = localLicenses.repoze;
@@ -266,12 +308,6 @@ self: super: {
     };
   });
 
-  "supervisor" = super."supervisor".override (attrs: {
-    patches = [
-      ./patches/supervisor/patch-rlimits-old-kernel.diff
-    ];
-  });
-
   # Avoid that base packages screw up the build process
   inherit (basePythonPackages)
     setuptools;

pkgs/shell-generate.nix

@@ -4,7 +4,11 @@
 
 with pkgs.lib;
 
-let _pythonPackages = pythonPackages; in
+let
+  _pythonPackages = pythonPackages;
+
+in
+
 let
   pythonPackages = getAttr _pythonPackages pkgs;
 
@@ -18,16 +22,21 @@ in
 
 pkgs.stdenv.mkDerivation {
   name = "pip2nix-generated";
+
   buildInputs = [
     # Allows to generate python packages
     pip2nix.pip2nix
     pythonPackages.pip-tools
+    # compile using ffi
+    pkgs.libffi
+
+    pythonPackages.cython
 
     # Allows to generate node dependencies
     pkgs.nodePackages.node2nix
 
     # We need mysql_config to be around
-    pkgs.mysql
+    pkgs.libmysqlclient
 
     # We need postgresql to be around
     pkgs.postgresql
@@ -41,6 +50,12 @@ pkgs.stdenv.mkDerivation {
     pkgs.curl
   ];
 
+  LC_ALL = "en_US.UTF-8";
+  LOCALE_ARCHIVE =
+    if pkgs.stdenv.isLinux
+    then "${pkgs.glibcLocales}/lib/locale/locale-archive"
+    else "";
+
   shellHook = ''
     runHook preShellHook
     runHook postShellHook
@@ -48,8 +63,14 @@ pkgs.stdenv.mkDerivation {
 
   preShellHook = ''
     echo "Starting Generate Shell"
+    # set unpack source date to 1980 to fix ZIP problems that does not support <1980
+    export SOURCE_DATE_EPOCH=315532800
+    export TMPDIR=/tmp
+
     # Custom prompt to distinguish from other dev envs.
-    export PS1="\n\[\033[1;32m\][Generate-shell:\w]$\[\033[0m\] "
+    export PS1="\n\[\033[1;32m\][pip2nix-generate-shell]$\[\033[0m\] "
+
     export PYCURL_SSL_LIBRARY=openssl
+
   '';
 }

requirements.txt

@@ -6,12 +6,13 @@ beaker==1.9.1
 bleach==3.1.3
 celery==4.3.0
 channelstream==0.5.2
-click==7.0
+click==7.1.2
 colander==1.7.0
 # our custom configobj
 https://code.rhodecode.com/upstream/configobj/artifacts/download/0-012de99a-b1e1-4f64-a5c0-07a98a41b324.tar.gz?md5=6a513f51fe04b2c18cf84c1395a7c626#egg=configobj==5.0.6
 cssselect==1.0.3
 cryptography==2.6.1
+cython==0.29.17
 decorator==4.1.2
 deform==2.0.8
 docutils==0.16.0
@@ -24,8 +25,8 @@ infrae.cache==1.0.1
 iso8601==0.1.12
 itsdangerous==0.24
 kombu==4.6.6
-lxml==4.2.5
+lxml==4.5.0
-mako==1.1.0
+mako==1.1.2
 markdown==2.6.11
 markupsafe==1.1.1
 msgpack-python==0.5.6
@@ -81,7 +82,7 @@ zope.event==4.4.0
 zope.interface==4.6.0
 
 # DB drivers
-mysql-python==1.2.5
+mysqlclient==1.4.6
 pymysql==0.8.1
 pysqlite==2.8.3
 psycopg2==2.8.4
@@ -107,11 +108,10 @@ waitress==1.3.1
 
 ## debug
 ipdb==0.13.2
-ipython==5.1.0
+ipython==5.10.0
 
 ## rhodecode-tools, special case, use file://PATH.tar.gz#egg=rhodecode-tools==X.Y.Z, to test local version
-https://code.rhodecode.com/rhodecode-tools-ce/artifacts/download/0-ed54e749-2ef5-4bc7-ae7f-7900e3c2aa15.tar.gz?sha256=76f024bad3a1e55fdb3d64f13f5b77ff21a12fee699918de2110fe21effd5a3a#egg=rhodecode-tools==1.4.0
+https://code.rhodecode.com/rhodecode-tools-ce/artifacts/download/0-0cf09d55-fcb5-4ab0-ad48-e71f65090875.tar.gz?sha256=ae458b6845f278aed1bcb90939ef01cdb581016ba0e7f58602d1ac6513f94707#egg=rhodecode-tools==2.0.0
-
 
 ## appenlight
 appenlight-client==0.6.26

requirements_pinned.txt

@@ -1,6 +1,6 @@
 # contains not directly required libraries we want to pin the version.
 
-atomicwrites==1.3.0
+atomicwrites==1.4.0
 attrs==19.3.0
 asn1crypto==0.24.0
 billiard==3.6.1.0
@@ -21,7 +21,13 @@ pyramid-jinja2==2.7
 scandir==1.10.0
 setproctitle==1.1.10
 tempita==0.5.2
-testpath==0.4.4
+
 transaction==2.4.0
 vine==1.3.0
 wcwidth==0.1.9
+
+# ptyprocess backport, pypi version doesn't support egg/source installs on python 2.X
+https://code.rhodecode.com/upstream/ptyprocess/artifacts/download/0-c8b019b1-c4d3-46ac-a0ad-1206ec3fb3cb.tar.gz?sha256=50394f2c5e117fcab4360bf99c8bc40be7211ee1a5860aeb3809b44249550c3e#egg=ptyprocess==0.6.0.rhodecode-upstream1
+
+# testpath backport, pypi version doesn't support egg/source installs on python 2.X
+https://code.rhodecode.com/upstream/testpath/artifacts/download/0-618e6b32-6ca5-428a-bda0-494bb347a56d.tar.gz?sha256=fd95bafd89ee2fb2bb0d82be34c9c5bba3a290f52cafc67a12a74ef825527019#egg=testpath==0.4.4.rhodecode-upstream1

requirements_test.txt

@@ -1,9 +1,9 @@
 # test related requirements
-pytest==4.6.5
+pytest==4.6.9
-py==1.8.0
+py==1.8.1
-pytest-cov==2.7.1
+pytest-cov==2.8.1
-pytest-sugar==0.9.2
+pytest-sugar==0.9.3
-pytest-runner==5.1.0
+pytest-runner==5.2.0
 pytest-profiling==1.7.0
 pytest-timeout==1.3.3
 gprof2dot==2017.9.19

setup.cfg

@@ -1,10 +1,6 @@
 [aliases]
 test = pytest
 
-[egg_info]
-tag_build =
-tag_svn_revision = false
-
 # Babel configuration
 [compile_catalog]
 domain = rhodecode

shell.nix

@@ -27,12 +27,12 @@ let
   optionalDevelopInstall = attributeName:
     let
       path = pkgs.lib.attrByPath [attributeName] null sources;
-      doIt = doDevelopInstall && path != null;
+      doDI = doDevelopInstall && path != null;
 
     in
       # do develop installation with empty hosts to skip any package duplicates to
       # be replaced. This only pushes the package to be locally available
-      pkgs.lib.optionalString doIt (''
+      pkgs.lib.optionalString doDI (''
         echo "[BEGIN] Develop install of '${attributeName}' from '${path}'"
         pushd ${path}
         python setup.py develop --prefix $tmp_path --allow-hosts ""
@@ -47,8 +47,8 @@ let
   optionalDevelopInstallBuildInputs = attributeName:
     let
       path = pkgs.lib.attrByPath [attributeName] null sources;
-      doIt = doDevelopInstall && path != null && pkgs.lib.pathExists "${nixFile}";
       nixFile = "${path}/default.nix";
+      doDI = doDevelopInstall && path != null && pkgs.lib.pathExists "${nixFile}";
 
       derivate = import "${nixFile}" {
         inherit
@@ -56,13 +56,30 @@ let
           pythonPackages;
       };
     in
-      pkgs.lib.lists.optionals doIt (
+      pkgs.lib.lists.optionals doDI (
         derivate.propagatedBuildInputs
       );
 
+  optionalBinDeps = attributeName:
+    let
+      path = pkgs.lib.attrByPath [attributeName] null sources;
+      nixFile = "${path}/default.nix";
+      doDI = doDevelopInstall && path != null && pkgs.lib.pathExists "${nixFile}";
+
+      derivate = import "${nixFile}" {
+        inherit
+          doCheck
+          pythonPackages;
+      };
+    in
+      pkgs.lib.optionalString doDI (''
+            echo "Wrapping PATH with vcsserver vcs binaries"
+            export PATH="${derivate.vcs_pkgs.subversion}/bin:${derivate.vcs_pkgs.git}/bin:${derivate.pythonPackages.mercurial}/bin:$PATH"
+      '');
+
   developInstalls = [ "rhodecode-vcsserver" ];
 
-in enterprise-ce.override (attrs: {
+in enterprise-ce.override (attrs: rec {
   # Avoid that we dump any sources into the store when entering the shell and
   # make development a little bit more convenient.
   src = null;
@@ -90,10 +107,14 @@ in enterprise-ce.override (attrs: {
   '';
 
   preShellHook = ''
-    echo "Entering CE-Shell"
+    echo "Entering rhodecode-ce"
 
     # Custom prompt to distinguish from other dev envs.
-    export PS1="\n\[\033[1;32m\][CE-shell:\w]$\[\033[0m\] "
+    export PS1="\n\[\033[1;32m\][rhodecode-ce-shell:\w]$\[\033[0m\] "
+
+    # Set locale
+    export LOCALE_ARCHIVE="${pkgs.glibcLocales}/lib/locale/locale-archive"
+    export LC_ALL="en_US.UTF-8"
 
     echo "Building frontend assets"
     ${enterprise-ce.linkNodePackages}
@@ -111,9 +132,8 @@ in enterprise-ce.override (attrs: {
 
   postShellHook = ''
     echo "** Additional develop installs **"
-    '' +
+    '' + pkgs.lib.strings.concatMapStrings optionalDevelopInstall developInstalls + ''
-    pkgs.lib.strings.concatMapStrings optionalDevelopInstall developInstalls
+    '' + pkgs.lib.strings.concatMapStrings optionalBinDeps [ "rhodecode-vcsserver" ] + ''
-    + ''
   '';
 
 })