Merge 02ac9c6cd8 into fb9f0441c9

contrib/package/nix/copyparty/default.nix

@@ -2,6 +2,9 @@
   lib,
   buildPythonApplication,
   fetchurl,
+  runCommand,
+  compressDrvWeb,
+  pigz,
   util-linux,
   python,
   setuptools,
@@ -20,6 +23,16 @@
   partftpy,
   fusepy, # for partyfuse
 
+  # deps in /copyparty/web/deps
+  marked,
+  easy-mde,
+  busy-mp3,
+  copyparty-fonts,
+  asmcrypto,
+  hash-wasm,
+  prism,
+  fusepy-copyparty,
+
   # use argon2id-hashed passwords in config files (sha2 is always available)
   withHashedPasswords ? true,
 
@@ -72,6 +85,26 @@
 let
   pinData = lib.importJSON ./pin.json;
   runtimeDeps = ([ util-linux ] ++ extraPackages ++ lib.optional withMediaProcessing ffmpeg);
+
+  sitePackages = drv: "${drv.outPath}/lib/python${lib.versions.majorMinor python.version}/site-packages";
+  nodeModules = drv: "${drv.outPath}/lib/node_modules";
+  webDeps = runCommand "copyparty-webdeps" {
+    nativeBuildInputs = [ util-linux ];
+  } ''
+    mkdir $out
+    cp --verbose -r --target-directory $out \
+      ${copyparty-fonts}/*  ${prism}/* \
+      ${sitePackages fusepy-copyparty}/fuse.py \
+      ${nodeModules easy-mde}/easymde/dist/* \
+      ${nodeModules marked}/marked/marked.min.js
+    cp ${busy-mp3} $out/busy.mp3.gz
+    cp ${hash-wasm}/sha512.umd.min.js $out/sha512.hw.js
+    cp ${nodeModules asmcrypto}/@openpgp/asmcrypto.js/asmcrypto.all.es5.js $out/sha512.ac.js
+    rename -v ".min" "" $out/*
+  '';
+  smolWebDeps = compressDrvWeb webDeps {
+    compressors.gz = "${lib.getExe pigz} -11 -I 2048 --force {}";
+  };
 in
 buildPythonApplication {
   pname = "copyparty";
@@ -103,6 +136,14 @@ buildPythonApplication {
   build-system = [
     setuptools
   ];
+  # Build webdeps when building from source
+  preBuild = lib.optionalString (true) ''
+    # TODO: remove this when we build this from source.
+    # This is just so that we can try this without building from source
+    rm -r copyparty/web/deps/* && touch copyparty/web/deps/__init__.py
+
+    cp -vr ${smolWebDeps}/* copyparty/web/deps/
+  '';
   meta = {
     description = "Turn almost any device into a file server";
     longDescription = ''

contrib/package/nix/deps/asmcrypto/default.nix

@@ -0,0 +1,28 @@
+{ buildNpmPackage, fetchFromGitHub, jq, nodejs_20}:
+buildNpmPackage rec {
+  pname = "asmcrypto";
+  version = "c72492f4a66e17a0e5dd8ad7874de354f3ccdaa5";
+
+  src = fetchFromGitHub {
+    owner = "openpgpjs";
+    repo = "asmcrypto.js";
+    rev = version;
+    hash = "sha256-nuRVRrET+HIjho+d5MHVIk4iKJg967CqMAuvAQoJNmI=";
+  };
+
+  nativeBuildInputs = [ jq ];
+
+  postPatch = ''
+    echo "export { Sha512 } from './hash/sha512/sha512';" > src/entry-export_all.ts
+  '';
+
+  npmDepsHash = "sha256-QlGyILr4B3H1bW4vpAeuT7FkkUWGQg2DqpZn8uDh2po=";
+
+  # Some assertion fails with newer node
+  nodejs = nodejs_20;
+
+  npmBuildScript = "prepare";
+
+  # This has broken symlinks for some reason, and Nix doesn't like it
+  postInstall = "rm -r $out/lib/node_modules/@openpgp/asmcrypto.js/node_modules/.bin/";
+}

contrib/package/nix/deps/busy-mp3/default.nix

@@ -0,0 +1,8 @@
+{ runCommand, sox, lame, pigz }: runCommand "busy-mp3"
+{
+  nativeBuildInputs = [ sox lame pigz ];
+}
+  ''
+    ${builtins.readFile ./../../../../../scripts/deps-docker/busy-mp3.sh}
+    mv /dev/shm/busy.mp3.gz $out
+  ''

contrib/package/nix/deps/codemirror/default.nix

@@ -0,0 +1,32 @@
+{
+  lib,
+  buildNpmPackage,
+  fetchFromGitHub,
+}:
+
+buildNpmPackage rec {
+  pname = "codemirror";
+  version = "5.65.18";
+
+  src = fetchFromGitHub {
+    owner = "codemirror";
+    repo = "codemirror5";
+    tag = version;
+    hash = "sha256-VQTpLaTYfJRUKUjLgomE6TijHIZszQqH0L+khErruAU=";
+  };
+
+  env.PUPPETEER_SKIP_CHROMIUM_DOWNLOAD = true;
+
+  npmDepsHash = "sha256-OjftAKA4YQIHJqfZ0yuZbIWtgAJOLSeroLua3FA+rTk=";
+
+  patches = [
+    ./../../../../../scripts/deps-docker/codemirror.patch
+  ];
+
+  postPatch = ''
+    # Upstream doesn't have a package-lock.json
+    cp ${./package-lock.json} ./package-lock.json
+
+    sed -ri '/^var urlRE = /d' mode/gfm/gfm.js
+  '';
+}

contrib/package/nix/deps/copyparty-fonts/default.nix

@@ -0,0 +1,32 @@
+{ runCommand, fetchzip, fetchurl, python3 }:
+let
+  fontawesomeVersion = "5.13.0";
+  fontawesomeSrc = fetchzip {
+    url = "https://github.com/FortAwesome/Font-Awesome/releases/download/${fontawesomeVersion}/fontawesome-free-${fontawesomeVersion}-web.zip";
+    hash = "sha256-gK21ztU2egNORebAyKlmxGKmoshp5ZZgMAv5jWIYBkI=";
+  };
+  scp = fetchurl {
+    url = "https://fonts.gstatic.com/s/sourcecodepro/v11/HI_SiYsKILxRpg3hIP6sJ7fM7PqlPevW.woff2";
+    hash = "sha256-ZZ/2tZan3bZIzWWlQpiTvmVWKcDTancDgXpjoIcOwCA=";
+  };
+  python = python3.withPackages (ps: with ps; [ fonttools zopfli brotli ]);
+  rawScript = ./../../../../../scripts/deps-docker/mini-fa.sh;
+
+in
+runCommand "copyparty-fonts"
+{
+  nativeBuildInputs = [ python ];
+}
+  ''
+    mkdir -p $out/no-pk
+    substitute ${rawScript} nix-mini-fa.sh \
+      --replace-fail "/z/fontawesome-fre*" "${fontawesomeSrc}" \
+      --replace-fail "/z/dist" $out \
+      --replace-fail "/z/mini-fa.css" "${./../../../../../scripts/deps-docker/mini-fa.css}" \
+      --replace-fail "/z/icon.list" "/build/icon.list" \
+      --replace-fail "/z/scp.woff2" "${scp}"
+    bash nix-mini-fa.sh
+    mv $out/no-pk/* $out
+    rmdir $out/no-pk
+  ''
+

contrib/package/nix/deps/easy-mde/default.nix

@@ -0,0 +1,31 @@
+{fetchFromGitHub, buildNpmPackage, marked, codemirror}:
+buildNpmPackage rec {
+  pname = "easy-mde";
+  version = "2.18.0";
+
+  src = fetchFromGitHub {
+    owner = "Ionaru";
+    repo = "easy-markdown-editor";
+    tag = version;
+    hash = "sha256-g3ZjvT/gqKztyrItn+xOysOWcyQ70xCkFY+dWlK7hL8=";
+  };
+
+  npmDepsHash = "sha256-Vkyw8PisYD7XznOQjzb2S9No7Ec5tGYUImRQOJLin3M=";
+
+  env.CYPRESS_INSTALL_BINARY = "0";
+
+  patches = [
+    ./../../../../../scripts/deps-docker/easymde-ln.patch
+  ];
+  postPatch = ''
+    sed -ri 's`^var marked = require\(.marked.\).marked;$`var marked = window.marked;`' src/js/easymde.js
+  '';
+
+  preBuild = ''
+    rm -r node_modules/{codemirror,marked}
+    ln -s ${codemirror}/lib/node_modules/codemirror node_modules/codemirror
+    ln -s ${marked}/lib/node_modules/marked node_modules/marked
+  '';
+
+  npmBuildScript = "prepare";
+}

contrib/package/nix/deps/hash-wasm/default.nix

@@ -0,0 +1,21 @@
+{ fetchzip, breakpointHook, stdenvNoCC }: stdenvNoCC.mkDerivation rec {
+  # The build script for this uses docker, so let's just use the release tarball
+  name = "hash-wasm";
+  version = "4.12.0";
+
+  src = fetchzip {
+    url = "https://github.com/Daninet/hash-wasm/releases/download/v${version}/hash-wasm@${version}.zip";
+    hash = "sha256-bjHpMzOXPCoKVoucVY7CXZ70vPtn8+UXMxBD7KOHg2k=";
+  };
+
+  dontBuild = true;
+
+  installPhase = ''
+      mkdir $out
+      cp $src/* -r --target-directory=$out
+    '';
+
+  nativeBuildInputs = [ breakpointHook ];
+
+  npmDepsHash = "sha256-S9g8YlrsD7t0y9qi/CsN7qHZcT/TDfC1XxiArSGjAe4=";
+}

contrib/package/nix/deps/marked/default.nix

@@ -0,0 +1,32 @@
+{ buildNpmPackage, fetchFromGitHub, breakpointHook }:
+buildNpmPackage rec {
+  pname = "marked";
+  version = "4.3.0";
+
+  src = fetchFromGitHub {
+    owner = "markedjs";
+    repo = "marked";
+    tag = "v${version}";
+    hash = "sha256-xJotgIUoZSmjgLRIgJP9134PR1GYKjVXM5jp+JpM7bg=";
+  };
+
+  dompurify = fetchFromGitHub {
+    owner = "cure53";
+    repo = "DOMPurify";
+    tag = "3.2.7";
+    hash = "sha256-lwBi/3uqWBa0UTPV1iKIIXDC6+Fc9uwIgs4+pRUjdPA=";
+  };
+
+  npmDepsHash = "sha256-omsoER0I28ZHN78X0aku2jNJApPvpqYKgfPhRbEi+HM=";
+
+  patches = [
+    ./../../../../../scripts/deps-docker/marked.patch
+    ./../../../../../scripts/deps-docker/marked-ln.patch
+  ];
+
+  nativeBuildInputs = [ breakpointHook ];
+  postBuild = ''
+    # Inject DOMPurify to minified marked
+    cat ${dompurify}/dist/purify.min.js >> marked.min.js
+  '';
+}

contrib/package/nix/deps/prism/default.nix

@@ -0,0 +1,35 @@
+{ stdenvNoCC, fetchFromGitHub, python3 }: stdenvNoCC.mkDerivation rec {
+  name = "prism";
+  version = "1.29.0";
+
+  src = fetchFromGitHub {
+    owner = "PrismJS";
+    repo = "prism";
+    tag = "v${version}";
+    hash = "sha256-KEoICg4xviKsmN9M8ceJdAJ1NhTO7urDJnJknuP4GoQ=";
+  };
+
+  nativeBuildInputs = [ python3 ];
+
+  depsScriptDir = ../../../../../scripts/deps-docker;
+
+  postUnpack = ''
+    cp ${depsScriptDir}/genprism.* .
+  '';
+
+  postPatch = ''
+    substituteInPlace ../genprism.sh \
+      --replace-fail "/z/dist" "$out" \
+      --replace-fail 'prism-$1' "$src" \
+      --replace-fail "./genprism.py" "python3 ./genprism.py"
+  '';
+
+  buildPhase = ''
+    mkdir $out
+    cd ..
+    bash ./genprism.sh ${version}
+  '';
+
+  # genprism.sh already moves things into place
+  dontInstall = true;
+}

contrib/package/nix/overlay.nix

@@ -3,9 +3,30 @@ final: prev: {
     ffmpeg = final.ffmpeg-full;
   };
 
+  # Dependencies
+  codemirror = final.callPackage ./deps/codemirror { };
+  marked = final.callPackage ./deps/marked { };
+  easy-mde = final.callPackage ./deps/easy-mde { };
+  busy-mp3 = final.callPackage ./deps/busy-mp3 { };
+  copyparty-fonts = final.callPackage ./deps/copyparty-fonts { };
+  asmcrypto = final.callPackage ./deps/asmcrypto { };
+  hash-wasm = final.callPackage ./deps/hash-wasm { };
+  prism = final.callPackage ./deps/prism { };
+
   python3 = prev.python3.override {
     packageOverrides = pyFinal: pyPrev: {
       partftpy = pyFinal.callPackage ./partftpy { };
+      fusepy-copyparty = pyPrev.fusepy.overridePythonAttrs (old: {
+        pname = "${old.pname}-copyparty";
+        patchPhase = old.patchPhase + ''
+          python3 ${./../../../scripts/uncomment.py} fuse.py
+          sed -ri '/self.__critical_exception = e/d' fuse.py
+          awk '/^log =/{s=0} !s; /^from traceback im/{s=1;print"from functools import partial";print"basestring = str"}' < fuse.py > temp
+          awk '/LoggingMixIn:/{exit} --s<0;/self.use_ns = getattr/{s=7}' < temp > fuse.py
+          awk "/if _machine =/{s=0} /'(mips|ppc|ppc64)'/{s=1} !s" < fuse.py > temp
+          mv temp fuse.py
+        '';
+      });
     };
   };
 }

flake.nix

@@ -39,6 +39,19 @@
           withSMB = true;
         };
 
+        # Make sure that the nix dependencies don't drift away
+        checks.nix-deps-updated = pkgs.runCommandLocal "nix-deps-updated-check" { } ''
+          hash="6e3a014f303f86992e75446c3ce3aaf704fc838b850ba3688c9a1f5d358bc9f4"
+          dockerfilePath="${self.outPath}/scripts/deps-docker/Dockerfile"
+
+          echo "If you can see this, the dependencies dockerfile updated."
+          echo "Please update the nix packages (if necessary) and the hash on this check"
+          echo $hash $dockerfilePath | sha256sum --check --status
+
+          # Need to make an empty folder so that Nix doesn't complain
+          mkdir $out
+        '';
+
         packages = {
           inherit (pkgs)
             copyparty