diff --git a/CHANGELOG.md b/CHANGELOG.md index 3488d42..6a64904 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#91](https://github.com/utensils/comfyui-nix/issues/91)) ### Changed +- Linux now runs on Python 3.13 (from 3.12), + the version upstream ComfyUI recommends and ships in its portable builds. + macOS stays on 3.12 because the pinned PyTorch 2.5.1 has no Python 3.13 wheel for macOS. + Packages that ComfyUI-Manager installed into `/.venv/lib/python3.12` are not visible to Python 3.13, + so the launcher prints a note asking to reinstall them. +- PyTorch wheel pins in `nix/versions.nix` are now a URL template plus one hash per interpreter tag, + so moving to another Python only requires adding hashes. + The new `wheel-pins` check fails evaluation when a pin lacks the hash for the selected Python. +- Updated the bundled PyAV wheel `17.0.0` -> `17.0.1`, + because `17.0.0` hangs imageio's fork regression test on Python 3.13. +- Reduced the Python package overrides that Linux no longer needs on nixpkgs' default Python. + The overrides for open-clip-torch, torchmetrics, inline-snapshot, backrefs, jupyter-server, ultralytics, mss and filterpy + now apply to macOS only, + so on Linux those packages come from the binary cache or build with their test suites enabled. + The timm and torchsde overrides now apply only where PyTorch comes from a pre-built wheel. + The albumentations and bitsandbytes overrides and the `allowBrokenPredicate` for open-clip-torch are removed, + since nixpkgs dropped albumentations, already builds bitsandbytes with ninja, + and no longer marks open-clip-torch as broken. - Updated ComfyUI to upstream `v0.37.0` (from `v0.34.0`). Updated its pinned dependencies: frontend `1.49.6` -> `1.52.7`, workflow templates `0.11.48` -> `0.11.66` (core `0.3.322` -> `0.3.357`, JSON `0.1.57` -> `0.1.92`, diff --git a/CLAUDE.md b/CLAUDE.md index 6bcd06c..0a250d7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -43,6 +43,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co - `nixfmt`: Verify Nix formatting (nixfmt-rfc-style) - `shellcheck`: Lint shell scripts in `scripts/` - `package`: Full build of the default package + - `wheel-pins`: Every interpreter-specific wheel pin in `nix/versions.nix` has a hash for the selected Python (evaluation only) - **Manual linting in dev shell** (`nix develop`): - `ruff check src/` — lint Python (or `ruff check src/path/to/file.py` for a single file) - `ruff format src/` — format Python (or `ruff format src/path/to/file.py`) @@ -57,7 +58,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co - Vendored wheels (spandrel, frontend, docs, etc.) also pinned in `nix/versions.nix` - Template input files: auto-generated in `nix/template-inputs.nix` - Update with: `./scripts/update-template-inputs.sh && git add nix/template-inputs.nix` -- Python version: 3.12 +- Python version: 3.13 on Linux, 3.12 on macOS (torch 2.5.1 has no cp313 macOS wheel) — selected in `nix/python.nix` - PyTorch: macOS uses pre-built wheels (2.5.1, pinned to work around MPS bugs on macOS 26); CUDA uses pre-built wheels from pytorch.org (cu130); ROCm uses pre-built wheels from pytorch.org (rocm7.1); Intel XPU uses pre-built wheels from pytorch.org (xpu, oneAPI/SYCL — no IPEX); Linux CPU uses nixpkgs ## Project Architecture @@ -91,7 +92,7 @@ Each module branches on this value to select platform-specific PyTorch wheels, r **XPU-specific notes** (Intel oneAPI / SYCL): -- Wheel channel: `https://download.pytorch.org/whl/xpu` — `torch 2.10.0+xpu`, `torchvision 0.25.0+xpu`, `torchaudio 2.10.0+xpu` (cp312, Linux x86_64 only) +- Wheel channel: `https://download.pytorch.org/whl/xpu` — `torch 2.10.0+xpu`, `torchvision 0.25.0+xpu`, `torchaudio 2.10.0+xpu` (cp313, Linux x86_64 only) - No IPEX required — ComfyUI uses `torch.xpu.is_available()` auto-detection - Host must provide Level Zero loader + Intel compute-runtime at runtime. The launcher prefers `/run/opengl-driver/lib` (NixOS `hardware.graphics.extraPackages`) and falls back to Nix-bundled `pkgs.level-zero` + `pkgs.intel-compute-runtime` + `pkgs.ocl-icd` for non-NixOS Linux - Runtime env vars set by launcher: `SYCL_CACHE_PERSISTENT=1`, `SYCL_CACHE_DIR=/.cache/libsycl_cache`. Opt-in: `COMFY_ENABLE_XPU_FP64_EMULATION=1` enables `OverrideDefaultFP64Settings` + `IGC_EnableDPEmulation` for iGPUs without native FP64 diff --git a/README.md b/README.md index 1cc3c02..b125ccd 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![NixOS](https://img.shields.io/badge/NixOS-unstable-blue.svg?logo=nixos)](https://nixos.org) -A slightly opinionated, pure Nix flake for [ComfyUI] with Python 3.12 and curated custom nodes. Supports macOS (Apple Silicon) and Linux. +A slightly opinionated, pure Nix flake for [ComfyUI] with Python 3.13 (3.12 on macOS) and curated custom nodes. Supports macOS (Apple Silicon) and Linux.

ComfyUI Demo @@ -645,7 +645,7 @@ nix run .#buildDockerXpu # Intel XPU ## Development ```bash -nix develop # Dev shell with Python 3.12, ruff, pyright +nix develop # Dev shell with Python 3.13, ruff, pyright nix flake check # Run all checks (build, lint, type-check, nixfmt) nix run .#update # Check for ComfyUI updates ``` diff --git a/flake.nix b/flake.nix index 9d846e3..7323253 100644 --- a/flake.nix +++ b/flake.nix @@ -1,5 +1,5 @@ { - description = "A Nix flake for ComfyUI with Python 3.12"; + description = "A Nix flake for ComfyUI with Python 3.13"; nixConfig = { extra-substituters = [ @@ -82,14 +82,12 @@ system = "x86_64-linux"; config = { allowUnfree = true; - allowBrokenPredicate = pkg: (pkg.pname or "") == "open-clip-torch"; }; }; pkgsLinuxArm64 = import nixpkgs { system = "aarch64-linux"; config = { allowUnfree = true; - allowBrokenPredicate = pkg: (pkg.pname or "") == "open-clip-torch"; }; }; @@ -106,7 +104,7 @@ mkPython = pkgs: gpuSupport: - pkgs.python312.override { + (import ./nix/python.nix { inherit pkgs; }).override { packageOverrides = pythonOverridesFor pkgs gpuSupport "rocm71"; }; @@ -194,7 +192,6 @@ inherit system; config = { allowUnfree = true; - allowBrokenPredicate = pkg: (pkg.pname or "") == "open-clip-torch"; # Keep the x86_64-only ROCm development shell evaluable on aarch64-linux. allowUnsupportedSystem = system == "aarch64-linux"; }; diff --git a/nix/checks.nix b/nix/checks.nix index 9e93efd..6f6a25d 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -197,7 +197,8 @@ let touch $out ''; mssRuntimeDeps = - assert !pythonPackages.mss.doInstallCheck; + # The override that disables mss' tests only applies on Darwin. + assert pkgs.stdenv.hostPlatform.isDarwin -> !pythonPackages.mss.doInstallCheck; pythonPackages.mss; # Every backend's torch comes from a pre-built wheel whose metadata names # distributions nixpkgs supplies outside PyPI. Those requirements are dropped @@ -276,6 +277,28 @@ let runtimeDepsChecks = pkgs.lib.mapAttrs' ( name: package: pkgs.lib.nameValuePair "${name}-runtime-deps" package ) runtimeDepsPackages; + # Interpreter-specific wheel pins carry one hash per interpreter tag. + # Most backends are never built in a single check run, + # so find the pins that lack a hash for the selected Python by evaluation alone. + basePython = pythonFor "none"; + pythonTag = "cp" + pkgs.lib.replaceStrings [ "." ] [ "" ] basePython.pythonVersion; + versions = import ./versions.nix; + wheelChannels = + if pkgs.stdenv.hostPlatform.isDarwin then + { inherit (versions.pytorchWheels) darwinArm64; } + else + removeAttrs versions.pytorchWheels [ "darwinArm64" ] + // { + xpuRuntime = { inherit (versions.xpuRuntime) triton-xpu; }; + }; + wheelPinsMissingHash = pkgs.lib.concatLists ( + pkgs.lib.mapAttrsToList ( + channel: wheels: + pkgs.lib.mapAttrsToList (name: _: "${channel}.${name}") ( + pkgs.lib.filterAttrs (_: spec: !(spec.hashes ? ${pythonTag})) wheels + ) + ) wheelChannels + ); in { package = packages.default; @@ -351,6 +374,12 @@ in } // runtimeDepsChecks // torchRuntimeDepsChecks +// { + wheel-pins = + assert pkgs.lib.assertMsg (wheelPinsMissingHash == [ ]) + "nix/versions.nix: no ${pythonTag} hash pinned for ${pkgs.lib.concatStringsSep ", " wheelPinsMissingHash}"; + pkgs.runCommand "wheel-pins" { } "touch $out"; +} // pkgs.lib.optionalAttrs ( diff --git a/nix/packages.nix b/nix/packages.nix index cc46afa..baf4650 100644 --- a/nix/packages.nix +++ b/nix/packages.nix @@ -81,7 +81,7 @@ let ]; }; - python = pkgs.python312.override { packageOverrides = pythonOverrides; }; + python = (import ./python.nix { inherit pkgs; }).override { packageOverrides = pythonOverrides; }; vendored = import ./vendored-packages.nix { inherit pkgs python versions; }; @@ -656,25 +656,41 @@ let # This allows both pip and uv to install packages to a writable location # while keeping the Nix store read-only VENV_DIR="$BASE_DIR/.venv" - SITE_PACKAGES="$VENV_DIR/lib/python3.12/site-packages" + SITE_PACKAGES="$VENV_DIR/lib/python${python.pythonVersion}/site-packages" mkdir -p "$SITE_PACKAGES" mkdir -p "$VENV_DIR/bin" - # Create pyvenv.cfg if it doesn't exist (required for PEP 405 compliance) - if [[ ! -e "$VENV_DIR/pyvenv.cfg" ]]; then - cat > "$VENV_DIR/pyvenv.cfg" << PYVENV + # Packages are installed per Python minor version, + # so anything the Manager installed under a previous interpreter is no longer on sys.path. + # Say so instead of letting nodes fail with a bare ImportError. + for old_site in "$VENV_DIR"/lib/python3.*/site-packages; do + if [[ -d "$old_site" && "$old_site" != "$SITE_PACKAGES" ]] \ + && [[ -n "$(find "$old_site" -mindepth 1 -print -quit 2>/dev/null)" ]]; then + echo "Note: $old_site was populated by an older Python and is ignored by Python ${python.pythonVersion}." + echo " Reinstall the affected custom node dependencies via ComfyUI-Manager, then remove that directory." + fi + done + + # Write pyvenv.cfg (required for PEP 405 compliance). + # Rewritten on every start because it names the interpreter's store path and version, + # both of which change whenever the flake is updated. + cat > "$VENV_DIR/pyvenv.cfg" << PYVENV home = ${pythonRuntime}/bin include-system-site-packages = true - version = 3.12.9 + version = ${python.version} PYVENV - fi - # Symlink Python executable into venv bin (some tools expect this) - if [[ ! -e "$VENV_DIR/bin/python" ]]; then - ln -sf "${pythonRuntime}/bin/python" "$VENV_DIR/bin/python" - ln -sf "${pythonRuntime}/bin/python3" "$VENV_DIR/bin/python3" - ln -sf "${pythonRuntime}/bin/python3.12" "$VENV_DIR/bin/python3.12" - fi + # Symlink Python executable into venv bin (some tools expect this). + # Refreshed on every start for the same reason as pyvenv.cfg, + # and links left behind by another Python minor version are dropped. + for old_link in "$VENV_DIR"/bin/python3.*; do + if [[ -L "$old_link" && "$old_link" != "$VENV_DIR/bin/python${python.pythonVersion}" ]]; then + rm -f "$old_link" + fi + done + ln -sfn "${pythonRuntime}/bin/python" "$VENV_DIR/bin/python" + ln -sfn "${pythonRuntime}/bin/python3" "$VENV_DIR/bin/python3" + ln -sfn "${pythonRuntime}/bin/python${python.pythonVersion}" "$VENV_DIR/bin/python${python.pythonVersion}" # Set VIRTUAL_ENV so uv installs to our mutable venv instead of Nix store export VIRTUAL_ENV="$VENV_DIR" diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix index 9455aab..06191cf 100644 --- a/nix/python-overrides.nix +++ b/nix/python-overrides.nix @@ -11,27 +11,52 @@ let # Intel XPU wheels are Linux x86_64 only (no aarch64 upstream) useXpu = gpuSupport == "xpu" && pkgs.stdenv.hostPlatform.isLinux && pkgs.stdenv.hostPlatform.isx86_64; - useDarwinArm64 = pkgs.stdenv.hostPlatform.isDarwin && pkgs.stdenv.hostPlatform.isAarch64; + isDarwin = pkgs.stdenv.hostPlatform.isDarwin; + useDarwinArm64 = isDarwin && pkgs.stdenv.hostPlatform.isAarch64; + # Whether torch is one of the pre-built wheels below instead of nixpkgs' own build. + # Packages depending on torch are then rebuilt against it, running their test suites here. + wheelTorch = useCuda || useRocm || useXpu || useDarwinArm64; sentencepieceNoGperf = pkgs.sentencepiece.override { withGPerfTools = false; }; cudaPackages = pkgs.cudaPackages_13; + # Interpreter tag of the Python selected in python.nix, e.g. "cp313". + pythonTag = + "cp" + lib.replaceStrings [ "." ] [ "" ] (import ./python.nix { inherit pkgs; }).pythonVersion; + + # Interpreter-specific wheels are pinned in versions.nix as a URL template plus one hash per tag, + # where "{py}" in the URL stands for the interpreter tag. + # Resolve a pin to the plain { url, hash } pair for the selected Python. + # Evaluation fails with a pointer to versions.nix when the hash for that tag is missing, + # which is the only thing a Python bump has to add. + resolveWheel = + name: spec: + spec + // { + url = lib.replaceStrings [ "{py}" ] [ pythonTag ] spec.url; + hash = + spec.hashes.${pythonTag} + or (throw "nix/versions.nix: no ${pythonTag} hash pinned for ${name} ${spec.version}"); + }; + resolveWheels = lib.mapAttrs resolveWheel; + # Pre-built PyTorch CUDA wheels from pytorch.org # These avoid compiling PyTorch from source (which requires 30-60GB RAM and hours of build time) # CUDA runtime libraries are supplied by nixpkgs, so no separate toolkit is needed at runtime - cudaWheels = versions.pytorchWheels.cu130; + cudaWheels = resolveWheels versions.pytorchWheels.cu130; # Pre-built PyTorch ROCm wheels from pytorch.org. # These avoid compiling PyTorch from source (which requires 30-60GB RAM and hours of build time). # `rocmChannel` selects the wheel set: "rocm71" or "rocm72" (the gfx1151 / # Strix Halo path). See versions.nix. - rocmWheels = versions.pytorchWheels.${rocmChannel}; + rocmWheels = resolveWheels versions.pytorchWheels.${rocmChannel}; # Pre-built PyTorch XPU wheels from pytorch.org (Intel oneAPI / SYCL) # Unlike CUDA/ROCm, the XPU torch wheel does NOT bundle its SYCL / MKL / # Intel compiler runtimes — they ship as ~20 separate PyPI wheels declared # in Requires-Dist. All pinned in versions.xpuRuntime and packaged below. # Host still provides Level Zero loader + Intel compute-runtime at runtime. - xpuWheels = versions.pytorchWheels.xpu; + xpuWheels = resolveWheels versions.pytorchWheels.xpu; + tritonXpuWheel = resolveWheel "triton-xpu" versions.xpuRuntime.triton-xpu; # Ignore-list for deps neither the wheels nor nixpkgs provide. # Intel wheel-to-wheel SONAME refs are resolved via cross-wheel buildInputs @@ -64,7 +89,7 @@ let # Pre-built PyTorch wheels for macOS Apple Silicon # PyTorch 2.5.1 is used instead of 2.9.x due to MPS bugs on macOS 26 (Tahoe) # See: https://github.com/pytorch/pytorch/issues/167679 - darwinWheels = versions.pytorchWheels.darwinArm64; + darwinWheels = resolveWheels versions.pytorchWheels.darwinArm64; # Common build inputs for PyTorch wheels (manylinux compatibility) wheelBuildInputs = [ @@ -606,11 +631,10 @@ lib.optionalAttrs useCuda { # to nixpkgs' non-XPU triton or fails at dynamo compile time. tritonXpu = final.buildPythonPackage { pname = "triton"; - version = versions.xpuRuntime.triton-xpu.version; + version = tritonXpuWheel.version; format = "wheel"; src = pkgs.fetchurl { - url = versions.xpuRuntime.triton-xpu.url; - hash = versions.xpuRuntime.triton-xpu.hash; + inherit (tritonXpuWheel) url hash; }; dontBuild = true; dontConfigure = true; @@ -889,12 +913,98 @@ lib.optionalAttrs useCuda { # Note: When useCuda=true, torch/torchvision/torchaudio are replaced with pre-built wheels # above. Packages that depend on torch (kornia, accelerate, etc.) will automatically # use our wheel-based torch via final.torch since we've overridden it in the overlay. -// lib.optionalAttrs (pkgs.stdenv.hostPlatform.isDarwin && prev ? sentencepiece) { - sentencepiece = prev.sentencepiece.overridePythonAttrs (old: { - buildInputs = [ sentencepieceNoGperf.dev ]; - nativeBuildInputs = old.nativeBuildInputs or [ ]; - }); -} +# Overrides that only Darwin needs, limited to the packages the pinned nixpkgs has. +# +# Most of them work around test suites. +# Darwin runs Python 3.12, whose package set is not in the binary cache, so everything is built from source there. +# Linux runs nixpkgs' default Python, +# where the same packages come from the binary cache or build and pass their tests unmodified. +// lib.optionalAttrs isDarwin ( + lib.filterAttrs (name: _: prev ? ${name}) { + sentencepiece = prev.sentencepiece.overridePythonAttrs (old: { + buildInputs = [ sentencepieceNoGperf.dev ]; + nativeBuildInputs = old.nativeBuildInputs or [ ]; + }); + + # Disable tests for open-clip-torch (they hang waiting for model downloads) + open-clip-torch = prev.open-clip-torch.overridePythonAttrs (_old: { + doCheck = false; + }); + + # Fix torchmetrics build: nixpkgs torchmetrics doesn't declare setuptools as a build backend + # dependency, so PEP517 build fails with "Cannot import 'setuptools.build_meta'". + torchmetrics = prev.torchmetrics.overridePythonAttrs (old: { + nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ + final.setuptools + final.wheel + ]; + }); + + # inline-snapshot 0.34.2 fails three of its own documentation tests on the + # pinned nixpkgs (they assert formatter output that has since changed) and is + # not in the binary cache, so it gets built from source. It reaches us only as + # a test helper in fastapi's checkInputs, where its own doc tests say nothing + # about whether it works. Without this, fastapi and everything downstream of + # it (openai, openapi-core, sqlframe) cannot build. + inline-snapshot = prev.inline-snapshot.overridePythonAttrs (_old: { + doCheck = false; + }); + + # backrefs' test_timeout asserts that a regex operation exceeds a wall-clock + # timeout, which does not hold on every builder: it passes locally but fails + # on CI runners with "DID NOT RAISE TimeoutError". backrefs is not in the + # binary cache on the pinned nixpkgs, so it is built from source and the flake + # test runs. It reaches us transitively through einops' check inputs, and the + # failure cascades all the way up to pyright-check. + backrefs = prev.backrefs.overridePythonAttrs (old: { + disabledTests = (old.disabledTests or [ ]) ++ [ "test_timeout" ]; + }); + + # These asynchronous kernel-disconnect tests are sensitive to load in the Nix + # sandbox: orphan cleanup can miss its deadline, which also makes the companion + # file-descriptor assertion report the sockets that are still being cleaned up. + # jupyter-server reaches us through einops' check inputs, like backrefs above. + jupyter-server = prev.jupyter-server.overridePythonAttrs (old: { + disabledTests = (old.disabledTests or [ ]) ++ [ + "test_no_fd_leak_on_disconnect_with_orphaned_kernel_info_channel" + "test_disconnect_resolves_orphaned_kernel_info_future" + ]; + }); + + # This dataset utility test downloads fixture archives from GitHub. + ultralytics = prev.ultralytics.overridePythonAttrs (old: { + disabledTests = (old.disabledTests or [ ]) ++ [ "test_data_utils" ]; + }); + + # Isolate Valkey's Redis test server from existing services on Darwin hosts. + valkey = prev.valkey.overridePythonAttrs (old: { + # Darwin builds share the host's ports. The Redis test hook otherwise + # retries forever when an existing service occupies its default port 6379. + preCheck = (old.preCheck or "") + '' + redisTestPort="$(${final.python.interpreter} - <<'PY' + import socket + with socket.socket() as server: + server.bind(("127.0.0.1", 0)) + print(server.getsockname()[1]) + PY + )" + pytestFlagsArray+=("--valkey-url=valkey://127.0.0.1:$redisTestPort/0") + ''; + }); + + # mss runs screenshot tests through a virtual X server. They are not relevant + # to ComfyUI's runtime dependency closure and can fail in a headless Nix build + # sandbox. Remove this override once nixpkgs' mss checks are sandbox-independent. + mss = prev.mss.overridePythonAttrs (_old: { + doCheck = false; + }); + + # Disable filterpy tests on Darwin (test_hinfinity triggers BPT trap in pytest) + filterpy = prev.filterpy.overridePythonAttrs (_old: { + doCheck = false; + }); + } +) # Note: On Darwin, av uses ffmpeg 7.x and torchaudio uses ffmpeg 6.x. # These versions are mutually incompatible for building. The resulting runtime # warning about duplicate Objective-C classes is harmless in practice. @@ -902,30 +1012,40 @@ lib.optionalAttrs useCuda { # Override av (PyAV) to use pre-built wheel for comfy_api_nodes compatibility # Using wheels avoids FFmpeg version issues (wheels bundle their own FFmpeg) # This fixes build failures when nixpkgs has FFmpeg 8.x (AVFMT_ALLOW_FLUSH removed) +# +# Keep this at 17.0.1 or newer. +# With the 17.0.0 wheel on Python 3.13, +# imageio's test_lagging_video_stream fails the build every time: +# a forked child that runs gc.collect() after a video was read never returns. +# 17.0.1 passes it. +# +# The bundled FFmpeg also keeps av independent of the caller's LD_LIBRARY_PATH. +# nixpkgs' av links the system FFmpeg, which loads libasound, +# and a libasound from a newer host glibc on LD_LIBRARY_PATH breaks `import av` at startup. // lib.optionalAttrs (prev ? av) { av = let - # Use platform-specific abi3 wheels from PyPI (av 17.0.0, Python 3.12) + # Use platform-specific abi3 wheels from PyPI (av 17.0.1; cp311-abi3, so any Python >= 3.11) wheelSrc = if pkgs.stdenv.hostPlatform.isLinux && pkgs.stdenv.hostPlatform.isx86_64 then pkgs.fetchurl { - url = "https://files.pythonhosted.org/packages/d2/59/d19bc3257dd985d55337d7f0414c019414b97e16cd3690ebf9941a847543/av-17.0.0-cp311-abi3-manylinux_2_28_x86_64.whl"; - hash = "sha256-EGDLqF+X9KM3MRFp2SwLXhQ0Us+lyg5l+kmdeVXoWS4="; + url = "https://files.pythonhosted.org/packages/02/28/167b291356c2cc315a2d62a95b0ceace72b5b0bf547de30b89313110f032/av-17.0.1-cp311-abi3-manylinux_2_28_x86_64.whl"; + hash = "sha256-xYxxv/2Tg5CMhWlaxh0xhMZorMsEpb0bJi4PuNCfYKU="; } else if pkgs.stdenv.hostPlatform.isLinux && pkgs.stdenv.hostPlatform.isAarch64 then pkgs.fetchurl { - url = "https://files.pythonhosted.org/packages/00/c0/637721f3cd5bb8bd16105a1a08efd781fc12f449931bdb3a4d0cfd63fa55/av-17.0.0-cp311-abi3-manylinux_2_28_aarch64.whl"; - hash = "sha256-tEDaasR9oGKdUJMW8kvNhY8zFY290PG3KT1x6ZvrJt4="; + url = "https://files.pythonhosted.org/packages/a8/8b/8de3fd21c4b0b74d44337421abeab0e71462337fb6a28fff888e0c356cbd/av-17.0.1-cp311-abi3-manylinux_2_28_aarch64.whl"; + hash = "sha256-5u7oSvpI0OkyEEfNPk+s1EtAFJP2vcdT4uHR58nm0T4="; } else if pkgs.stdenv.hostPlatform.isDarwin && pkgs.stdenv.hostPlatform.isx86_64 then pkgs.fetchurl { - url = "https://files.pythonhosted.org/packages/b1/fb/55e3b5b5d1fc61466292f26fbcbabafa2642f378dc48875f8f554591e1a4/av-17.0.0-cp311-abi3-macosx_11_0_x86_64.whl"; - hash = "sha256-7UAT+sd8MJpKaBQdz2FI8YIbsQc6NtQok3l2KmNy9xE="; + url = "https://files.pythonhosted.org/packages/4c/82/e7007dcef7bd2d2c377e2e85977701384f42d19fc808c2ccb3a99eaf58f2/av-17.0.1-cp311-abi3-macosx_11_0_x86_64.whl"; + hash = "sha256-mH9PRs6uTabGFNy9K4FJvp2/aAw7t6aEHFivnP9NkjA="; } else if pkgs.stdenv.hostPlatform.isDarwin && pkgs.stdenv.hostPlatform.isAarch64 then pkgs.fetchurl { - url = "https://files.pythonhosted.org/packages/52/03/9ace1acc08bc9ae38c14bf3a4b1360e995e4d999d1d33c2cbd7c9e77582a/av-17.0.0-cp311-abi3-macosx_14_0_arm64.whl"; - hash = "sha256-5Etsg+nzvp957ofQt3onzqmpzWe9YwNiyGt+VqdI37s="; + url = "https://files.pythonhosted.org/packages/6b/aa/858b09a08ea6f83f91be44b5a5adad13ae8d9ac8b80fda27e73c24bfb160/av-17.0.1-cp311-abi3-macosx_14_0_arm64.whl"; + hash = "sha256-2X9U5VsYp0kS9HnBl4qt0TQdONiS3ulbtcLy3M+nLzI="; } else # Fallback to source build for unsupported platforms @@ -934,7 +1054,7 @@ lib.optionalAttrs useCuda { if wheelSrc != null then final.buildPythonPackage { pname = "av"; - version = "17.0.0"; + version = "17.0.1"; format = "wheel"; src = wheelSrc; # Wheel contains bundled FFmpeg libraries @@ -955,68 +1075,6 @@ lib.optionalAttrs useCuda { prev.av; } -# Disable tests for open-clip-torch (they hang waiting for model downloads) -// lib.optionalAttrs (prev ? open-clip-torch) { - open-clip-torch = prev.open-clip-torch.overridePythonAttrs (old: { - doCheck = false; - }); -} - -# Disable tests for albumentations (very slow test suite, well-tested upstream) -// lib.optionalAttrs (prev ? albumentations) { - albumentations = prev.albumentations.overridePythonAttrs (old: { - doCheck = false; - }); -} - -# Fix torchmetrics build: nixpkgs torchmetrics doesn't declare setuptools as a build backend -# dependency, so PEP517 build fails with "Cannot import 'setuptools.build_meta'". -// lib.optionalAttrs (prev ? torchmetrics) { - torchmetrics = prev.torchmetrics.overridePythonAttrs (old: { - nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ - final.setuptools - final.wheel - ]; - }); -} - -# inline-snapshot 0.34.2 fails three of its own documentation tests on the -# pinned nixpkgs (they assert formatter output that has since changed) and is -# not in the binary cache, so it gets built from source. It reaches us only as -# a test helper in fastapi's checkInputs, where its own doc tests say nothing -# about whether it works. Without this, fastapi and everything downstream of -# it (openai, openapi-core, sqlframe) cannot build. -// lib.optionalAttrs (prev ? inline-snapshot) { - inline-snapshot = prev.inline-snapshot.overridePythonAttrs (_old: { - doCheck = false; - }); -} - -# backrefs' test_timeout asserts that a regex operation exceeds a wall-clock -# timeout, which does not hold on every builder: it passes locally but fails -# on CI runners with "DID NOT RAISE TimeoutError". backrefs is not in the -# binary cache on the pinned nixpkgs, so it is built from source and the flake -# test runs. It reaches us transitively through einops' check inputs, and the -# failure cascades all the way up to pyright-check. -// lib.optionalAttrs (prev ? backrefs) { - backrefs = prev.backrefs.overridePythonAttrs (old: { - disabledTests = (old.disabledTests or [ ]) ++ [ "test_timeout" ]; - }); -} - -# These asynchronous kernel-disconnect tests are sensitive to load in the Nix -# sandbox: orphan cleanup can miss its deadline, which also makes the companion -# file-descriptor assertion report the sockets that are still being cleaned up. -# jupyter-server reaches us through einops' check inputs, like backrefs above. -// lib.optionalAttrs (prev ? jupyter-server) { - jupyter-server = prev.jupyter-server.overridePythonAttrs (old: { - disabledTests = (old.disabledTests or [ ]) ++ [ - "test_no_fd_leak_on_disconnect_with_orphaned_kernel_info_channel" - "test_disconnect_resolves_orphaned_kernel_info_future" - ]; - }); -} - # Keep accelerate's tests compatible with our platform-specific torch wheels. // lib.optionalAttrs (prev ? accelerate) { accelerate = prev.accelerate.overridePythonAttrs (old: { @@ -1030,34 +1088,9 @@ lib.optionalAttrs useCuda { }); } -# This dataset utility test downloads fixture archives from GitHub. Disable it -# on every platform so uncached builds also work without sandbox networking. -// lib.optionalAttrs (prev ? ultralytics) { - ultralytics = prev.ultralytics.overridePythonAttrs (old: { - disabledTests = (old.disabledTests or [ ]) ++ [ "test_data_utils" ]; - }); -} - -# Isolate Valkey's Redis test server from existing services on Darwin hosts. -// lib.optionalAttrs (pkgs.stdenv.hostPlatform.isDarwin && prev ? valkey) { - valkey = prev.valkey.overridePythonAttrs (old: { - # Darwin builds share the host's ports. The Redis test hook otherwise - # retries forever when an existing service occupies its default port 6379. - preCheck = (old.preCheck or "") + '' - redisTestPort="$(${final.python.interpreter} - <<'PY' - import socket - with socket.socket() as server: - server.bind(("127.0.0.1", 0)) - print(server.getsockname()[1]) - PY - )" - pytestFlagsArray+=("--valkey-url=valkey://127.0.0.1:$redisTestPort/0") - ''; - }); -} - -# Disable failing timm test (torch dynamo/inductor test needs setuptools at runtime) -// lib.optionalAttrs (prev ? timm) { +# Disable failing timm test (torch dynamo/inductor cannot compile in the Nix sandbox) +# Only where torch comes from a wheel: against nixpkgs' own torch, timm comes from the binary cache. +// lib.optionalAttrs (wheelTorch && prev ? timm) { timm = prev.timm.overridePythonAttrs (old: { disabledTests = (old.disabledTests or [ ]) ++ [ "test_kron" ]; # test_optim needs setuptools at runtime (torch dynamo/inductor) @@ -1088,15 +1121,6 @@ lib.optionalAttrs useCuda { }); } -# mss runs screenshot tests through a virtual X server. They are not relevant -# to ComfyUI's runtime dependency closure and can fail in a headless Nix build -# sandbox. Remove this override once nixpkgs' mss checks are sandbox-independent. -// lib.optionalAttrs (prev ? mss) { - mss = prev.mss.overridePythonAttrs (_old: { - doCheck = false; - }); -} - # torchsde's test suite costs ~3.5 minutes of every build of this flake, and it # is statistical rather than functional: test_normality_conditional asserts a # Kolmogorov-Smirnov p-value stays above 1e-5, so a borderline draw lands under @@ -1104,26 +1128,15 @@ lib.optionalAttrs useCuda { # suite rather than just that test buys back the build time; pythonImportsCheck # still runs, and ComfyUI only uses torchsde's samplers. # See: https://github.com/utensils/comfyui-nix/issues/91 -// lib.optionalAttrs (prev ? torchsde) { +# +# Only where torch comes from a wheel: against nixpkgs' own torch, +# torchsde comes from the binary cache and its suite never runs here. +// lib.optionalAttrs (wheelTorch && prev ? torchsde) { torchsde = prev.torchsde.overridePythonAttrs (_old: { doCheck = false; }); } -# Disable filterpy tests on Darwin (test_hinfinity triggers BPT trap in pytest) -// lib.optionalAttrs (prev ? filterpy) { - filterpy = prev.filterpy.overridePythonAttrs (old: { - doCheck = if pkgs.stdenv.hostPlatform.isDarwin then false else (old.doCheck or true); - }); -} - -# Fix bitsandbytes build - needs ninja for wheel building phase -// lib.optionalAttrs (prev ? bitsandbytes) { - bitsandbytes = prev.bitsandbytes.overridePythonAttrs (old: { - nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ final.ninja ]; - }); -} - # color-matcher - not in older nixpkgs, needed for KJNodes // { "color-matcher" = final.buildPythonPackage rec { diff --git a/nix/python.nix b/nix/python.nix new file mode 100644 index 0000000..fe08c5d --- /dev/null +++ b/nix/python.nix @@ -0,0 +1,11 @@ +# Base CPython interpreter shared by every package set in this flake. +# +# Linux follows upstream ComfyUI, +# which calls Python 3.13 "very well supported" and ships it in the official portable builds. +# +# Apple Silicon stays on 3.12. +# torch is pinned to 2.5.1 there because of MPS regressions on macOS 26, +# and that release has no cp313 macOS wheel. +# The first one is torch 2.6.0. +{ pkgs }: +if pkgs.stdenv.hostPlatform.isDarwin then pkgs.python312 else pkgs.python313 diff --git a/nix/versions.nix b/nix/versions.nix index e256f3e..feec036 100644 --- a/nix/versions.nix +++ b/nix/versions.nix @@ -180,59 +180,71 @@ # These avoid compiling PyTorch from source (which requires 30-60GB RAM) # CUDA wheels bundle CUDA libraries, so no separate CUDA toolkit needed at runtime # macOS wheels use PyTorch 2.5.1 to avoid MPS issues on macOS 26 (Tahoe) + # + # These wheels are built per interpreter. + # "{py}" in a URL stands for the interpreter tag of the Python selected in python.nix (e.g. "cp313"), + # and `hashes` holds one hash per tag. + # To move to another Python, add its hash here. + # python-overrides.nix resolves the rest. pytorchWheels = { # macOS Apple Silicon (arm64) - PyTorch 2.5.1 (2.9.x has MPS bugs on macOS 26) darwinArm64 = { torch = { version = "2.5.1"; - url = "https://download.pytorch.org/whl/cpu/torch-2.5.1-cp312-none-macosx_11_0_arm64.whl"; - hash = "sha256-jHEt9hEBlk6xGRCoRlFAEfC29ZIMVdv1Z7/4o0Fj1bE="; + url = "https://download.pytorch.org/whl/cpu/torch-2.5.1-{py}-none-macosx_11_0_arm64.whl"; + hashes.cp312 = "sha256-jHEt9hEBlk6xGRCoRlFAEfC29ZIMVdv1Z7/4o0Fj1bE="; }; torchvision = { version = "0.20.1"; - url = "https://download.pytorch.org/whl/cpu/torchvision-0.20.1-cp312-cp312-macosx_11_0_arm64.whl"; - hash = "sha256-GjElb/lF1k8Aa7MGgTp8laUx/ha/slNcg33UwQRTPXo="; + url = "https://download.pytorch.org/whl/cpu/torchvision-0.20.1-{py}-{py}-macosx_11_0_arm64.whl"; + hashes.cp312 = "sha256-GjElb/lF1k8Aa7MGgTp8laUx/ha/slNcg33UwQRTPXo="; }; torchaudio = { version = "2.5.1"; - url = "https://download.pytorch.org/whl/cpu/torchaudio-2.5.1-cp312-cp312-macosx_11_0_arm64.whl"; - hash = "sha256-8cv9/Ru9++conUenTzb/bF2HwyBWBiAv71p/tpP2HPA="; + url = "https://download.pytorch.org/whl/cpu/torchaudio-2.5.1-{py}-{py}-macosx_11_0_arm64.whl"; + hashes.cp312 = "sha256-8cv9/Ru9++conUenTzb/bF2HwyBWBiAv71p/tpP2HPA="; }; }; # Linux x86_64 CUDA 13.0 cu130 = { torch = { version = "2.10.0"; - url = "https://download.pytorch.org/whl/cu130/torch-2.10.0%2Bcu130-cp312-cp312-manylinux_2_28_x86_64.whl"; - hash = "sha256-hY8MvMeNcm/qlJnrNGT6qYOS+gk4RaMmIgm9Imt4RNY="; + url = "https://download.pytorch.org/whl/cu130/torch-2.10.0%2Bcu130-{py}-{py}-manylinux_2_28_x86_64.whl"; + hashes.cp312 = "sha256-hY8MvMeNcm/qlJnrNGT6qYOS+gk4RaMmIgm9Imt4RNY="; + hashes.cp313 = "sha256-fg2ZIunpH3gLJ2GgxeusPBXJdAurBC4bWRSa+m1kdOs="; }; torchvision = { version = "0.25.0"; - url = "https://download.pytorch.org/whl/cu130/torchvision-0.25.0%2Bcu130-cp312-cp312-manylinux_2_28_x86_64.whl"; - hash = "sha256-wvXjjwzVeieW5FA8DxM2XeugHbwWfvgg8L7sfKlvXy4="; + url = "https://download.pytorch.org/whl/cu130/torchvision-0.25.0%2Bcu130-{py}-{py}-manylinux_2_28_x86_64.whl"; + hashes.cp312 = "sha256-wvXjjwzVeieW5FA8DxM2XeugHbwWfvgg8L7sfKlvXy4="; + hashes.cp313 = "sha256-6FM31Zvb9wBs0MdgEto2Y97VYG2txo9yKA2fKrHpGRo="; }; torchaudio = { version = "2.10.0"; - url = "https://download.pytorch.org/whl/cu130/torchaudio-2.10.0%2Bcu130-cp312-cp312-manylinux_2_28_x86_64.whl"; - hash = "sha256-7Vc+cxm6mm4Te0iV7d9Mn81CS7ThCmKSZ9ZLVazFf7s="; + url = "https://download.pytorch.org/whl/cu130/torchaudio-2.10.0%2Bcu130-{py}-{py}-manylinux_2_28_x86_64.whl"; + hashes.cp312 = "sha256-7Vc+cxm6mm4Te0iV7d9Mn81CS7ThCmKSZ9ZLVazFf7s="; + hashes.cp313 = "sha256-ixwxw885dQ1HZ1K9ZQimz7G/rx1vDYhe195Kln7VLus="; }; }; # Linux x86_64 ROCm 7.1 rocm71 = { torch = { version = "2.10.0"; - url = "https://download.pytorch.org/whl/rocm7.1/torch-2.10.0%2Brocm7.1-cp312-cp312-manylinux_2_28_x86_64.whl"; - hash = "sha256-AI7g13u4tfn07h8AISAZxGGRcePEGV3lbyUzMbO8Mg0="; + url = "https://download.pytorch.org/whl/rocm7.1/torch-2.10.0%2Brocm7.1-{py}-{py}-manylinux_2_28_x86_64.whl"; + hashes.cp312 = "sha256-AI7g13u4tfn07h8AISAZxGGRcePEGV3lbyUzMbO8Mg0="; + hashes.cp313 = "sha256-PxiwbyM9SHG6qv24umYq5p/5HsEaob92ynbd4xtD6c0="; }; torchvision = { version = "0.25.0"; - url = "https://download.pytorch.org/whl/rocm7.1/torchvision-0.25.0%2Brocm7.1-cp312-cp312-manylinux_2_28_x86_64.whl"; - hash = "sha256-iuo929t0gB0zdFd6ELPwTUmJfCet0jXLJTE99uZbGSk="; + url = "https://download.pytorch.org/whl/rocm7.1/torchvision-0.25.0%2Brocm7.1-{py}-{py}-manylinux_2_28_x86_64.whl"; + hashes.cp312 = "sha256-iuo929t0gB0zdFd6ELPwTUmJfCet0jXLJTE99uZbGSk="; + hashes.cp313 = "sha256-VMBvKeIO+jZSLu3MWg+28DYn/w/yPMe+9G9xowQtsHQ="; }; torchaudio = { version = "2.10.0"; - url = "https://download.pytorch.org/whl/rocm7.1/torchaudio-2.10.0%2Brocm7.1-cp312-cp312-manylinux_2_28_x86_64.whl"; - hash = "sha256-pUuMH2HeAbGrlGWJqgFYId0RCTVEcCJJ0gq6VpE8aLs="; + url = "https://download.pytorch.org/whl/rocm7.1/torchaudio-2.10.0%2Brocm7.1-{py}-{py}-manylinux_2_28_x86_64.whl"; + hashes.cp312 = "sha256-pUuMH2HeAbGrlGWJqgFYId0RCTVEcCJJ0gq6VpE8aLs="; + hashes.cp313 = "sha256-OmI7RHzbLDuQ/deOz6+lE3gvD3HKlWEgAFG6fVBwS3Y="; }; }; # Linux x86_64 ROCm 7.2 @@ -248,18 +260,21 @@ rocm72 = { torch = { version = "2.14.1"; - url = "https://download.pytorch.org/whl/rocm7.2/torch-2.14.1%2Brocm7.2-cp312-cp312-manylinux_2_28_x86_64.whl"; - hash = "sha256-/DXkj9gzKfXWCrkllR0Tjttab86EMjYAcDY3+8aTVBw="; + url = "https://download.pytorch.org/whl/rocm7.2/torch-2.14.1%2Brocm7.2-{py}-{py}-manylinux_2_28_x86_64.whl"; + hashes.cp312 = "sha256-/DXkj9gzKfXWCrkllR0Tjttab86EMjYAcDY3+8aTVBw="; + hashes.cp313 = "sha256-u5dlN83aOOxV7RhFLK5636Y2op0WCbuOl/6OW2pSh3g="; }; torchvision = { version = "0.29.1"; - url = "https://download.pytorch.org/whl/rocm7.2/torchvision-0.29.1%2Brocm7.2-cp312-cp312-manylinux_2_28_x86_64.whl"; - hash = "sha256-wIfdLaVB6kQGTZVdmTy+jotS20NnR8xpDXxqBf8MuUs="; + url = "https://download.pytorch.org/whl/rocm7.2/torchvision-0.29.1%2Brocm7.2-{py}-{py}-manylinux_2_28_x86_64.whl"; + hashes.cp312 = "sha256-wIfdLaVB6kQGTZVdmTy+jotS20NnR8xpDXxqBf8MuUs="; + hashes.cp313 = "sha256-5ghqFc5kY2GY2UmnnxSnN/VAvFk3NgLC9Hmvs81aQGQ="; }; torchaudio = { version = "2.11.0"; - url = "https://download.pytorch.org/whl/rocm7.2/torchaudio-2.11.0%2Brocm7.2-cp312-cp312-manylinux_2_28_x86_64.whl"; - hash = "sha256-4DFtk4NGz6T/BLkUBakezFC9GHgQI+yldjt58Pe5vIw="; + url = "https://download.pytorch.org/whl/rocm7.2/torchaudio-2.11.0%2Brocm7.2-{py}-{py}-manylinux_2_28_x86_64.whl"; + hashes.cp312 = "sha256-4DFtk4NGz6T/BLkUBakezFC9GHgQI+yldjt58Pe5vIw="; + hashes.cp313 = "sha256-kijhxTTQTYD6e0YgX0SNqUf9BvVXO6hgLwQbUE7bNZY="; }; }; # Linux x86_64 Intel XPU (oneAPI / SYCL) @@ -274,18 +289,21 @@ xpu = { torch = { version = "2.10.0"; - url = "https://download.pytorch.org/whl/xpu/torch-2.10.0%2Bxpu-cp312-cp312-linux_x86_64.whl"; - hash = "sha256-tHNXHUeJEvkogcwT8V+hj4Rj+w+4oGjJbtR6fUWk2go="; + url = "https://download.pytorch.org/whl/xpu/torch-2.10.0%2Bxpu-{py}-{py}-linux_x86_64.whl"; + hashes.cp312 = "sha256-tHNXHUeJEvkogcwT8V+hj4Rj+w+4oGjJbtR6fUWk2go="; + hashes.cp313 = "sha256-O8ZKdG/yWpPeFAkCxgyegZ10E/XOoeiNgJmcJ6WQHpw="; }; torchvision = { version = "0.25.0"; - url = "https://download.pytorch.org/whl/xpu/torchvision-0.25.0%2Bxpu-cp312-cp312-manylinux_2_28_x86_64.whl"; - hash = "sha256-atJUNJa8KeWdPdYUqU0JqphwMYrttmBFNE//3f7dLPg="; + url = "https://download.pytorch.org/whl/xpu/torchvision-0.25.0%2Bxpu-{py}-{py}-manylinux_2_28_x86_64.whl"; + hashes.cp312 = "sha256-atJUNJa8KeWdPdYUqU0JqphwMYrttmBFNE//3f7dLPg="; + hashes.cp313 = "sha256-gCafN4ZfzYtX8g5Hhu+uIgC/orJyeSbDx6zILw59NUg="; }; torchaudio = { version = "2.10.0"; - url = "https://download.pytorch.org/whl/xpu/torchaudio-2.10.0%2Bxpu-cp312-cp312-manylinux_2_28_x86_64.whl"; - hash = "sha256-oD/un3Hi9OZC/P8vaokN6kYwm7cGrahDr40K5CBa4p8="; + url = "https://download.pytorch.org/whl/xpu/torchaudio-2.10.0%2Bxpu-{py}-{py}-manylinux_2_28_x86_64.whl"; + hashes.cp312 = "sha256-oD/un3Hi9OZC/P8vaokN6kYwm7cGrahDr40K5CBa4p8="; + hashes.cp313 = "sha256-KWSjQHSaW5UeRT1ew+z0l7DDPfKdktCbmPHF/Bf4u2k="; }; }; }; @@ -402,8 +420,9 @@ }; triton-xpu = { version = "3.6.0"; - url = "https://download.pytorch.org/whl/triton_xpu-3.6.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl"; - hash = "sha256-e6neluXTZAC5lrn7nrOEQxlkU678Efr+GnuyJrxfuLs="; + url = "https://download.pytorch.org/whl/triton_xpu-3.6.0-{py}-{py}-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl"; + hashes.cp312 = "sha256-e6neluXTZAC5lrn7nrOEQxlkU678Efr+GnuyJrxfuLs="; + hashes.cp313 = "sha256-b2i0GTEJkACA6I8I7jXV/ifboM91BOu+7HS/6sBmrJU="; }; }; diff --git a/pyproject.toml b/pyproject.toml index da3ce2c..2c2f6e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "comfyui-nix" version = "0.7.0" -description = "A Nix flake for ComfyUI with Python 3.12" +description = "A Nix flake for ComfyUI with Python 3.13" requires-python = ">=3.12" [tool.ruff]