From 645f79a916bad35fa5710ba4ad7465c0799b3659 Mon Sep 17 00:00:00 2001 From: Marien Zwart Date: Sun, 26 May 2024 21:59:11 +1000 Subject: [PATCH 1/5] Disable submodule fetching by default This may turn out to be too much (but it does at least build). Motivation: CI's git checkouts consume an unmanageably large amount of cache (over 3 GiB per snapshot out of 10 GiB quota), and must be cached for acceptable build speeds. Dropping submodules should help somewhat directly, but I want to try switching most of CI over to fetchTree's github fetcher, which won't include submodules. This change should help maintain parity. I'm also seeing a submodule fetch failure in CI (for stan-mode) that I may not need to debug if the package functions without that submodule. --- default.nix | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/default.nix b/default.nix index e6039e9..24f4558 100644 --- a/default.nix +++ b/default.nix @@ -240,7 +240,11 @@ let inherit url; rev = pin; allRefs = true; - submodules = !(p.recipe.nonrecursive or false); + # Skip submodules by default because they seem to be hitting + # https://github.com/NixOS/nix/issues/10773 (or a similar caching issue) and for + # parity between fetchTree's github fetcher and fetchGit (Github's exports don't + # seem to contain submodules). + submodules = !(p.recipe.nonrecursive or true); # TODO: pull ref from derivation.src when not pulling it from p.recipe? # Note Doom does have packages with pin + branch (or nonrecursive) set, # expecting to inherit the rest of the recipe from Straight. From 2af26fcfa1795543d20ffddd18d6a6b84ea38fe4 Mon Sep 17 00:00:00 2001 From: Marien Zwart Date: Sun, 26 May 2024 22:42:51 +1000 Subject: [PATCH 2/5] Use fetchTree / github fetcher for CI This seems to be much more space-efficient: ~/.cache/nix/tarball-cache is about 700MiB uncompressed, under 300MiB as tzst (using tar's defaults, CI uses zstdmt but I assume will be in the same ballpark). The gitv3 cache is multiple GiB. CI will still build doom-example using fetchGit. I intend to shrink the number of modules enabled in the example to keep gitv3 cache size under control. --- checks.nix | 1 + default.nix | 55 ++++++++++++++++++++++++++++++++++------------------- flake.nix | 1 + 3 files changed, 37 insertions(+), 20 deletions(-) diff --git a/checks.nix b/checks.nix index b0cc7a8..f2695af 100644 --- a/checks.nix +++ b/checks.nix @@ -31,6 +31,7 @@ let # TODO: drop after NixOS 24.05 release. emacs = emacs29; doomLocalDir = "~/.local/share/nix-doom-unstraightened"; + experimentalFetchTree = true; }; mkDoom = args: (makeDoomPackages (common // args)).doomEmacs; mkDoomDir = args: writeTextDir "init.el" (toInit args); diff --git a/default.nix b/default.nix index 24f4558..506d4ff 100644 --- a/default.nix +++ b/default.nix @@ -39,6 +39,8 @@ profileName ? "nix", /* Disable profile early in startup, so "normal" cache/state dirs are used. */ noProfileHack ? false, + /* Use fetchTree instead of fetchGit for package fetches. */ + experimentalFetchTree ? false, callPackages, git, @@ -233,27 +235,40 @@ let else (throw "${name}: cannot derive url from recipe ${p.recipe or ""}")); # Use builtins.fetchGit instead of nixpkgs's fetchFromGitHub because # fetchGit allows fetching a specific git commit without a hash. - # TODO: port to fetchTree once (mostly) stable - # (in particular the github fetcher may be noticably more efficient) - src = builtins.fetchGit ( - { - inherit url; - rev = pin; - allRefs = true; - # Skip submodules by default because they seem to be hitting - # https://github.com/NixOS/nix/issues/10773 (or a similar caching issue) and for - # parity between fetchTree's github fetcher and fetchGit (Github's exports don't - # seem to contain submodules). - submodules = !(p.recipe.nonrecursive or true); - # TODO: pull ref from derivation.src when not pulling it from p.recipe? - # Note Doom does have packages with pin + branch (or nonrecursive) set, - # expecting to inherit the rest of the recipe from Straight. + fetchGitArgs = { + inherit url; + rev = pin; + allRefs = true; + # Skip submodules by default because they seem to be hitting + # https://github.com/NixOS/nix/issues/10773 (or a similar caching issue) and for + # parity between fetchTree's github fetcher and fetchGit (Github's exports don't + # seem to contain submodules). + submodules = !(p.recipe.nonrecursive or true); + # TODO: pull ref from derivation.src when not pulling it from p.recipe? + # Note Doom does have packages with pin + branch (or nonrecursive) set, + # expecting to inherit the rest of the recipe from Straight. - # Always specify a ref to work around https://github.com/NixOS/nix/issues/10773 - ref = p.recipe.branch or "HEAD"; - } - // optionalAttrs (p ? recipe.depth) { shallow = p.recipe.depth == 1; } - ); + # Always specify a ref to work around https://github.com/NixOS/nix/issues/10773 + ref = p.recipe.branch or "HEAD"; + } + // optionalAttrs (p ? recipe.depth) { shallow = p.recipe.depth == 1; }; + src = + if experimentalFetchTree + then builtins.fetchTree ( + if lib.hasPrefix "https://github.com/" url + then let + tail = lib.removePrefix "https://github.com/" url; + split = lib.splitString "/" tail; + owner = lib.head split; + repo = lib.removeSuffix ".git" (lib.elemAt split 1); + in { + type = "github"; + inherit owner repo; + rev = pin; + } else ({ + type = "git"; + } // fetchGitArgs)) + else builtins.fetchGit fetchGitArgs; # Run locally to avoid a network roundtrip. reqfile = runCommandLocal "${name}-deps" { } '' ${lib.getExe emacs} -Q --batch --script \ diff --git a/flake.nix b/flake.nix index c015a1f..712c321 100644 --- a/flake.nix +++ b/flake.nix @@ -101,6 +101,7 @@ doomDir = ./doomdirs/minimal; doomLocalDir = "~/.local/share/nix-doom-unstraightened"; full = true; + experimentalFetchTree = true; }).doomEmacs.emacsWithPackages.deps; }; }); From 0e996d2ccba48e5bd188fbffd5adc7593e6bdd1a Mon Sep 17 00:00:00 2001 From: Marien Zwart Date: Sun, 26 May 2024 23:01:30 +1000 Subject: [PATCH 3/5] Rework CI caching, adding tarball cache Using the lockfile hash as cache key did not make much sense, since changes to Unstraightened itself change what we fetch. And a static restore key means we never flush our cache: that might be ok if cache size was reasonable, but it is not (over 3 GiB). Switch to using the date as cache key, and only restore caches from the current month. Add tarballs as a second cache. I considered caching all of ~/.cache/nix, but do not want to find out the hard way any (new) caches are not portable if I add CI for other platforms. The tarball-cache seems to be another git tree, so it should be ok. --- .github/workflows/ci.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b05900..fc6929b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,12 +56,26 @@ jobs: nix flake update --commit-lock-file - name: Check flake.lock uses: DeterminateSystems/flake-checker-action@main + # Update the caches daily, flush the cache monthly. + - name: Set cache keys + id: cache-keys + run: | + { + echo "key=$(date +'%Y-%m-%d')" + echo "restore=$(date +'%Y-%m-')" + } >> "$GITHUB_OUTPUT" - name: Cache git checkouts uses: actions/cache@v4 with: path: ~/.cache/nix/gitv3 - key: nix-gitv3-cache-${{ hashFiles('flake.lock') }} - restore-keys: nix-gitv3-cache- + key: nix-gitv3-cache-${{ steps.cache-keys.outputs.key }} + restore-keys: nix-gitv3-cache-${{ steps.cache-keys.outputs.restore }} + - name: Cache tarballs + uses: actions/cache@v4 + with: + path: ~/.cache/nix/tarball-cache + key: nix-tarball-cache-${{ steps.cache-keys.outputs.key }} + restore-keys: nix-tarball-cache-${{ steps.cache-keys.outputs.restore }} - name: nix flake check run: nix flake check -L --show-trace - name: Build packages for Cachix From af7c87700806783249a33742bc46e2ec7c3cf17e Mon Sep 17 00:00:00 2001 From: Marien Zwart Date: Sun, 26 May 2024 23:14:54 +1000 Subject: [PATCH 4/5] Prune dependencies of example doomdir This now matters for CI, as the example is still built using fetchGit instead of the github fetcher. The example was doing double duty as an example and a stress-test, enabling some problematic or interesting modules. That is no longer necessary (once the branch with dynamically generated init.el with all modules/flags enabled lands). --- doomdirs/example/init.el | 54 ------------------------------------ doomdirs/example/packages.el | 7 ----- 2 files changed, 61 deletions(-) diff --git a/doomdirs/example/init.el b/doomdirs/example/init.el index f46253f..16e3eff 100644 --- a/doomdirs/example/init.el +++ b/doomdirs/example/init.el @@ -15,88 +15,34 @@ ;; limitations under the License. (doom! :completion - (company +childframe) - (corfu +orderless +icons +dabbrev) vertico :ui doom doom-dashboard - (emoji +unicode) - hl-todo - hydra modeline nav-flash ophints (popup +defaults) - (vc-gutter +pretty +diff-hl) window-select :editor evil - file-templates - format - snippets :emacs - dired - electric - ibuffer undo - vc :term eshell vterm - :checkers - (syntax +childframe) - spell - - :tools - direnv - editorconfig - (eval +overlay) - (lookup +docsets) - (lsp +peek) - (magit +forge) - taskrunner - :os (:if (featurep :system 'macos) macos) (tty +osc) :lang - (cc +lsp) - csharp - data - (dart +flutter +lsp) emacs-lisp - (go +lsp) - (graphql +lsp) - (haskell +lsp) - json - (java +lsp) - (javascript +lsp) - latex - lua - markdown (nix +lsp) - (org +pretty) - (python +lsp +pyright +cython) - (ruby +rails) - (rust +lsp) - (scheme +guile) - (sh +fish) - (web +css +html) - yaml - - :email - (mu4e +org +gmail) - (wanderlust +gmail) - - :app - (rss +org) :config (default +bindings +smartparens)) diff --git a/doomdirs/example/packages.el b/doomdirs/example/packages.el index 9a191fc..6b2bdba 100644 --- a/doomdirs/example/packages.el +++ b/doomdirs/example/packages.el @@ -18,10 +18,3 @@ (package! evil-escape :disable t) (package! dotenv-mode) -(package! nginx-mode) -(package! protobuf-mode) -(package! systemd) -(package! meson-mode) -(package! caddyfile-mode) -(package! vala-mode) -(package! bpftrace-mode) From c042f8d6f040048e8b7b4f76b4b47f88cbea6d45 Mon Sep 17 00:00:00 2001 From: Marien Zwart Date: Sun, 26 May 2024 23:29:18 +1000 Subject: [PATCH 5/5] Disable magic nix cache's flakehub integration This should not be doing anything, and I saw it stall (to retry requests) once or twice. --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc6929b..85ec2f6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,6 +48,8 @@ jobs: uses: DeterminateSystems/nix-installer-action@main - name: Enable Magic Nix Cache uses: DeterminateSystems/magic-nix-cache-action@main + with: + use-flakehub: false - name: Update flake.lock if: github.event_name == 'schedule' || ( github.event_name == 'workflow_dispatch' && inputs.updateFlakeLock ) run: |