Merge pull request #10 from marienz/fetchtree

Improve fetching performance (mainly for CI)

.github/workflows/ci.yml

@@ -48,6 +48,8 @@ jobs:
         uses: DeterminateSystems/nix-installer-action@main
       - name: Enable Magic Nix Cache
         uses: DeterminateSystems/magic-nix-cache-action@main
+        with:
+          use-flakehub: false
       - name: Update flake.lock
         if: github.event_name == 'schedule' || ( github.event_name == 'workflow_dispatch' && inputs.updateFlakeLock )
         run: |
@@ -56,12 +58,26 @@ jobs:
           nix flake update --commit-lock-file
       - name: Check flake.lock
         uses: DeterminateSystems/flake-checker-action@main
+      # Update the caches daily, flush the cache monthly.
+      - name: Set cache keys
+        id: cache-keys
+        run: |
+          {
+            echo "key=$(date +'%Y-%m-%d')"
+            echo "restore=$(date +'%Y-%m-')"
+          } >> "$GITHUB_OUTPUT"
       - name: Cache git checkouts
         uses: actions/cache@v4
         with:
           path: ~/.cache/nix/gitv3
-          key: nix-gitv3-cache-${{ hashFiles('flake.lock') }}
-          restore-keys: nix-gitv3-cache-
+          key: nix-gitv3-cache-${{ steps.cache-keys.outputs.key }}
+          restore-keys: nix-gitv3-cache-${{ steps.cache-keys.outputs.restore }}
+      - name: Cache tarballs
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/nix/tarball-cache
+          key: nix-tarball-cache-${{ steps.cache-keys.outputs.key }}
+          restore-keys: nix-tarball-cache-${{ steps.cache-keys.outputs.restore }}
       - name: nix flake check
         run: nix flake check -L --show-trace
       - name: Build packages for Cachix

checks.nix

@@ -31,6 +31,7 @@ let
     # TODO: drop after NixOS 24.05 release.
     emacs = emacs29;
     doomLocalDir = "~/.local/share/nix-doom-unstraightened";
+    experimentalFetchTree = true;
   };
   mkDoom = args: (makeDoomPackages (common // args)).doomEmacs;
   mkDoomDir = args: writeTextDir "init.el" (toInit args);

default.nix

@@ -39,6 +39,8 @@
   profileName ? "nix",
   /* Disable profile early in startup, so "normal" cache/state dirs are used. */
   noProfileHack ? false,
+  /* Use fetchTree instead of fetchGit for package fetches. */
+  experimentalFetchTree ? false,
 
   callPackages,
   git,
@@ -233,14 +235,15 @@ let
                   else (throw "${name}: cannot derive url from recipe ${p.recipe or "<missing>"}"));
             # Use builtins.fetchGit instead of nixpkgs's fetchFromGitHub because
             # fetchGit allows fetching a specific git commit without a hash.
-            # TODO: port to fetchTree once (mostly) stable
-            # (in particular the github fetcher may be noticably more efficient)
-            src = builtins.fetchGit (
-              {
+            fetchGitArgs = {
               inherit url;
               rev = pin;
               allRefs = true;
-                submodules = !(p.recipe.nonrecursive or false);
+              # Skip submodules by default because they seem to be hitting
+              # https://github.com/NixOS/nix/issues/10773 (or a similar caching issue) and for
+              # parity between fetchTree's github fetcher and fetchGit (Github's exports don't
+              # seem to contain submodules).
+              submodules = !(p.recipe.nonrecursive or true);
               # TODO: pull ref from derivation.src when not pulling it from p.recipe?
               # Note Doom does have packages with pin + branch (or nonrecursive) set,
               # expecting to inherit the rest of the recipe from Straight.
@@ -248,8 +251,24 @@ let
               # Always specify a ref to work around https://github.com/NixOS/nix/issues/10773
               ref = p.recipe.branch or "HEAD";
             }
-              // optionalAttrs (p ? recipe.depth) { shallow = p.recipe.depth == 1; }
-            );
+            // optionalAttrs (p ? recipe.depth) { shallow = p.recipe.depth == 1; };
+            src =
+              if experimentalFetchTree
+              then builtins.fetchTree (
+                if lib.hasPrefix "https://github.com/" url
+                then let
+                  tail = lib.removePrefix "https://github.com/" url;
+                  split = lib.splitString "/" tail;
+                  owner = lib.head split;
+                  repo = lib.removeSuffix ".git" (lib.elemAt split 1);
+                in {
+                  type = "github";
+                  inherit owner repo;
+                  rev = pin;
+                } else ({
+                  type = "git";
+                } // fetchGitArgs))
+              else builtins.fetchGit fetchGitArgs;
             # Run locally to avoid a network roundtrip.
             reqfile = runCommandLocal "${name}-deps" { } ''
               ${lib.getExe emacs} -Q --batch --script \

doomdirs/example/init.el

@@ -15,88 +15,34 @@
 ;; limitations under the License.
 
 (doom! :completion
-       (company +childframe)
-       (corfu +orderless +icons +dabbrev)
        vertico
 
        :ui
        doom
        doom-dashboard
-       (emoji +unicode)
-       hl-todo
-       hydra
        modeline
        nav-flash
        ophints
        (popup +defaults)
-       (vc-gutter +pretty +diff-hl)
        window-select
 
        :editor
        evil
-       file-templates
-       format
-       snippets
 
        :emacs
-       dired
-       electric
-       ibuffer
        undo
-       vc
 
        :term
        eshell
        vterm
 
-       :checkers
-       (syntax +childframe)
-       spell
-
-       :tools
-       direnv
-       editorconfig
-       (eval +overlay)
-       (lookup +docsets)
-       (lsp +peek)
-       (magit +forge)
-       taskrunner
-
        :os
        (:if (featurep :system 'macos) macos)
        (tty +osc)
 
        :lang
-       (cc +lsp)
-       csharp
-       data
-       (dart +flutter +lsp)
        emacs-lisp
-       (go +lsp)
-       (graphql +lsp)
-       (haskell +lsp)
-       json
-       (java +lsp)
-       (javascript +lsp)
-       latex
-       lua
-       markdown
        (nix +lsp)
-       (org +pretty)
-       (python +lsp +pyright +cython)
-       (ruby +rails)
-       (rust +lsp)
-       (scheme +guile)
-       (sh +fish)
-       (web +css +html)
-       yaml
-
-       :email
-       (mu4e +org +gmail)
-       (wanderlust +gmail)
-
-       :app
-       (rss +org)
 
        :config
        (default +bindings +smartparens))

doomdirs/example/packages.el

@@ -18,10 +18,3 @@
 (package! evil-escape :disable t)
 
 (package! dotenv-mode)
-(package! nginx-mode)
-(package! protobuf-mode)
-(package! systemd)
-(package! meson-mode)
-(package! caddyfile-mode)
-(package! vala-mode)
-(package! bpftrace-mode)

flake.nix

@@ -101,6 +101,7 @@
             doomDir = ./doomdirs/minimal;
             doomLocalDir = "~/.local/share/nix-doom-unstraightened";
             full = true;
+            experimentalFetchTree = true;
           }).doomEmacs.emacsWithPackages.deps;
         };
       });