Support the MacOS sandbox for larger projects

CHANGELOG.md

@@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Fixed
 * Workspace inheritance of `lints` in git dependencies is now correctly handled
+* Crane can now build large (>500 crate) projects inside the OSX sandbox.
 
 ## [0.15.1] - 2023-11-30
 

docs/API.md

@@ -859,22 +859,24 @@ craneLib.devShell {
 }
 ```
 
-### `craneLib.downloadCargoPackage`
+### `craneLib.downloadCargoPackages`
 
-`downloadCargoPackage :: set -> drv`
+`downloadCargoPackages :: set -> drv`
 
-Download a packaged cargo crate (e.g. from crates.io) and prepare it for
-vendoring.
+Downloads a collection of cargo crates (e.g. from crates.io) and extract
+into a single directory.
 
 The registry's `fetchurlExtraArgs` will be passed through to `fetchurl` when
 downloading the crate, making it possible to influence interacting with the
 registry's API if necessary.
 
 #### Required input attributes
-* `checksum`: the (sha256) checksum recorded in the Cargo.lock file
-* `name`: the name of the crate
-* `source`: the source key recorded in the Cargo.lock file
-* `version`: the version of the crate
+* `shard`: an identifier for the shard. This will become a part of the derivation name.
+* `packages`: a list of attribute sets, each one containing:
+  * `checksum`: the (sha256) checksum recorded in the Cargo.lock file
+  * `name`: the name of the crate
+  * `source`: the source key recorded in the Cargo.lock file
+  * `version`: the version of the crate
 
 ### `craneLib.downloadCargoPackageFromGit`
 

lib/default.nix

@@ -43,7 +43,7 @@ in
     indexUrl = "https://github.com/rust-lang/crates.io-index";
   };
 
-  downloadCargoPackage = callPackage ./downloadCargoPackage.nix { };
+  downloadCargoPackages = callPackage ./downloadCargoPackages.nix { };
   downloadCargoPackageFromGit = callPackage ./downloadCargoPackageFromGit.nix { };
   filterCargoSources = callPackage ./filterCargoSources.nix { };
   findCargoFiles = callPackage ./findCargoFiles.nix { };

lib/downloadCargoPackages.nix

@@ -0,0 +1,37 @@
+{ fetchurl
+, urlForCargoPackage
+, runCommand
+, lib
+}:
+
+{ shardName, packages }:
+let
+  getTarball = ({ name, version, checksum, ... }@args:
+    let
+      pkgInfo = urlForCargoPackage args;
+    in
+    {
+      inherit name version checksum;
+      tarball = fetchurl (pkgInfo.fetchurlExtraArgs // {
+        inherit (pkgInfo) url;
+        name = "${name}-${version}";
+        sha256 = checksum;
+      });
+    });
+
+  tarballs = map getTarball packages;
+
+  extract = (tarball:
+    let outPath = "$out/${lib.escapeShellArg "${tarball.name}-${tarball.version}"}";
+    in ''
+      mkdir -p ${outPath}
+      tar -xf ${tarball.tarball} -C ${outPath} --strip-components=1
+      echo '{"files":{}, "package":"${tarball.checksum}"}' > ${outPath}/.cargo-checksum.json
+    '');
+
+  name = if builtins.stringLength shardName == 0 then "extract-cargo-packages" else "extract-cargo-packages-${shardName}";
+in
+runCommand name { } ''
+  mkdir -p $out
+  ${lib.strings.concatMapStrings extract tarballs}
+''

lib/vendorCargoRegistries.nix

@@ -1,4 +1,4 @@
-{ downloadCargoPackage
+{ downloadCargoPackages
 , lib
 , runCommandLocal
 }:
@@ -44,12 +44,22 @@ let
     lockPackages;
   lockedRegistryGroups = groupBy (p: p.source) lockedPackagesFromRegistry;
 
-  vendorSingleRegistry = packages: runCommandLocal "vendor-registry" { } ''
-    mkdir -p $out
-    ${concatMapStrings (p: ''
-      ln -s ${escapeShellArg (downloadCargoPackage p)} $out/${escapeShellArg "${p.name}-${p.version}"}
-    '') packages}
-  '';
+  vendorSingleRegistry = (packages:
+    let
+      packageCount = builtins.length packages;
+      shardCharacters = if packageCount < 32 then 0 else if packageCount < 2048 then 1 else 2;
+      shard = package: builtins.substring 0 shardCharacters (builtins.hashString "sha256" package.name);
+      grouped = builtins.groupBy shard packages;
+      vendoredShards = builtins.mapAttrs (shardName: packages: downloadCargoPackages { inherit shardName packages; }) grouped;
+      vendoredShardsList = builtins.attrValues vendoredShards;
+    in
+    runCommandLocal "vendor-registry" { } ''
+      mkdir -p $out
+      ${concatMapStrings (p: ''
+      for dir in ${p}/*/; do ln -s "$(realpath "$dir")" "$out/''${dir##*/}"; done
+      '') vendoredShardsList}
+    ''
+  );
 
   parsedCargoConfigTomls = map (p: builtins.fromTOML (readFile p)) cargoConfigs;
   allCargoRegistries = flatten (map (c: c.registries or [ ]) parsedCargoConfigTomls);