Set empty CARGO_ENCODED_RUSTFLAGS in workspace metadata retrieval (#249)

Prevents wiping the bin directory, which is harmful for
self-hosted runners.

.github/workflows/buildjet.yml

@@ -16,7 +16,7 @@ jobs:
       CARGO_TERM_COLOR: always
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - run: rustup toolchain install stable --profile minimal --no-self-update
 

.github/workflows/check-dist.yml

@@ -1,26 +1,26 @@
 name: check dist/
 
 on:
-   push:
-     branches:
-       - master
-     paths-ignore:
-       - '**.md'
-   pull_request:
-     paths-ignore:
-       - '**.md'
-   workflow_dispatch:
+  push:
+    branches:
+      - master
+    paths-ignore:
+      - "**.md"
+  pull_request:
+    paths-ignore:
+      - "**.md"
+  workflow_dispatch:
 
 jobs:
   check-dist:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - name: Setup Node.js 16.x
-        uses: actions/setup-node@v3
+      - name: Setup Node.js 20.x
+        uses: actions/setup-node@v4
         with:
-          node-version: 16.x
+          node-version: 20.x
           cache: npm
 
       - name: Install dependencies
@@ -38,7 +38,7 @@ jobs:
           fi
         id: diff
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: ${{ failure() && steps.diff.conclusion == 'failure' }}
         with:
           name: dist

.github/workflows/coverage.yml

@@ -16,7 +16,7 @@ jobs:
       CARGO_TERM_COLOR: always
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - run: rustup toolchain install stable --profile minimal --component llvm-tools-preview --no-self-update
 

.github/workflows/git-registry.yml

@@ -17,7 +17,7 @@ jobs:
       CARGO_REGISTRIES_CRATES_IO_PROTOCOL: git
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - run: rustup toolchain install stable --profile minimal --no-self-update
 

.github/workflows/install.yml

@@ -16,7 +16,7 @@ jobs:
       CARGO_TERM_COLOR: always
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - run: rustup toolchain install stable --profile minimal --no-self-update
 

.github/workflows/simple.yml

@@ -16,7 +16,7 @@ jobs:
       CARGO_TERM_COLOR: always
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - run: rustup toolchain install stable --profile minimal --no-self-update
 

.github/workflows/target-dir.yml

@@ -16,7 +16,7 @@ jobs:
       CARGO_TERM_COLOR: always
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - run: rustup toolchain install stable --profile minimal --no-self-update
 

.github/workflows/workspaces.yml

@@ -16,7 +16,7 @@ jobs:
       CARGO_TERM_COLOR: always
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - run: rustup toolchain install stable --profile minimal --target wasm32-unknown-unknown --no-self-update
 

CHANGELOG.md

@@ -1,5 +1,41 @@
 # Changelog
 
+## 2.8.0
+
+- Add support for `warpbuild` cache provider
+- Add new `cache-workspace-crates` feature
+
+## 2.7.8
+
+- Include CPU arch in the cache key
+
+## 2.7.7
+
+- Also cache `cargo install` metadata
+
+## 2.7.6
+
+- Allow opting out of caching $CARGO_HOME/bin
+- Add runner OS in cache key
+- Adds an option to do lookup-only of the cache
+
+## 2.7.5
+
+- Support Cargo.lock format cargo-lock v4
+- Only run macOsWorkaround() on macOS
+
+## 2.7.3
+
+- Work around upstream problem that causes cache saving to hang for minutes.
+
+## 2.7.2
+
+- Only key by `Cargo.toml` and `Cargo.lock` files of workspace members.
+
+## 2.7.1
+
+- Update toml parser to fix parsing errors.
+
 ## 2.7.0
 
 - Properly cache `trybuild` tests.

README.md

@@ -6,7 +6,7 @@ sensible defaults.
 ## Example usage
 
 ```yaml
-- uses: actions/checkout@v3
+- uses: actions/checkout@v4
 
 # selecting a toolchain either by action or manual `rustup` calls should happen
 # before the plugin, as the cache uses the current rustc version as its cache key
@@ -60,7 +60,13 @@ sensible defaults.
     # default: "false"
     cache-all-crates: ""
 
-    # Determiners whether the cache should be saved.
+    # Similar to cache-all-crates.
+    # If `true` the workspace crates will be cached.
+    # Useful if the workspace contains libraries that are only updated sporadically.
+    # default: "false"
+    cache-workspace-crates: ""
+
+    # Determines whether the cache should be saved.
     # If `false`, the cache is only restored.
     # Useful for jobs where the matrix is additive e.g. additional Cargo features,
     # or when only runs from `master` should be saved to the cache.
@@ -69,10 +75,20 @@ sensible defaults.
     # To only cache runs from `master`:
     save-if: ${{ github.ref == 'refs/heads/master' }}
 
+    # Determines whether the cache should be restored.
+    # If `true` the cache key will be checked and the `cache-hit` output will be set
+    # but the cache itself won't be restored
+    # default: "false"
+    lookup-only: ""
+
     # Specifies what to use as the backend providing cache
-    # Can be set to either "github" or "buildjet"
+    # Can be set to "github", "buildjet", or "warpbuild"
     # default: "github"
     cache-provider: ""
+
+    # Determines whether to cache the ~/.cargo/bin directory.
+    # default: "true"
+    cache-bin: ""
 ```
 
 Further examples are available in the [.github/workflows](./.github/workflows/) directory.
@@ -169,4 +185,7 @@ to see those details as well as further details related to caching operations.
 ## Known issues
 
 - The cache cleaning process currently removes all the files from `~/.cargo/bin`
-  that were present before the action ran (for example `rustc`).
+  that were present before the action ran (for example `rustc`), by default.
+  This can be an issue on long-running self-hosted runners, where such state
+  is expected to be preserved across runs.  You can work around this by setting
+  `cache-bin: "false"`.

action.yml

@@ -32,19 +32,31 @@ inputs:
     description: "Determines which crates are cached. If `true` all crates will be cached, otherwise only dependent crates will be cached."
     required: false
     default: "false"
+  cache-workspace-crates:
+    description: "Similar to cache-all-crates. If `true` the workspace crates will be cached."
+    required: false
+    default: "false"
   save-if:
     description: "Determiners whether the cache should be saved. If `false`, the cache is only restored."
     required: false
     default: "true"
   cache-provider:
-    description: "Determines which provider to use for caching. Options are github or buildjet, defaults to github."
+    description: "Determines which provider to use for caching. Options are github, buildjet, or warpbuild. Defaults to github."
     required: false
     default: "github"
+  cache-bin:
+    description: "Determines whether to cache ${CARGO_HOME}/bin."
+    required: false
+    default: "true"
+  lookup-only:
+    description: "Check if a cache entry exists without downloading the cache"
+    required: false
+    default: "false"
 outputs:
   cache-hit:
     description: "A boolean value that indicates an exact match was found."
 runs:
-  using: "node16"
+  using: "node20"
   main: "dist/restore/index.js"
   post: "dist/save/index.js"
   post-if: "success() || env.CACHE_ON_FAILURE == 'true'"

package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "rust-cache",
-  "version": "2.7.0",
+  "version": "2.8.0",
   "description": "A GitHub Action that implements smart caching for rust/cargo projects with sensible defaults.",
   "keywords": [
     "actions",
@@ -23,17 +23,19 @@
   "homepage": "https://github.com/Swatinem/rust-cache#readme",
   "dependencies": {
     "@actions/buildjet-cache": "npm:github-actions.cache-buildjet@0.2.0",
-    "@actions/cache": "^3.2.2",
-    "@actions/core": "^1.10.0",
+    "@actions/warpbuild-cache": "npm:github-actions.warp-cache@1.4.5",
+    "@actions/cache": "^4.0.0",
+    "@actions/core": "^1.11.1",
     "@actions/exec": "^1.1.1",
-    "@actions/glob": "^0.4.0",
+    "@actions/glob": "^0.5.0",
     "@actions/io": "^1.1.3",
-    "smol-toml": "^1.1.1"
+    "smol-toml": "^1.3.0"
   },
   "devDependencies": {
-    "@vercel/ncc": "^0.36.1",
+    "@types/node": "^22.7.5",
+    "@vercel/ncc": "^0.38.2",
     "linefix": "^0.1.1",
-    "typescript": "5.1.6"
+    "typescript": "5.7.2"
   },
   "scripts": {
     "prepare": "ncc build --target es2020 -o dist/restore src/restore.ts && ncc build --target es2020 -o dist/save src/save.ts && linefix dist"

src/cleanup.ts

@@ -4,6 +4,7 @@ import fs from "fs";
 import path from "path";
 
 import { CARGO_HOME } from "./config";
+import { exists } from "./utils";
 import { Packages } from "./workspace";
 
 export async function cleanTargetDir(targetDir: string, packages: Packages, checkTimestamp = false) {
@@ -42,16 +43,16 @@ async function cleanProfileTarget(profileDir: string, packages: Packages, checkT
       // https://github.com/vertexclique/kaos/blob/9876f6c890339741cc5be4b7cb9df72baa5a6d79/src/cargo.rs#L25
       // https://github.com/eupn/macrotest/blob/c4151a5f9f545942f4971980b5d264ebcd0b1d11/src/cargo.rs#L27
       cleanTargetDir(path.join(profileDir, "target"), packages, checkTimestamp);
-    } catch { }
+    } catch {}
     try {
       // https://github.com/dtolnay/trybuild/blob/eec8ca6cb9b8f53d0caf1aa499d99df52cae8b40/src/cargo.rs#L50
       cleanTargetDir(path.join(profileDir, "trybuild"), packages, checkTimestamp);
-    } catch { }
+    } catch {}
 
     // Delete everything else.
     await rmExcept(profileDir, new Set(["target", "trybuild"]), checkTimestamp);
 
-    return
+    return;
   }
 
   let keepProfile = new Set(["build", ".fingerprint", "deps"]);
@@ -308,12 +309,3 @@ async function rmRF(dirName: string) {
   core.debug(`deleting "${dirName}"`);
   await io.rmRF(dirName);
 }
-
-async function exists(path: string) {
-  try {
-    await fs.promises.access(path);
-    return true;
-  } catch {
-    return false;
-  }
-}

src/config.ts

@@ -8,7 +8,7 @@ import path from "path";
 import * as toml from "smol-toml";
 
 import { getCargoBins } from "./cleanup";
-import { CacheProvider, getCmdOutput } from "./utils";
+import { CacheProvider, exists, getCmdOutput } from "./utils";
 import { Workspace } from "./workspace";
 
 const HOME = os.homedir();
@@ -25,6 +25,9 @@ export class CacheConfig {
   /** The secondary (restore) key that only contains the prefix and environment */
   public restoreKey = "";
 
+  /** Whether to cache CARGO_HOME/.bin */
+  public cacheBin: boolean = true;
+
   /** The workspace configurations */
   public workspaces: Array<Workspace> = [];
 
@@ -71,6 +74,11 @@ export class CacheConfig {
       }
     }
 
+    // Add runner OS and CPU architecture to the key to avoid cross-contamination of cache
+    const runnerOS = os.type();
+    const runnerArch = os.arch();
+    key += `-${runnerOS}-${runnerArch}`;
+
     self.keyPrefix = key;
 
     // Construct environment portion of the key:
@@ -116,6 +124,8 @@ export class CacheConfig {
     // This considers all the files found via globbing for various manifests
     // and lockfiles.
 
+    self.cacheBin = core.getInput("cache-bin").toLowerCase() == "true";
+
     // Constructs the workspace config and paths to restore:
     // The workspaces are given using a `$workspace -> $target` syntax.
 
@@ -142,7 +152,9 @@ export class CacheConfig {
         )),
       );
 
-      const cargo_manifests = sort_and_uniq(await globFiles(`${root}/**/Cargo.toml`));
+      const workspaceMembers = await workspace.getWorkspaceMembers();
+
+      const cargo_manifests = sort_and_uniq(workspaceMembers.map((member) => path.join(member.path, "Cargo.toml")));
 
       for (const cargo_manifest of cargo_manifests) {
         try {
@@ -183,23 +195,23 @@ export class CacheConfig {
           hasher.update(JSON.stringify(parsed));
 
           parsedKeyFiles.push(cargo_manifest);
-        } catch (e) { // Fallback to caching them as regular file
+        } catch (e) {
+          // Fallback to caching them as regular file
           core.warning(`Error parsing Cargo.toml manifest, fallback to caching entire file: ${e}`);
           keyFiles.push(cargo_manifest);
         }
       }
 
-      const cargo_locks = sort_and_uniq(await globFiles(`${root}/**/Cargo.lock`));
-
-      for (const cargo_lock of cargo_locks) {
+      const cargo_lock = path.join(workspace.root, "Cargo.lock");
+      if (await exists(cargo_lock)) {
         try {
           const content = await fs_promises.readFile(cargo_lock, { encoding: "utf8" });
           const parsed = toml.parse(content);
 
-          if (parsed.version !== 3 || !("package" in parsed)) {
+          if ((parsed.version !== 3 && parsed.version !== 4) || !("package" in parsed)) {
             // Fallback to caching them as regular file since this action
             // can only handle Cargo.lock format version 3
-            core.warning('Unsupported Cargo.lock format, fallback to caching entire file');
+            core.warning("Unsupported Cargo.lock format, fallback to caching entire file");
             keyFiles.push(cargo_lock);
             continue;
           }
@@ -211,7 +223,8 @@ export class CacheConfig {
           hasher.update(JSON.stringify(packages));
 
           parsedKeyFiles.push(cargo_lock);
-        } catch (e) { // Fallback to caching them as regular file
+        } catch (e) {
+          // Fallback to caching them as regular file
           core.warning(`Error parsing Cargo.lock manifest, fallback to caching entire file: ${e}`);
           keyFiles.push(cargo_lock);
         }
@@ -233,7 +246,15 @@ export class CacheConfig {
     key += `-${lockHash}`;
     self.cacheKey = key;
 
-    self.cachePaths = [CARGO_HOME];
+    self.cachePaths = [path.join(CARGO_HOME, "registry"), path.join(CARGO_HOME, "git")];
+    if (self.cacheBin) {
+      self.cachePaths = [
+        path.join(CARGO_HOME, "bin"),
+        path.join(CARGO_HOME, ".crates.toml"),
+        path.join(CARGO_HOME, ".crates2.json"),
+        ...self.cachePaths,
+      ];
+    }
     const cacheTargets = core.getInput("cache-targets").toLowerCase() || "true";
     if (cacheTargets === "true") {
       self.cachePaths.push(...workspaces.map((ws) => ws.target));

src/restore.ts

@@ -24,6 +24,8 @@ async function run() {
     if (cacheOnFailure !== "true") {
       cacheOnFailure = "false";
     }
+    var lookupOnly = core.getInput("lookup-only").toLowerCase() === "true";
+
     core.exportVariable("CACHE_ON_FAILURE", cacheOnFailure);
     core.exportVariable("CARGO_INCREMENTAL", 0);
 
@@ -31,15 +33,17 @@ async function run() {
     config.printInfo(cacheProvider);
     core.info("");
 
-    core.info(`... Restoring cache ...`);
+    core.info(`... ${lookupOnly ? "Checking" : "Restoring"} cache ...`);
     const key = config.cacheKey;
     // Pass a copy of cachePaths to avoid mutating the original array as reported by:
     // https://github.com/actions/toolkit/pull/1378
     // TODO: remove this once the underlying bug is fixed.
-    const restoreKey = await cacheProvider.cache.restoreCache(config.cachePaths.slice(), key, [config.restoreKey]);
+    const restoreKey = await cacheProvider.cache.restoreCache(config.cachePaths.slice(), key, [config.restoreKey], {
+      lookupOnly,
+    });
     if (restoreKey) {
       const match = restoreKey === key;
-      core.info(`Restored from cache key "${restoreKey}" full match: ${match}.`);
+      core.info(`${lookupOnly ? "Found" : "Restored from"} cache key "${restoreKey}" full match: ${match}.`);
       if (!match) {
         // pre-clean the target directory on cache mismatch
         for (const workspace of config.workspaces) {
@@ -64,6 +68,7 @@ async function run() {
 
     reportError(e);
   }
+  process.exit();
 }
 
 function setCacheHitOutput(cacheHit: boolean): void {

src/save.ts

@@ -32,11 +32,18 @@ async function run() {
     core.info("");
 
     // TODO: remove this once https://github.com/actions/toolkit/pull/553 lands
-    await macOsWorkaround();
+    if (process.env["RUNNER_OS"] == "macOS") {
+      await macOsWorkaround();
+    }
 
+    const workspaceCrates = core.getInput("cache-workspace-crates").toLowerCase() || "false";
     const allPackages = [];
     for (const workspace of config.workspaces) {
-      const packages = await workspace.getPackages();
+      const packages = await workspace.getPackagesOutsideWorkspaceRoot();
+      if (workspaceCrates === "true") {
+        const wsMembers = await workspace.getWorkspaceMembers();
+        packages.push(...wsMembers);
+      }
       allPackages.push(...packages);
       try {
         core.info(`... Cleaning ${workspace.target} ...`);
@@ -54,11 +61,13 @@ async function run() {
       core.debug(`${(e as any).stack}`);
     }
 
-    try {
-      core.info(`... Cleaning cargo/bin ...`);
-      await cleanBin(config.cargoBins);
-    } catch (e) {
-      core.debug(`${(e as any).stack}`);
+    if (config.cacheBin) {
+      try {
+        core.info(`... Cleaning cargo/bin ...`);
+        await cleanBin(config.cargoBins);
+      } catch (e) {
+        core.debug(`${(e as any).stack}`);
+      }
     }
 
     try {
@@ -76,6 +85,7 @@ async function run() {
   } catch (e) {
     reportError(e);
   }
+  process.exit();
 }
 
 run();

src/utils.ts

@@ -1,7 +1,9 @@
 import * as core from "@actions/core";
 import * as exec from "@actions/exec";
 import * as buildjetCache from "@actions/buildjet-cache";
+import * as warpbuildCache from "@actions/warpbuild-cache";
 import * as ghCache from "@actions/cache";
+import fs from "fs";
 
 export function reportError(e: any) {
   const { commandFailed } = e;
@@ -43,17 +45,32 @@ export async function getCmdOutput(
   return stdout;
 }
 
+export interface GhCache {
+  isFeatureAvailable: typeof ghCache.isFeatureAvailable;
+  restoreCache: typeof ghCache.restoreCache;
+  saveCache: (paths: string[], key: string) => Promise<string | number>;
+}
+
 export interface CacheProvider {
   name: string;
-  cache: typeof ghCache;
+  cache: GhCache;
 }
 
 export function getCacheProvider(): CacheProvider {
   const cacheProvider = core.getInput("cache-provider");
-  const cache = cacheProvider === "github" ? ghCache : cacheProvider === "buildjet" ? buildjetCache : undefined;
-
-  if (!cache) {
-    throw new Error(`The \`cache-provider\` \`{cacheProvider}\` is not valid.`);
+  let cache: GhCache;
+  switch (cacheProvider) {
+    case "github":
+      cache = ghCache;
+      break;
+    case "buildjet":
+      cache = buildjetCache;
+      break;
+    case "warpbuild":
+      cache = warpbuildCache;
+      break;
+    default:
+      throw new Error(`The \`cache-provider\` \`${cacheProvider}\` is not valid.`);
   }
 
   return {
@@ -61,3 +78,12 @@ export function getCacheProvider(): CacheProvider {
     cache: cache,
   };
 }
+
+export async function exists(path: string) {
+  try {
+    await fs.promises.access(path);
+    return true;
+  } catch {
+    return false;
+  }
+}

src/workspace.ts

@@ -8,26 +8,34 @@ const SAVE_TARGETS = new Set(["lib", "proc-macro"]);
 export class Workspace {
   constructor(public root: string, public target: string) {}
 
-  public async getPackages(): Promise<Packages> {
+  async getPackages(filter: (p: Meta["packages"][0]) => boolean, ...extraArgs: string[]): Promise<Packages> {
     let packages: Packages = [];
     try {
       core.debug(`collecting metadata for "${this.root}"`);
       const meta: Meta = JSON.parse(
-        await getCmdOutput("cargo", ["metadata", "--all-features", "--format-version", "1"], {
+        await getCmdOutput("cargo", ["metadata", "--all-features", "--format-version", "1", ...extraArgs], {
           cwd: this.root,
+          env: { "CARGO_ENCODED_RUSTFLAGS": "" },
         }),
       );
       core.debug(`workspace "${this.root}" has ${meta.packages.length} packages`);
-      for (const pkg of meta.packages) {
-        if (pkg.manifest_path.startsWith(this.root)) {
-          continue;
-        }
+      for (const pkg of meta.packages.filter(filter)) {
         const targets = pkg.targets.filter((t) => t.kind.some((kind) => SAVE_TARGETS.has(kind))).map((t) => t.name);
         packages.push({ name: pkg.name, version: pkg.version, targets, path: path.dirname(pkg.manifest_path) });
       }
-    } catch {}
+    } catch (err) {
+      console.error(err);
+    }
     return packages;
   }
+
+  public async getPackagesOutsideWorkspaceRoot(): Promise<Packages> {
+    return await this.getPackages((pkg) => !pkg.manifest_path.startsWith(this.root));
+  }
+
+  public async getWorkspaceMembers(): Promise<Packages> {
+    return await this.getPackages((_) => true, "--no-deps");
+  }
 }
 
 export interface PackageDefinition {

tests/Cargo.toml

@@ -6,11 +6,12 @@ authors = ["Arpad Borsos <arpad.borsos@googlemail.com>"]
 edition = "2021"
 
 [dependencies]
-reqwest = "0.11.18"
+reqwest = "0.12.1"
+jsonpath_lib_polars_vendor = "0.0.1"
 watto = { git = "https://github.com/getsentry/watto", rev = "d71c8218506bddba102a124a460d64da25e303dc", features = ["strings"] }
 
 [dev-dependencies]
 trybuild = "1"
 
 [target.'cfg(not(target_env = "msvc"))'.dependencies]
-tikv-jemallocator = "0.5.4"
+tikv-jemallocator = "0.6.0"

tests/wasm-workspace/Cargo.toml

@@ -1,4 +1,5 @@
 [workspace]
+resolver = "2"
 members = [
     "crates/one",
     "crates/two",

tests/wasm-workspace/crates/one/Cargo.toml

@@ -5,7 +5,7 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-reqwest = "0.11.0"
+reqwest = "0.12"
 async-std = "1"
 tracing = "0.1"
 tracing-futures = "0.2"

tests/wasm-workspace/crates/two/Cargo.toml

@@ -5,4 +5,4 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-clap = "3"
+clap = "4"