diff --git a/doc/manual/rl-next/gc-age-gate.md b/doc/manual/rl-next/gc-age-gate.md new file mode 100644 index 00000000000..59de0abfdc7 --- /dev/null +++ b/doc/manual/rl-next/gc-age-gate.md @@ -0,0 +1,22 @@ +--- +synopsis: "Store GCs can now be restricted with a minimum age" +prs: [14725] +issues: [7572] +--- + +Nix store GCs invoked using either `nix-collect-garbage` or `nix store gc` may +now be restricted to only deleting paths older than a certain minimum age, i.e. +whose most recent usage is more than *n* days in the past. "Usage" is +intentionally defined ambiguously, however in general all operations which +produce/require the presence of a given store path count as "usage". + + +Example usage: + +```bash +# Delete store paths older than 7 days +nix store gc --older-than 7d + +# Alternatively: +nix-collect-garbage --path-older-than 7d +``` diff --git a/doc/manual/source/command-ref/nix-collect-garbage.md b/doc/manual/source/command-ref/nix-collect-garbage.md index 763179b8ee1..0a0dbefaf00 100644 --- a/doc/manual/source/command-ref/nix-collect-garbage.md +++ b/doc/manual/source/command-ref/nix-collect-garbage.md @@ -4,7 +4,7 @@ # Synopsis -`nix-collect-garbage` [`--delete-old`] [`-d`] [`--delete-older-than` *period*] [`--max-freed` *bytes*] [`--dry-run`] +`nix-collect-garbage` [`--delete-old`] [`-d`] [`--delete-older-than` *period*] [`--delete-paths-older-than` *period*] [`--max-freed` *bytes*] [`--dry-run`] # Description @@ -62,7 +62,17 @@ These options are for deleting old [profiles] prior to deleting unreachable [sto This is the equivalent of invoking [`nix-env --delete-generations `](@docroot@/command-ref/nix-env/delete-generations.md#generations-time) on each found profile. See the documentation of that command for additional information about the *period* argument. - - [`--max-freed`](#opt-max-freed) *bytes* +- [`--delete-paths-older-than`](#opt-delete-paths-older-than) *period* + + Only delete store paths older than the specified amount, i.e. restrict the GC + to only deleting store paths whose most recent usage is more than *N* days in + the past. *period* is a value such as `30d`, which would mean 30 days. + + "Usage" is intentionally defined ambiguously, however in general all + operations which produce/require the presence of a given store path count as + "usage". + +- [`--max-freed`](#opt-max-freed) *bytes* diff --git a/doc/manual/source/protocols/json/schema/store-object-info-v2.yaml b/doc/manual/source/protocols/json/schema/store-object-info-v2.yaml index fb63795be13..dd664a3dbc1 100644 --- a/doc/manual/source/protocols/json/schema/store-object-info-v2.yaml +++ b/doc/manual/source/protocols/json/schema/store-object-info-v2.yaml @@ -118,6 +118,7 @@ $defs: # impure - deriver - registrationTime + - lastUsageTime - ultimate - signatures properties: @@ -147,6 +148,15 @@ $defs: > This is an "impure" field that may not be included in certain contexts. + lastUsageTime: + type: ["integer", "null"] + title: Last Usage Time + description: | + If known, when this derivation was last used (Unix timestamp). + Otherwise `null`. + + > This is an "impure" field that may not be included in certain contexts. + ultimate: type: boolean title: Ultimate @@ -195,6 +205,7 @@ $defs: # impure - deriver - registrationTime + - lastUsageTime - ultimate - signatures # nar @@ -211,6 +222,7 @@ $defs: ca: { $ref: "#/$defs/base/properties/ca" } deriver: { $ref: "#/$defs/impure/properties/deriver" } registrationTime: { $ref: "#/$defs/impure/properties/registrationTime" } + lastUsageTime: { $ref: "#/$defs/impure/properties/lastUsageTime" } ultimate: { $ref: "#/$defs/impure/properties/ultimate" } signatures: { $ref: "#/$defs/impure/properties/signatures" } closureSize: { $ref: "#/$defs/impure/properties/closureSize" } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index ea10d4c55b5..e8cd830a1d1 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1936,6 +1936,8 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value ** args, V auto path2 = state.store->toStorePath(path.abs()).first; if (!settings.readOnlyMode) state.store->ensurePath(path2); + else + state.store->bumpLastUsageTime(path2); context.insert(NixStringContextElem::Opaque{.path = path2}); v.mkString(path.abs(), context, state.mem); } diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 70c13e2985b..867d72c5400 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -273,6 +273,8 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value ** arg auto namePath = state.store->parseStorePath(name); if (!settings.readOnlyMode) state.store->ensurePath(namePath); + else + state.store->bumpLastUsageTime(namePath); state.forceAttrs(*i.value, i.pos, "while evaluating the value of a string context"); if (auto attr = i.value->attrs()->get(sPath)) { diff --git a/src/libstore-tests/data/dummy-store/one-flat-file.json b/src/libstore-tests/data/dummy-store/one-flat-file.json index d3993cb3284..a552e9b43dd 100644 --- a/src/libstore-tests/data/dummy-store/one-flat-file.json +++ b/src/libstore-tests/data/dummy-store/one-flat-file.json @@ -28,6 +28,7 @@ "narSize": 120, "references": [], "registrationTime": null, + "lastUsageTime": null, "signatures": [], "ultimate": false, "version": 2 diff --git a/src/libstore-tests/data/nar-info/json-1/impure.json b/src/libstore-tests/data/nar-info/json-1/impure.json index d4dfc472b18..24ff059a678 100644 --- a/src/libstore-tests/data/nar-info/json-1/impure.json +++ b/src/libstore-tests/data/nar-info/json-1/impure.json @@ -11,6 +11,7 @@ "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" ], "registrationTime": 23423, + "lastUsageTime": 34534, "signatures": [ "asdf", "qwer" diff --git a/src/libstore-tests/data/nar-info/json-2/impure.json b/src/libstore-tests/data/nar-info/json-2/impure.json index d262e26e5d1..be231a9b355 100644 --- a/src/libstore-tests/data/nar-info/json-2/impure.json +++ b/src/libstore-tests/data/nar-info/json-2/impure.json @@ -26,6 +26,7 @@ "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" ], "registrationTime": 23423, + "lastUsageTime": 34534, "signatures": [ "asdf", "qwer" diff --git a/src/libstore-tests/data/path-info/json-1/empty_impure.json b/src/libstore-tests/data/path-info/json-1/empty_impure.json index fa36bf3716e..df8af94fca7 100644 --- a/src/libstore-tests/data/path-info/json-1/empty_impure.json +++ b/src/libstore-tests/data/path-info/json-1/empty_impure.json @@ -5,6 +5,7 @@ "narSize": 0, "references": [], "registrationTime": null, + "lastUsageTime": null, "signatures": [], "ultimate": false, "version": 1 diff --git a/src/libstore-tests/data/path-info/json-1/impure.json b/src/libstore-tests/data/path-info/json-1/impure.json index 1185e855823..43d7766ad14 100644 --- a/src/libstore-tests/data/path-info/json-1/impure.json +++ b/src/libstore-tests/data/path-info/json-1/impure.json @@ -8,6 +8,7 @@ "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" ], "registrationTime": 23423, + "lastUsageTime": 34534, "signatures": [ "asdf", "qwer" diff --git a/src/libstore-tests/data/path-info/json-2/empty_impure.json b/src/libstore-tests/data/path-info/json-2/empty_impure.json index eae2a63d34f..6e5058ef72e 100644 --- a/src/libstore-tests/data/path-info/json-2/empty_impure.json +++ b/src/libstore-tests/data/path-info/json-2/empty_impure.json @@ -9,6 +9,7 @@ "narSize": 0, "references": [], "registrationTime": null, + "lastUsageTime": null, "signatures": [], "ultimate": false, "version": 2 diff --git a/src/libstore-tests/data/path-info/json-2/impure.json b/src/libstore-tests/data/path-info/json-2/impure.json index e3f17bb1df8..9b1643220a4 100644 --- a/src/libstore-tests/data/path-info/json-2/impure.json +++ b/src/libstore-tests/data/path-info/json-2/impure.json @@ -19,6 +19,7 @@ "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" ], "registrationTime": 23423, + "lastUsageTime": 34534, "signatures": [ "asdf", "qwer" diff --git a/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.3.json b/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.3.json index 32841527109..e67590f3772 100644 --- a/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.3.json +++ b/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.3.json @@ -10,6 +10,7 @@ "narSize": 34878, "references": [], "registrationTime": null, + "lastUsageTime": null, "signatures": [], "ultimate": false, "version": 2 @@ -27,6 +28,7 @@ "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo.drv" ], "registrationTime": null, + "lastUsageTime": null, "signatures": [], "ultimate": false, "version": 2 diff --git a/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.json b/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.json index 2fbfa8f0068..5e3da1b5991 100644 --- a/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.json +++ b/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.json @@ -12,6 +12,7 @@ "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo.drv" ], "registrationTime": null, + "lastUsageTime": null, "signatures": [], "ultimate": false, "version": 2 @@ -37,6 +38,7 @@ "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" ], "registrationTime": null, + "lastUsageTime": null, "signatures": [ "fake-sig-1", "fake-sig-2" diff --git a/src/libstore-tests/data/worker-protocol/unkeyed-valid-path-info-1.15.json b/src/libstore-tests/data/worker-protocol/unkeyed-valid-path-info-1.15.json index 2408405ada5..8fbf797670b 100644 --- a/src/libstore-tests/data/worker-protocol/unkeyed-valid-path-info-1.15.json +++ b/src/libstore-tests/data/worker-protocol/unkeyed-valid-path-info-1.15.json @@ -10,6 +10,7 @@ "narSize": 34878, "references": [], "registrationTime": 23423, + "lastUsageTime": null, "signatures": [], "ultimate": false, "version": 2 @@ -27,6 +28,7 @@ "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo.drv" ], "registrationTime": 23423, + "lastUsageTime": null, "signatures": [], "ultimate": false, "version": 2 diff --git a/src/libstore-tests/data/worker-protocol/valid-path-info-1.15.json b/src/libstore-tests/data/worker-protocol/valid-path-info-1.15.json index b75b1ff9f93..c9d59d209cc 100644 --- a/src/libstore-tests/data/worker-protocol/valid-path-info-1.15.json +++ b/src/libstore-tests/data/worker-protocol/valid-path-info-1.15.json @@ -11,6 +11,7 @@ "path": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", "references": [], "registrationTime": 23423, + "lastUsageTime": null, "signatures": [], "ultimate": false, "version": 2 @@ -30,6 +31,7 @@ "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo" ], "registrationTime": 23423, + "lastUsageTime": null, "signatures": [], "ultimate": false, "version": 2 diff --git a/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.json b/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.json index a61c483965f..f196380747e 100644 --- a/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.json +++ b/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.json @@ -11,6 +11,7 @@ "path": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", "references": [], "registrationTime": 23423, + "lastUsageTime": null, "signatures": [], "ultimate": true, "version": 2 @@ -30,6 +31,7 @@ "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo" ], "registrationTime": 23423, + "lastUsageTime": null, "signatures": [ "fake-sig-1", "fake-sig-2" @@ -59,6 +61,7 @@ "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" ], "registrationTime": 23423, + "lastUsageTime": null, "signatures": [], "ultimate": false, "version": 2 diff --git a/src/libstore-tests/nar-info.cc b/src/libstore-tests/nar-info.cc index 493ca2a8c37..2c4acf08d4d 100644 --- a/src/libstore-tests/nar-info.cc +++ b/src/libstore-tests/nar-info.cc @@ -58,6 +58,7 @@ static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", }; info.registrationTime = 23423; + info.lastUsageTime = 34534; info.ultimate = true; info.sigs = {"asdf", "qwer"}; diff --git a/src/libstore-tests/path-info.cc b/src/libstore-tests/path-info.cc index 07942649c03..1096cd0013a 100644 --- a/src/libstore-tests/path-info.cc +++ b/src/libstore-tests/path-info.cc @@ -64,6 +64,7 @@ static ValidPathInfo makeFullKeyed(const Store & store, bool includeImpureInfo) "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", }; info.registrationTime = 23423; + info.lastUsageTime = 34534; info.ultimate = true; info.sigs = {"asdf", "qwer"}; } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 848669ae84f..3b3958bc32d 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -295,6 +295,7 @@ void BinaryCacheStore::addToStore( { if (!repair && isValidPath(info.path)) { // FIXME: copyNAR -> null sink + bumpLastUsageTime(info.path); narSource.drain(); return; } diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index 8221e12c697..1c76f762d09 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -117,8 +117,10 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution(bool storeDerivation) } for (auto & i : drv->inputSrcs) { - if (worker.store.isValidPath(i)) + if (worker.store.isValidPath(i)) { + worker.store.bumpLastUsageTime(i); continue; + } if (!settings.useSubstitutes) throw Error( "dependency '%s' of '%s' does not exist, and substitution is disabled", diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index b36685a242c..a2f2358d934 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -91,6 +91,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) /* If they are all valid, then we're done. */ if (checkResult && checkResult->second == PathStatus::Valid && buildMode == bmNormal) { + worker.store.bumpLastUsageTime(checkResult->first.outPath); co_return doneSuccess(BuildResult::Success::AlreadyValid, checkResult->first); } diff --git a/src/libstore/build/derivation-trampoline-goal.cc b/src/libstore/build/derivation-trampoline-goal.cc index 963156aa584..386610c6a2b 100644 --- a/src/libstore/build/derivation-trampoline-goal.cc +++ b/src/libstore/build/derivation-trampoline-goal.cc @@ -116,8 +116,10 @@ Goal::Co DerivationTrampolineGoal::init() */ auto drv = [&] { for (auto * drvStore : {&worker.evalStore, &worker.store}) - if (drvStore->isValidPath(drvPath)) + if (drvStore->isValidPath(drvPath)) { + drvStore->bumpLastUsageTime(drvPath); return drvStore->readDerivation(drvPath); + } assert(false); }(); diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 4bbd4c8f059..c9d2e6fac13 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -92,8 +92,10 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat void Store::ensurePath(const StorePath & path) { /* If the path is already valid, we're done. */ - if (isValidPath(path)) + if (isValidPath(path)) { + bumpLastUsageTime(path); return; + } Worker worker(*this, *this); GoalPtr goal = worker.makePathSubstitutionGoal(path); diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index ac18de304b7..b49b9c46301 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -53,6 +53,7 @@ Goal::Co PathSubstitutionGoal::init() /* If the path already exists we're done. */ if (!repair && worker.store.isValidPath(storePath)) { + worker.store.bumpLastUsageTime(storePath); co_return doneSuccess(BuildResult::Success::AlreadyValid); } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 8fbec768912..f1991c2322d 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -424,6 +424,14 @@ static void performOp( break; } + case WorkerProto::Op::BumpLastUsageTime: { + auto path = WorkerProto::Serialise::read(*store, rconn); + logger->startWork(); + store->bumpLastUsageTime(path); + logger->stopWork(); + break; + } + case WorkerProto::Op::AddToStore: { if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 25) { auto name = readString(conn.from); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 842ef9056ee..103d664925d 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -139,8 +139,10 @@ StorePath Store::writeDerivation(const Derivation & drv, RepairFlag repair) { auto [suffix, contents, references, path] = infoForDerivation(*this, drv); - if (isValidPath(path) && !repair) + if (isValidPath(path) && !repair) { + bumpLastUsageTime(path); return path; + } StringSource s{contents}; auto path2 = addToStoreFromDump( diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 4846d445fe1..ff703ea54f6 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -742,6 +742,13 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) if (options.action == GCOptions::gcDeleteSpecific && !options.pathsToDelete.count(*path)) return; + /* If this path is too young, bail out */ + if (options.olderThan.has_value() && isValidPath(*path) + && options.olderThan.value() < queryPathInfo(*path)->lastUsageTime) { + debug("not deleting '%s' because it's too young", printStorePath(*path)); + return markAlive(); + } + { auto hashPart = path->hashPart(); auto shared(_shared.lock()); diff --git a/src/libstore/include/nix/store/gc-store.hh b/src/libstore/include/nix/store/gc-store.hh index 5a4a6db1439..eb393083293 100644 --- a/src/libstore/include/nix/store/gc-store.hh +++ b/src/libstore/include/nix/store/gc-store.hh @@ -55,6 +55,11 @@ struct GCOptions * Stop after at least `maxFreed` bytes have been freed. */ uint64_t maxFreed{std::numeric_limits::max()}; + + /** + * Only delete paths older than a certain point in time. + */ + std::optional olderThan; }; struct GCResults diff --git a/src/libstore/include/nix/store/local-overlay-store.hh b/src/libstore/include/nix/store/local-overlay-store.hh index 1d69d341708..839fa4abcd0 100644 --- a/src/libstore/include/nix/store/local-overlay-store.hh +++ b/src/libstore/include/nix/store/local-overlay-store.hh @@ -163,6 +163,11 @@ private: */ std::optional queryPathFromHashPart(const std::string & hashPart) override; + /** + * Bump the last usage time in both the lower and upper DBs. + */ + void bumpLastUsageTime(const StorePath & path) override; + /** * First copy up any lower store realisation with the same key, so we * merge rather than mask it. diff --git a/src/libstore/include/nix/store/local-store.hh b/src/libstore/include/nix/store/local-store.hh index 212229e42f9..f314ccacdfc 100644 --- a/src/libstore/include/nix/store/local-store.hh +++ b/src/libstore/include/nix/store/local-store.hh @@ -175,10 +175,12 @@ private: }; /** - * Mutable state. It's behind a `ref` to reduce false sharing - * between immutable and mutable fields. + * Mutable state. It's behind a `ref` to reduce false sharing between + * immutable and mutable fields. Additionally, it uses a recursive mutex + * since operations like e.g. `bumpLastUsageTime` can be invoked from within + * other DB-modifying operations. */ - ref> _state; + ref> _state; public: @@ -230,6 +232,8 @@ public: std::optional queryPathFromHashPart(const std::string & hashPart) override; + void bumpLastUsageTime(const StorePath & path) override; + StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; bool pathInfoIsUntrusted(const ValidPathInfo &) override; diff --git a/src/libstore/include/nix/store/path-info.hh b/src/libstore/include/nix/store/path-info.hh index f81d0134795..5d2eb3ae40a 100644 --- a/src/libstore/include/nix/store/path-info.hh +++ b/src/libstore/include/nix/store/path-info.hh @@ -75,6 +75,11 @@ struct UnkeyedValidPathInfo */ time_t registrationTime = 0; + /** + * When this store object was last used, if known. + */ + time_t lastUsageTime = 0; + /** * 0 = unknown */ diff --git a/src/libstore/include/nix/store/remote-store.hh b/src/libstore/include/nix/store/remote-store.hh index b152e054b9d..35a264132eb 100644 --- a/src/libstore/include/nix/store/remote-store.hh +++ b/src/libstore/include/nix/store/remote-store.hh @@ -65,6 +65,8 @@ struct RemoteStore : public virtual Store, public virtual GcStore, public virtua queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore = nullptr) override; std::optional queryPathFromHashPart(const std::string & hashPart) override; + void bumpLastUsageTime(const StorePath & path) override; + StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) override; diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index db107fc0ce7..5f811f4c4ff 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -494,6 +494,11 @@ public: */ virtual std::optional queryPathFromHashPart(const std::string & hashPart) = 0; + /** + * Bumps the last usage time of the given store path to the current time. + */ + virtual void bumpLastUsageTime(const StorePath & path) {} + /** * Query which of the given paths have substitutes. */ diff --git a/src/libstore/include/nix/store/worker-protocol-connection.hh b/src/libstore/include/nix/store/worker-protocol-connection.hh index 31436395fe7..c436bbfef3e 100644 --- a/src/libstore/include/nix/store/worker-protocol-connection.hh +++ b/src/libstore/include/nix/store/worker-protocol-connection.hh @@ -41,6 +41,7 @@ struct WorkerProto::BasicConnection return WorkerProto::ReadConn{ .from = from, .version = protoVersion, + .features = &features, }; } @@ -57,6 +58,7 @@ struct WorkerProto::BasicConnection return WorkerProto::WriteConn{ .to = to, .version = protoVersion, + .features = &features, }; } }; diff --git a/src/libstore/include/nix/store/worker-protocol.hh b/src/libstore/include/nix/store/worker-protocol.hh index 6ae5fdcbc29..9cce0e13b03 100644 --- a/src/libstore/include/nix/store/worker-protocol.hh +++ b/src/libstore/include/nix/store/worker-protocol.hh @@ -58,6 +58,13 @@ struct WorkerProto */ using Version = unsigned int; + using Feature = std::string; + using FeatureSet = std::set>; + + static const Feature pathLastUsageTimeFeature; + + static const FeatureSet allFeatures; + /** * A unidirectional read connection, to be used by the read half of the * canonical serializers below. @@ -66,6 +73,7 @@ struct WorkerProto { Source & from; Version version; + const FeatureSet * features{nullptr}; }; /** @@ -76,6 +84,7 @@ struct WorkerProto { Sink & to; Version version; + const FeatureSet * features{nullptr}; }; /** @@ -132,11 +141,6 @@ struct WorkerProto { WorkerProto::Serialise::write(store, conn, t); } - - using Feature = std::string; - using FeatureSet = std::set>; - - static const FeatureSet allFeatures; }; enum struct WorkerProto::Op : uint64_t { @@ -184,6 +188,7 @@ enum struct WorkerProto::Op : uint64_t { AddBuildLog = 45, BuildPathsWithResults = 46, AddPermRoot = 47, + BumpLastUsageTime = 48, }; struct WorkerProto::ClientHandshakeInfo diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 1a38cac3b7f..75dbc4ebbc5 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -109,6 +109,7 @@ std::shared_ptr LocalFSStore::getFSAccessor(const StorePath & pa if (!pathExists(absPath)) return nullptr; } + bumpLastUsageTime(path); return std::make_shared(std::move(absPath)); } diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index c8aa1d1a2b6..4bd3f2c408c 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -179,6 +179,12 @@ std::optional LocalOverlayStore::queryPathFromHashPart(const std::str return lowerStore->queryPathFromHashPart(hashPart); } +void LocalOverlayStore::bumpLastUsageTime(const StorePath & path) +{ + LocalStore::bumpLastUsageTime(path); + lowerStore->bumpLastUsageTime(path); +} + void LocalOverlayStore::registerValidPaths(const ValidPathInfos & infos) { // First, get any from lower store so we merge diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 51392f014f0..a2bf8a7bb30 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -100,6 +100,7 @@ struct LocalStore::State::Stmts SQLiteStmt QueryPathInfo; SQLiteStmt QueryReferences; SQLiteStmt QueryReferrers; + SQLiteStmt BumpPathUsageTime; SQLiteStmt InvalidatePath; SQLiteStmt AddDerivationOutput; SQLiteStmt RegisterRealisedOutput; @@ -118,7 +119,7 @@ LocalStore::LocalStore(ref config) : Store{*config} , LocalFSStore{*config} , config{config} - , _state(make_ref>()) + , _state(make_ref>()) , dbDir(config->stateDir + "/db") , linksDir(config->realStoreDir + "/.links") , reservedPath(dbDir + "/reserved") @@ -336,16 +337,18 @@ LocalStore::LocalStore(ref config) state->db, "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs, ca) values (?, ?, ?, ?, ?, ?, ?, ?);"); state->stmts->UpdatePathInfo.create( - state->db, "update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ?, ca = ? where path = ?;"); + state->db, + "update ValidPaths set narSize = ?, hash = ?, lastUsageTime = ?, ultimate = ?, sigs = ?, ca = ? where path = ?;"); state->stmts->AddReference.create(state->db, "insert or replace into Refs (referrer, reference) values (?, ?);"); state->stmts->QueryPathInfo.create( state->db, - "select id, hash, registrationTime, deriver, narSize, ultimate, sigs, ca from ValidPaths where path = ?;"); + "select id, hash, registrationTime, lastUsageTime, deriver, narSize, ultimate, sigs, ca from ValidPaths where path = ?;"); state->stmts->QueryReferences.create( state->db, "select path from Refs join ValidPaths on reference = id where referrer = ?;"); state->stmts->QueryReferrers.create( state->db, "select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);"); + state->stmts->BumpPathUsageTime.create(state->db, "update ValidPaths set lastUsageTime = ? where path = ?;"); state->stmts->InvalidatePath.create(state->db, "delete from ValidPaths where path = ?;"); state->stmts->AddDerivationOutput.create( state->db, "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);"); @@ -595,6 +598,8 @@ void LocalStore::upgradeDBSchema(State & state) "20220326-ca-derivations", #include "ca-specific-schema.sql.gen.hh" ); + + doUpgrade("20251205-last-usage-time", "alter table ValidPaths add column lastUsageTime integer default 0 not null"); } /* To improve purity, users may want to make the Nix store a read-only @@ -761,22 +766,23 @@ std::shared_ptr LocalStore::queryPathInfoInternal(State & s info->id = id; - info->registrationTime = useQueryPathInfo.getInt(2); + info->registrationTime = (time_t) useQueryPathInfo.getInt(2); + info->lastUsageTime = std::max((time_t) useQueryPathInfo.getInt(3), info->registrationTime); - auto s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 3); + auto s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 4); if (s) info->deriver = parseStorePath(s); /* Note that narSize = NULL yields 0. */ - info->narSize = useQueryPathInfo.getInt(4); + info->narSize = useQueryPathInfo.getInt(5); - info->ultimate = useQueryPathInfo.getInt(5) == 1; + info->ultimate = useQueryPathInfo.getInt(6) == 1; - s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 6); + s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 7); if (s) info->sigs = tokenizeString(s, " "); - s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 7); + s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 8); if (s) info->ca = ContentAddress::parseOpt(s); @@ -793,7 +799,7 @@ std::shared_ptr LocalStore::queryPathInfoInternal(State & s void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { state.stmts->UpdatePathInfo - .use()(info.narSize, info.narSize != 0)(info.narHash.to_string(HashFormat::Base16, true))( + .use()(info.narSize, info.narSize != 0)(info.narHash.to_string(HashFormat::Base16, true))(time(0))( info.ultimate ? 1 : 0, info.ultimate)(concatStringsSep(" ", info.sigs), !info.sigs.empty())( renderContentAddress(info.ca), (bool) info.ca)(printStorePath(info.path)) .exec(); @@ -904,6 +910,13 @@ std::optional LocalStore::queryPathFromHashPart(const std::string & h }); } +void LocalStore::bumpLastUsageTime(const StorePath & path) +{ + if (config->readOnly) + return; + retrySQLite([&]() { _state->lock()->stmts->BumpPathUsageTime.use()(time(0))(printStorePath(path)).exec(); }); +} + StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) { if (!settings.useSubstitutes) @@ -1150,9 +1163,13 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairF } registerValidPath(info); + } else { + bumpLastUsageTime(info.path); } outputLock.setDeletion(true); + } else { + bumpLastUsageTime(info.path); } } @@ -1312,9 +1329,13 @@ StorePath LocalStore::addToStoreFromDump( auto info = ValidPathInfo::makeFromCA(*this, name, std::move(desc), narHash.hash); info.narSize = narHash.numBytesDigested; registerValidPath(info); + } else { + bumpLastUsageTime(dstPath); } outputLock.setDeletion(true); + } else { + bumpLastUsageTime(dstPath); } return dstPath; diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index a5133a97db8..f8f6215e080 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -28,6 +28,7 @@ GENERATE_CMP_EXT( me->narHash, me->references, me->registrationTime, + // me->lastUsageTime, me->narSize, // me->id, me->ultimate, @@ -200,6 +201,7 @@ UnkeyedValidPathInfo::toJSON(const StoreDirConfig * store, bool includeImpureInf jsonObject["deriver"] = deriver; } jsonObject["registrationTime"] = registrationTime ? std::optional{registrationTime} : std::nullopt; + jsonObject["lastUsageTime"] = lastUsageTime ? std::optional{lastUsageTime} : std::nullopt; jsonObject["ultimate"] = ultimate; @@ -269,6 +271,10 @@ UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON(const StoreDirConfig * store if (auto * rawRegistrationTime = getNullable(*rawRegistrationTime0)) res.registrationTime = getInteger(*rawRegistrationTime); + if (auto * rawLastUsageTime0 = optionalValueAt(json, "lastUsageTime")) + if (auto * rawLastUsageTime = getNullable(*rawLastUsageTime0)) + res.lastUsageTime = getInteger(*rawLastUsageTime); + if (auto * rawUltimate = optionalValueAt(json, "ultimate")) res.ultimate = getBoolean(*rawUltimate); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b07efc0241a..243d2cc40c8 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -305,6 +305,16 @@ std::optional RemoteStore::queryPathFromHashPart(const std::string & return WorkerProto::Serialise>::read(*this, *conn); } +void RemoteStore::bumpLastUsageTime(const StorePath & path) +{ + auto conn(getConnection()); + if (conn->features.contains(WorkerProto::pathLastUsageTimeFeature)) { + conn->to << WorkerProto::Op::BumpLastUsageTime; + WorkerProto::write(*this, *conn, path); + conn.processStderr(); + } +} + ref RemoteStore::addCAToStore( Source & dump, std::string_view name, diff --git a/src/libstore/restricted-store.cc b/src/libstore/restricted-store.cc index ef8aaa3801d..545528e1167 100644 --- a/src/libstore/restricted-store.cc +++ b/src/libstore/restricted-store.cc @@ -73,6 +73,8 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor throw Error("queryPathFromHashPart"); } + void bumpLastUsageTime(const StorePath & path) override; + StorePath addToStore( std::string_view name, const SourcePath & srcPath, @@ -181,6 +183,7 @@ void RestrictedStore::queryPathInfoUncached( auto info = std::make_shared(*next->queryPathInfo(path)); info->deriver.reset(); info->registrationTime = 0; + info->lastUsageTime = 0; info->ultimate = false; info->sigs.clear(); callback(info); @@ -201,6 +204,11 @@ RestrictedStore::queryPartialDerivationOutputMap(const StorePath & path, Store * return next->queryPartialDerivationOutputMap(path, evalStore); } +void RestrictedStore::bumpLastUsageTime(const StorePath & path) +{ + next->bumpLastUsageTime(path); +} + void RestrictedStore::addToStore( const ValidPathInfo & info, Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs) { @@ -234,6 +242,7 @@ void RestrictedStore::ensurePath(const StorePath & path) if (!goal.isAllowed(path)) throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path)); /* Nothing to be done; 'path' must already be valid. */ + bumpLastUsageTime(path); } void RestrictedStore::registerDrvOutput(const Realisation & info) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 921507add6c..f84fb3be8e9 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -182,6 +182,8 @@ void Store::addMultipleToStore(PathsSource && pathsToCopy, Activity & act, Repai showProgress(); return; } + } else { + bumpLastUsageTime(info.path); } nrDone++; @@ -295,6 +297,8 @@ ValidPathInfo Store::addToStoreSlow( if (!isValidPath(info.path)) { auto source = sinkToSource([&](Sink & scratchpadSink) { srcPath.dumpPath(scratchpadSink); }); addToStore(info, *source); + } else { + bumpLastUsageTime(info.path); } return info; @@ -853,8 +857,10 @@ void copyStorePath( { /* Bail out early (before starting a download from srcStore) if dstStore already has this path. */ - if (!repair && dstStore.isValidPath(storePath)) + if (!repair && dstStore.isValidPath(storePath)) { + dstStore.bumpLastUsageTime(storePath); return; + } const auto & srcCfg = srcStore.config; const auto & dstCfg = dstStore.config; diff --git a/src/libstore/worker-protocol-connection.cc b/src/libstore/worker-protocol-connection.cc index 8a37662904d..9dfadba5218 100644 --- a/src/libstore/worker-protocol-connection.cc +++ b/src/libstore/worker-protocol-connection.cc @@ -5,7 +5,9 @@ namespace nix { -const WorkerProto::FeatureSet WorkerProto::allFeatures{}; +const WorkerProto::Feature WorkerProto::pathLastUsageTimeFeature = "path-last-usage-time"; + +const WorkerProto::FeatureSet WorkerProto::allFeatures{WorkerProto::pathLastUsageTimeFeature}; WorkerProto::BasicClientConnection::~BasicClientConnection() { diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index a17d2c02857..0adb8a13fa6 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -262,6 +262,9 @@ UnkeyedValidPathInfo WorkerProto::Serialise::read(const St info.sigs = readStrings(conn.from); info.ca = ContentAddress::parseOpt(readString(conn.from)); } + if (conn.features && conn.features->contains(WorkerProto::pathLastUsageTimeFeature)) { + conn.from >> info.lastUsageTime; + } return info; } @@ -275,6 +278,9 @@ void WorkerProto::Serialise::write( if (GET_PROTOCOL_MINOR(conn.version) >= 16) { conn.to << pathInfo.ultimate << pathInfo.sigs << renderContentAddress(pathInfo.ca); } + if (conn.features && conn.features->contains(WorkerProto::pathLastUsageTimeFeature)) { + conn.to << pathInfo.lastUsageTime; + } } WorkerProto::ClientHandshakeInfo diff --git a/src/libutil/include/nix/util/sync.hh b/src/libutil/include/nix/util/sync.hh index 3a41d1bd808..7d0c9f7eb43 100644 --- a/src/libutil/include/nix/util/sync.hh +++ b/src/libutil/include/nix/util/sync.hh @@ -155,6 +155,10 @@ public: template using Sync = SyncBase, std::unique_lock>; +template +using SyncRec = + SyncBase, std::unique_lock>; + template using SharedSync = SyncBase, std::shared_lock>; diff --git a/src/nix/build-remote/build-remote.cc b/src/nix/build-remote/build-remote.cc index f62712d30ea..bb5ebfa8628 100644 --- a/src/nix/build-remote/build-remote.cc +++ b/src/nix/build-remote/build-remote.cc @@ -373,6 +373,8 @@ static int main_build_remote(int argc, char ** argv) assert(hopefullyOutputPath.second); if (!store->isValidPath(*hopefullyOutputPath.second)) missingPaths.insert(*hopefullyOutputPath.second); + else + store->bumpLastUsageTime(*hopefullyOutputPath.second); } } diff --git a/src/nix/nix-collect-garbage/nix-collect-garbage.cc b/src/nix/nix-collect-garbage/nix-collect-garbage.cc index 29ca17a5de2..62afd466e99 100644 --- a/src/nix/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix/nix-collect-garbage/nix-collect-garbage.cc @@ -78,6 +78,8 @@ static int main_nix_collect_garbage(int argc, char ** argv) else if (*arg == "--delete-older-than") { removeOld = true; deleteOlderThan = getArg(*arg, arg, end); + } else if (*arg == "--delete-paths-older-than") { + options.olderThan = parseOlderThanTimeSpec(getArg(*arg, arg, end)); } else if (*arg == "--dry-run") dryRun = true; else if (*arg == "--max-freed") diff --git a/src/nix/nix-store/nix-store.cc b/src/nix/nix-store/nix-store.cc index 3798c7fa015..4875ebcca95 100644 --- a/src/nix/nix-store/nix-store.cc +++ b/src/nix/nix-store/nix-store.cc @@ -116,6 +116,8 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true) store->ensurePath(path.path); else if (!store->isValidPath(path.path)) throw Error("path '%s' does not exist and cannot be created", store->printStorePath(path.path)); + else + store->bumpLastUsageTime(path.path); if (store2) { if (gcRoot == "") printGCWarning(); diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index d494b098686..e9bae1c6005 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -147,6 +147,8 @@ std::tuple prefetchFile( storePath = info.path; assert(info.ca); hash = info.ca->hash; + } else { + store->bumpLastUsageTime(storePath.value()); } return {storePath.value(), hash.value()}; diff --git a/src/nix/store-gc.cc b/src/nix/store-gc.cc index b0a627837ce..41225a3d601 100644 --- a/src/nix/store-gc.cc +++ b/src/nix/store-gc.cc @@ -4,12 +4,14 @@ #include "nix/store/store-api.hh" #include "nix/store/store-cast.hh" #include "nix/store/gc-store.hh" +#include "nix/store/profiles.hh" using namespace nix; struct CmdStoreGC : StoreCommand, MixDryRun { GCOptions options; + std::optional olderThan; CmdStoreGC() { @@ -19,6 +21,14 @@ struct CmdStoreGC : StoreCommand, MixDryRun .labels = {"n"}, .handler = {&options.maxFreed}, }); + addFlag({ + .longName = "older-than", + .description = "Only delete paths older than the specified age. *age* " + "must be in the format *N*`d`, where *N* denotes a number " + "of days.", + .labels = {"age"}, + .handler = {&olderThan}, + }); } std::string description() override @@ -38,6 +48,7 @@ struct CmdStoreGC : StoreCommand, MixDryRun auto & gcStore = require(*store); options.action = dryRun ? GCOptions::gcReturnDead : GCOptions::gcDeleteDead; + options.olderThan = olderThan.transform(parseOlderThanTimeSpec); GCResults results; PrintFreed freed(options.action == GCOptions::gcDeleteDead, results); gcStore.collectGarbage(options, results); diff --git a/src/nix/store-gc.md b/src/nix/store-gc.md index 956b3c8727a..5f4b23c1786 100644 --- a/src/nix/store-gc.md +++ b/src/nix/store-gc.md @@ -14,8 +14,19 @@ R""( # nix store gc --max 1G ``` +* Delete store paths older than 7 days: + + ```console + # nix store gc --older-than 7d + ``` + # Description This command deletes unreachable paths in the Nix store. +`--older-than Nd` restricts the GC to only deleting store paths whose most +recent usage is more than *N* days in the past. "Usage" is intentionally defined +ambiguously, however in general all operations which produce/require the +presence of a given store path count as "usage". + )""