diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index a343b5837db..f54ef174599 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -5,91 +5,112 @@ #include "nix/store/common-protocol.hh" #include "nix/store/common-protocol-impl.hh" -#include - namespace nix { -static void exportPath(Store & store, const StorePath & path, Sink & sink) -{ - auto info = store.queryPathInfo(path); - - HashSink hashSink(HashAlgorithm::SHA256); - TeeSink teeSink(sink, hashSink); - - store.narFromPath(path, teeSink); - - /* Refuse to export paths that have changed. This prevents - filesystem corruption from spreading to other machines. - Don't complain if the stored hash is zero (unknown). */ - Hash hash = hashSink.currentHash().hash; - if (hash != info->narHash && info->narHash != Hash(info->narHash.algo)) - throw Error( - "hash of path '%s' has changed from '%s' to '%s'!", - store.printStorePath(path), - info->narHash.to_string(HashFormat::Nix32, true), - hash.to_string(HashFormat::Nix32, true)); - - teeSink << exportMagic << store.printStorePath(path); - CommonProto::write(store, CommonProto::WriteConn{.to = teeSink}, info->references); - teeSink << (info->deriver ? store.printStorePath(*info->deriver) : "") << 0; -} +static const uint32_t exportMagicV1 = 0x4558494e; -void exportPaths(Store & store, const StorePathSet & paths, Sink & sink) +void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, unsigned int version) { auto sorted = store.topoSortPaths(paths); std::reverse(sorted.begin(), sorted.end()); - for (auto & path : sorted) { - sink << 1; - exportPath(store, path, sink); + auto dumpNar = [&](const ValidPathInfo & info) { + HashSink hashSink(HashAlgorithm::SHA256); + TeeSink teeSink(sink, hashSink); + + store.narFromPath(info.path, teeSink); + + /* Refuse to export paths that have changed. This prevents + filesystem corruption from spreading to other machines. + Don't complain if the stored hash is zero (unknown). */ + Hash hash = hashSink.currentHash().hash; + if (hash != info.narHash && info.narHash != Hash(info.narHash.algo)) + throw Error( + "hash of path '%s' has changed from '%s' to '%s'!", + store.printStorePath(info.path), + info.narHash.to_string(HashFormat::Nix32, true), + hash.to_string(HashFormat::Nix32, true)); + }; + + switch (version) { + + case 1: + for (auto & path : sorted) { + sink << 1; + auto info = store.queryPathInfo(path); + dumpNar(*info); + sink << exportMagicV1 << store.printStorePath(path); + CommonProto::write(store, CommonProto::WriteConn{.to = sink}, info->references); + sink << (info->deriver ? store.printStorePath(*info->deriver) : "") << 0; + } + sink << 0; + break; + + default: + throw Error("unsupported nario version %d", version); } - - sink << 0; } StorePaths importPaths(Store & store, Source & source, CheckSigsFlag checkSigs) { StorePaths res; - while (true) { - auto n = readNum(source); - if (n == 0) - break; - if (n != 1) - throw Error("input doesn't look like something created by 'nix-store --export'"); - - /* Extract the NAR from the source. */ - StringSink saved; - TeeSource tee{source, saved}; - NullFileSystemObjectSink ether; - parseDump(ether, tee); - - uint32_t magic = readInt(source); - if (magic != exportMagic) - throw Error("Nix archive cannot be imported; wrong format"); - - auto path = store.parseStorePath(readString(source)); - - // Activity act(*logger, lvlInfo, "importing path '%s'", info.path); - - auto references = CommonProto::Serialise::read(store, CommonProto::ReadConn{.from = source}); - auto deriver = readString(source); - auto narHash = hashString(HashAlgorithm::SHA256, saved.s); - - ValidPathInfo info{path, narHash}; - if (deriver != "") - info.deriver = store.parseStorePath(deriver); - info.references = references; - info.narSize = saved.s.size(); - - // Ignore optional legacy signature. - if (readInt(source) == 1) - readString(source); - - // Can't use underlying source, which would have been exhausted - auto source = StringSource(saved.s); - store.addToStore(info, source, NoRepair, checkSigs); - - res.push_back(info.path); + + auto version = readNum(source); + + /* Note: nario version 1 lacks an explicit header. The first + integer denotes whether a store path follows or not. So look + for 0 or 1. */ + switch (version) { + + case 0: + /* Empty version 1 nario, nothing to do. */ + break; + + case 1: + /* Non-empty version 1 nario. */ + while (true) { + /* Extract the NAR from the source. */ + StringSink saved; + TeeSource tee{source, saved}; + NullFileSystemObjectSink ether; + parseDump(ether, tee); + + uint32_t magic = readInt(source); + if (magic != exportMagicV1) + throw Error("nario cannot be imported; wrong format"); + + auto path = store.parseStorePath(readString(source)); + + auto references = CommonProto::Serialise::read(store, CommonProto::ReadConn{.from = source}); + auto deriver = readString(source); + auto narHash = hashString(HashAlgorithm::SHA256, saved.s); + + ValidPathInfo info{path, narHash}; + if (deriver != "") + info.deriver = store.parseStorePath(deriver); + info.references = references; + info.narSize = saved.s.size(); + + // Ignore optional legacy signature. + if (readInt(source) == 1) + readString(source); + + // Can't use underlying source, which would have been exhausted. + auto source2 = StringSource(saved.s); + store.addToStore(info, source2, NoRepair, checkSigs); + + res.push_back(info.path); + + auto n = readNum(source); + if (n == 0) + break; + if (n != 1) + throw Error("input doesn't look like a nario"); + } + break; + + default: + throw Error("input doesn't look like a nario"); } return res; diff --git a/src/libstore/include/nix/store/export-import.hh b/src/libstore/include/nix/store/export-import.hh index 15092202f1f..4ea696f992f 100644 --- a/src/libstore/include/nix/store/export-import.hh +++ b/src/libstore/include/nix/store/export-import.hh @@ -4,16 +4,11 @@ namespace nix { -/** - * Magic header of exportPath() output (obsolete). - */ -const uint32_t exportMagic = 0x4558494e; - /** * Export multiple paths in the format expected by `nix-store * --import`. The paths will be sorted topologically. */ -void exportPaths(Store & store, const StorePathSet & paths, Sink & sink); +void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, unsigned int version); /** * Import a sequence of NAR dumps created by `exportPaths()` into the diff --git a/src/nix/meson.build b/src/nix/meson.build index e989e80164f..d0b8e809caf 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -87,6 +87,7 @@ nix_sources = [ config_priv_h ] + files( 'make-content-addressed.cc', 'man-pages.cc', 'nar.cc', + 'nario.cc', 'optimise-store.cc', 'path-from-hash-part.cc', 'path-info.cc', diff --git a/src/nix/nario-export.md b/src/nix/nario-export.md new file mode 100644 index 00000000000..1aa124168e3 --- /dev/null +++ b/src/nix/nario-export.md @@ -0,0 +1,28 @@ +R""( + +# Examples + +* Export the closure of building `nixpkgs#hello`: + + ```console + # nix nario export --format 1 -r nixpkgs#hello > dump + ``` + + It can be imported in another store: + + ```console + # nix nario import < dump + ``` + +# Description + +This command prints on standard output a serialization of the specified store paths in `nario` format. This serialization can be imported into another store using `nix nario import`. + +References of a path are not exported by default; use `-r` to export a complete closure. +Paths are exported in topographically sorted order (i.e. if path `X` refers to `Y`, then `Y` appears before `X`). + +You must specify the desired `nario` version. Currently the following versions are supported: + +* `1`: This version is compatible with the legacy `nix-store --export` and `nix-store --import` commands. + +)"" diff --git a/src/nix/nario-import.md b/src/nix/nario-import.md new file mode 100644 index 00000000000..e2781995292 --- /dev/null +++ b/src/nix/nario-import.md @@ -0,0 +1,15 @@ +R""( + +# Examples + +* Import store paths from the file named `dump`: + + ```console + # nix nario import < dump + ``` + +# Description + +This command reads from standard input a serialization of store paths produced by `nix nario export` and adds them to the Nix store. + +)"" diff --git a/src/nix/nario-list.md b/src/nix/nario-list.md new file mode 100644 index 00000000000..80c1f10d7a0 --- /dev/null +++ b/src/nix/nario-list.md @@ -0,0 +1,18 @@ +R""( + +# Examples + +* List the contents of a nario file: + + ```console + # nix nario list < dump + /nix/store/4y1jj6cwvslmfh1bzkhbvhx77az6yf00-xgcc-14.2.1.20250322-libgcc: 201856 bytes + /nix/store/d8hnbm5hvbg2vza50garppb63y724i94-libunistring-1.3: 2070240 bytes + … + ``` + +# Description + +This command lists the contents of a nario file read from standard input. + +)"" diff --git a/src/nix/nario.cc b/src/nix/nario.cc new file mode 100644 index 00000000000..8e7708833f0 --- /dev/null +++ b/src/nix/nario.cc @@ -0,0 +1,189 @@ +#include "nix/cmd/command.hh" +#include "nix/main/shared.hh" +#include "nix/store/store-api.hh" +#include "nix/store/export-import.hh" +#include "nix/util/callback.hh" +#include "nix/util/fs-sink.hh" +#include "nix/util/archive.hh" + +using namespace nix; + +struct CmdNario : NixMultiCommand +{ + CmdNario() + : NixMultiCommand("nario", RegisterCommand::getCommandsFor({"nario"})) + { + } + + std::string description() override + { + return "operations for manipulating nario files"; + } + + Category category() override + { + return catUtility; + } +}; + +static auto rCmdNario = registerCommand("nario"); + +struct CmdNarioExport : StorePathsCommand +{ + unsigned int version = 0; + + CmdNarioExport() + { + addFlag({ + .longName = "format", + .description = "Version of the nario format to use. Must be `1`.", + .labels = {"nario-format"}, + .handler = {&version}, + }); + } + + std::string description() override + { + return "serialize store paths to standard output in nario format"; + } + + std::string doc() override + { + return +#include "nario-export.md" + ; + } + + void run(ref store, StorePaths && storePaths) override + { + if (!version) + throw UsageError("`nix nario export` requires `--format` argument"); + + FdSink sink(getStandardOutput()); + exportPaths(*store, StorePathSet(storePaths.begin(), storePaths.end()), sink, version); + } +}; + +static auto rCmdNarioExport = registerCommand2({"nario", "export"}); + +struct CmdNarioImport : StoreCommand +{ + std::string description() override + { + return "import store paths from a nario file on standard input"; + } + + std::string doc() override + { + return +#include "nario-import.md" + ; + } + + void run(ref store) override + { + FdSource source(getStandardInput()); + importPaths(*store, source, NoCheckSigs); // FIXME + } +}; + +static auto rCmdNarioImport = registerCommand2({"nario", "import"}); + +struct CmdNarioList : Command +{ + std::string description() override + { + return "list the contents of a nario file"; + } + + std::string doc() override + { + return +#include "nario-list.md" + ; + } + + void run() override + { + struct Config : StoreConfig + { + Config(const Params & params) + : StoreConfig(params) + { + } + + ref openStore() const override + { + abort(); + } + }; + + struct ListingStore : Store + { + ListingStore(ref config) + : Store{*config} + { + } + + void queryPathInfoUncached( + const StorePath & path, Callback> callback) noexcept override + { + callback(nullptr); + } + + std::optional isTrustedClient() override + { + return Trusted; + } + + std::optional queryPathFromHashPart(const std::string & hashPart) override + { + return std::nullopt; + } + + void + addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) override + { + logger->cout(fmt("%s: %d bytes", printStorePath(info.path), info.narSize)); + // Discard the NAR. + NullFileSystemObjectSink parseSink; + parseDump(parseSink, source); + } + + StorePath addToStoreFromDump( + Source & dump, + std::string_view name, + FileSerialisationMethod dumpMethod, + ContentAddressMethod hashMethod, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair) override + { + unsupported("addToStoreFromDump"); + } + + void narFromPath(const StorePath & path, Sink & sink) override + { + unsupported("narFromPath"); + } + + void queryRealisationUncached( + const DrvOutput &, Callback> callback) noexcept override + { + callback(nullptr); + } + + ref getFSAccessor(bool requireValidPath) override + { + return makeEmptySourceAccessor(); + } + }; + + FdSource source(getStandardInput()); + auto config = make_ref(StoreConfig::Params()); + ListingStore lister(config); + importPaths(lister, source, NoCheckSigs); + } +}; + +static auto rCmdNarioList = registerCommand2({"nario", "list"}); diff --git a/src/nix/nix-store/nix-store.cc b/src/nix/nix-store/nix-store.cc index 5f85e06f0b2..9b149cebee8 100644 --- a/src/nix/nix-store/nix-store.cc +++ b/src/nix/nix-store/nix-store.cc @@ -775,7 +775,7 @@ static void opExport(Strings opFlags, Strings opArgs) paths.insert(store->followLinksToStorePath(i)); FdSink sink(getStandardOutput()); - exportPaths(*store, paths, sink); + exportPaths(*store, paths, sink, 1); sink.flush(); } diff --git a/src/perl/lib/Nix/Store.xs b/src/perl/lib/Nix/Store.xs index 7aa918ba0c6..0d5a9f57172 100644 --- a/src/perl/lib/Nix/Store.xs +++ b/src/perl/lib/Nix/Store.xs @@ -234,7 +234,7 @@ StoreWrapper::exportPaths(int fd, ...) StorePathSet paths; for (int n = 2; n < items; ++n) paths.insert(THIS->store->parseStorePath(SvPV_nolen(ST(n)))); FdSink sink(fd); - exportPaths(*THIS->store, paths, sink); + exportPaths(*THIS->store, paths, sink, 1); } catch (Error & e) { croak("%s", e.what()); } diff --git a/tests/functional/export.sh b/tests/functional/export.sh index 3e895a5402d..6fe6c9d4e66 100755 --- a/tests/functional/export.sh +++ b/tests/functional/export.sh @@ -9,9 +9,14 @@ clearStore outPath=$(nix-build dependencies.nix --no-out-link) nix-store --export $outPath > $TEST_ROOT/exp +nix nario export --format 1 "$outPath" > $TEST_ROOT/exp2 +cmp "$TEST_ROOT/exp" "$TEST_ROOT/exp2" nix-store --export $(nix-store -qR $outPath) > $TEST_ROOT/exp_all +nix nario export --format 1 -r "$outPath" > $TEST_ROOT/exp_all2 +cmp "$TEST_ROOT/exp_all" "$TEST_ROOT/exp_all2" + if nix-store --export $outPath >/dev/full ; then echo "exporting to a bad file descriptor should fail" exit 1 @@ -38,3 +43,13 @@ clearStore # Regression test: the derivers in exp_all2 are empty, which shouldn't # cause a failure. nix-store --import < $TEST_ROOT/exp_all2 + + +# Test `nix nario import` on files created by `nix-store --export`. +clearStore +nix nario import < $TEST_ROOT/exp_all +nix path-info "$outPath" + + +# Test `nix nario list`. +nix nario list < $TEST_ROOT/exp_all | grepQuiet "dependencies-input-0: .* bytes"