diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 087b1c4..a88f13d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -11,6 +11,10 @@ jobs: steps: - uses: actions/checkout@v3 + - uses: webfactory/ssh-agent@v0.7.0 + with: + ssh-private-key: ${{ secrets.LOL_DETSYS_CI_SSH_PRIVATE_KEY }} + - uses: DeterminateSystems/nix-installer-action@main - uses: DeterminateSystems/magic-nix-cache-action@main @@ -32,6 +36,10 @@ jobs: steps: - uses: actions/checkout@v3 + - uses: webfactory/ssh-agent@v0.7.0 + with: + ssh-private-key: ${{ secrets.LOL_DETSYS_CI_SSH_PRIVATE_KEY }} + - uses: DeterminateSystems/flake-checker-action@main - uses: DeterminateSystems/nix-installer-action@main @@ -55,11 +63,15 @@ jobs: steps: - uses: actions/checkout@v3 + - uses: webfactory/ssh-agent@v0.7.0 + with: + ssh-private-key: ${{ secrets.LOL_DETSYS_CI_SSH_PRIVATE_KEY }} + - uses: DeterminateSystems/flake-checker-action@main - uses: DeterminateSystems/nix-installer-action@main - - uses: DeterminateSystems/magic-nix-cache-action@main + - uses: DeterminateSystems/magic-nix-cache-action-priv@attic-v2 - name: Build package run: "nix build .# -L --fallback" diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml index b028129..9f8143f 100644 --- a/.github/workflows/checks.yaml +++ b/.github/workflows/checks.yaml @@ -11,6 +11,10 @@ jobs: steps: - uses: actions/checkout@v3 + - uses: webfactory/ssh-agent@v0.7.0 + with: + ssh-private-key: ${{ secrets.LOL_DETSYS_CI_SSH_PRIVATE_KEY }} + - name: Install Nix uses: DeterminateSystems/nix-installer-action@main - uses: DeterminateSystems/magic-nix-cache-action@main diff --git a/.github/workflows/release-prs.yml b/.github/workflows/release-prs.yml index 6d899a4..0d87add 100644 --- a/.github/workflows/release-prs.yml +++ b/.github/workflows/release-prs.yml @@ -11,6 +11,7 @@ on: jobs: build: uses: ./.github/workflows/build.yaml + secrets: inherit release: needs: build @@ -19,7 +20,7 @@ jobs: # Only intra-repo PRs are allowed to have PR artifacts uploaded # We only want to trigger once the upload once in the case the upload label is added, not when any label is added if: | - github.event.pull_request.head.repo.full_name == 'DeterminateSystems/magic-nix-cache' + github.event.pull_request.head.repo.full_name == 'DeterminateSystems/magic-nix-cache-priv' && ( (github.event.action == 'labeled' && github.event.label.name == 'upload to s3') || (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'upload to s3')) diff --git a/Cargo.lock b/Cargo.lock index 03992b7..9f4b1e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,18 +2,118 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "aho-corasick" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +dependencies = [ + "memchr", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is-terminal", + "utf8parse", +] + [[package]] name = "anstyle" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" +[[package]] +name = "anstyle-parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +dependencies = [ + "windows-sys 0.48.0", +] + +[[package]] +name = "anstyle-wincon" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c677ab05e09154296dd37acecd46420c17b9713e8366facafa8fc0885167cf4c" +dependencies = [ + "anstyle", + "windows-sys 0.48.0", +] + [[package]] name = "anyhow" version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" +[[package]] +name = "async-channel" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" +dependencies = [ + "concurrent-queue", + "event-listener", + "futures-core", +] + +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.32", +] + [[package]] name = "async-trait" version = "0.1.68" @@ -22,7 +122,65 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.32", +] + +[[package]] +name = "attic" +version = "0.2.0" +source = "git+ssh://git@github.com/DeterminateSystems/attic-priv?branch=main#217cbe932ceb81f504621fead261edc792eb5e2c" +dependencies = [ + "async-stream", + "base64 0.21.2", + "bytes", + "cxx", + "cxx-build", + "digest", + "displaydoc", + "ed25519-compact", + "futures", + "hex", + "lazy_static", + "nix-base32", + "pkg-config", + "regex", + "serde", + "serde_with", + "sha2", + "tempfile", + "tokio", + "url", +] + +[[package]] +name = "attic-client" +version = "0.2.0" +source = "git+ssh://git@github.com/DeterminateSystems/attic-priv?branch=main#217cbe932ceb81f504621fead261edc792eb5e2c" +dependencies = [ + "anyhow", + "async-channel", + "attic", + "bytes", + "clap", + "clap_complete", + "const_format", + "dialoguer", + "displaydoc", + "enum-as-inner 0.6.0", + "futures", + "humantime", + "indicatif", + "lazy_static", + "notify", + "regex", + "reqwest", + "serde", + "serde_json", + "tokio", + "toml", + "tracing", + "tracing-subscriber", + "xdg", ] [[package]] @@ -88,9 +246,15 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.32", ] +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.21.2" @@ -132,9 +296,12 @@ checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" [[package]] name = "cc" -version = "1.0.79" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] [[package]] name = "cfg-if" @@ -142,6 +309,19 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "num-traits", + "serde", + "windows-targets 0.48.0", +] + [[package]] name = "clap" version = "4.3.8" @@ -159,12 +339,23 @@ version = "4.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a78fbdd3cc2914ddf37ba444114bc7765bbdcb55ec9cbe6fa054f0137400717" dependencies = [ + "anstream", "anstyle", "bitflags 1.3.2", "clap_lex", + "strsim", "terminal_size", ] +[[package]] +name = "clap_complete" +version = "4.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bffe91f06a11b4b9420f62103854e90867812cd5d01557f853c5ee8e791b12ae" +dependencies = [ + "clap", +] + [[package]] name = "clap_derive" version = "4.3.2" @@ -174,7 +365,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.32", ] [[package]] @@ -183,6 +374,64 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + +[[package]] +name = "concurrent-queue" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "console" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "unicode-width", + "windows-sys 0.45.0", +] + +[[package]] +name = "const_format" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a214c7af3d04997541b18d432afaff4c455e79e2029079647e72fc2bd27673" +dependencies = [ + "const_format_proc_macros", +] + +[[package]] +name = "const_format_proc_macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f6ff08fd20f4f299298a28e2dfa8a8ba1036e6cd2460ac1de7b425d76f2500" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + [[package]] name = "core-foundation" version = "0.9.3" @@ -208,6 +457,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +dependencies = [ + "cfg-if", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -218,6 +476,56 @@ dependencies = [ "typenum", ] +[[package]] +name = "ct-codecs" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3b7eb4404b8195a9abb6356f4ac07d8ba267045c8d6d220ac4dc992e6cc75df" + +[[package]] +name = "cxx" +version = "1.0.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88abab2f5abbe4c56e8f1fb431b784d710b709888f35755a160e62e33fe38e8" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c0c11acd0e63bae27dcd2afced407063312771212b7a823b4fd72d633be30fb" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn 2.0.32", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3816ed957c008ccd4728485511e3d9aaf7db419aa321e3d2c5a2f3411e36c8" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26acccf6f445af85ea056362561a24ef56cdc15fcc685f03aec50b9c702cb6d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.32", +] + [[package]] name = "daemonize" version = "0.5.0" @@ -227,12 +535,57 @@ dependencies = [ "libc", ] +[[package]] +name = "darling" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.32", +] + +[[package]] +name = "darling_macro" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.32", +] + [[package]] name = "data-encoding" version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" +[[package]] +name = "deranged" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" +dependencies = [ + "powerfmt", + "serde", +] + [[package]] name = "derivative" version = "2.2.0" @@ -244,6 +597,18 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "dialoguer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59c6f2989294b9a498d3ad5491a79c6deb604617378e1cdc4bfc1c1361fe2f87" +dependencies = [ + "console", + "shell-words", + "tempfile", + "zeroize", +] + [[package]] name = "digest" version = "0.10.7" @@ -252,8 +617,36 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] +[[package]] +name = "displaydoc" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.32", +] + +[[package]] +name = "ed25519-compact" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a3d382e8464107391c8706b4c14b087808ecb909f6c15c34114bc42e53a9e4c" +dependencies = [ + "ct-codecs", + "getrandom", +] + +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encoding_rs" version = "0.8.32" @@ -275,6 +668,24 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "enum-as-inner" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ffccbb6966c05b32ef8fbac435df276c4ae4d3dc55a8cd0eb9745e6c12f546a" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.32", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.3.1" @@ -296,6 +707,33 @@ dependencies = [ "libc", ] +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + +[[package]] +name = "filetime" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "windows-sys 0.48.0", +] + [[package]] name = "fnv" version = "1.0.7" @@ -304,9 +742,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "form_urlencoded" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] @@ -319,6 +757,7 @@ checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" dependencies = [ "futures-channel", "futures-core", + "futures-executor", "futures-io", "futures-sink", "futures-task", @@ -341,6 +780,17 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +[[package]] +name = "futures-executor" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.28" @@ -355,7 +805,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.32", ] [[package]] @@ -376,6 +826,7 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" dependencies = [ + "futures-channel", "futures-core", "futures-io", "futures-macro", @@ -441,7 +892,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 1.9.3", "slab", "tokio", "tokio-util", @@ -454,6 +905,12 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" + [[package]] name = "heck" version = "0.4.1" @@ -481,6 +938,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "hostname" version = "0.3.1" @@ -532,6 +998,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" version = "0.14.26" @@ -569,6 +1041,35 @@ dependencies = [ "tokio-rustls", ] +[[package]] +name = "iana-time-zone" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "0.2.3" @@ -582,9 +1083,9 @@ dependencies = [ [[package]] name = "idna" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -597,7 +1098,61 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +dependencies = [ + "equivalent", + "hashbrown 0.14.3", + "serde", +] + +[[package]] +name = "indicatif" +version = "0.17.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" +dependencies = [ + "console", + "instant", + "number_prefix", + "portable-atomic", + "unicode-width", +] + +[[package]] +name = "inotify" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8069d3ec154eb856955c1c0fbffefbf5f3c40a104ec912d4797314c1801abff" +dependencies = [ + "bitflags 1.3.2", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb" +dependencies = [ + "libc", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", ] [[package]] @@ -629,6 +1184,18 @@ version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" +[[package]] +name = "is-terminal" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" +dependencies = [ + "hermit-abi 0.3.1", + "io-lifetimes", + "rustix", + "windows-sys 0.48.0", +] + [[package]] name = "is_ci" version = "1.1.1" @@ -650,6 +1217,41 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jwt" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6204285f77fe7d9784db3fdc449ecce1a0114927a51d5a41c4c7a292011c015f" +dependencies = [ + "base64 0.13.1", + "crypto-common", + "digest", + "hmac", + "serde", + "serde_json", + "sha2", +] + +[[package]] +name = "kqueue" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7447f1ca1b7b563588a205fe93dea8df60fd981423a768bc1c0ded35ed147d0c" +dependencies = [ + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b" +dependencies = [ + "bitflags 1.3.2", + "libc", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -658,9 +1260,18 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.147" +version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" + +[[package]] +name = "link-cplusplus" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d240c6f7e1ba3a28b0249f774e6a9dd0175054b52dfbb61b16eb8505c3785c9" +dependencies = [ + "cc", +] [[package]] name = "linked-hash-map" @@ -703,12 +1314,17 @@ dependencies = [ name = "magic-nix-cache" version = "0.1.1" dependencies = [ + "attic", + "attic-client", "axum", "axum-macros", "clap", "daemonize", "gha-cache", + "indicatif", "is_ci", + "jwt", + "netrc-rs", "reqwest", "serde", "serde_json", @@ -768,10 +1384,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", + "log", "wasi", "windows-sys 0.48.0", ] +[[package]] +name = "netrc-rs" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea2970fbbc8c785e8246234a7bd004ed66cd1ed1a35ec73669a92545e419b836" + +[[package]] +name = "nix-base32" +version = "0.1.2-alpha.0" +source = "git+https://github.com/zhaofengli/nix-base32.git?rev=b850c6e9273d1c39bd93abb704a53345f5be92eb#b850c6e9273d1c39bd93abb704a53345f5be92eb" + +[[package]] +name = "notify" +version = "6.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d" +dependencies = [ + "bitflags 2.3.2", + "filetime", + "inotify", + "kqueue", + "libc", + "log", + "mio", + "walkdir", + "windows-sys 0.48.0", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -782,6 +1427,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-traits" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.15.0" @@ -792,6 +1446,12 @@ dependencies = [ "libc", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "once_cell" version = "1.18.0" @@ -830,14 +1490,14 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-targets", + "windows-targets 0.48.0", ] [[package]] name = "percent-encoding" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pin-project" @@ -856,7 +1516,7 @@ checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" dependencies = [ "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.32", ] [[package]] @@ -871,6 +1531,24 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" + +[[package]] +name = "portable-atomic" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bccab0e7fd7cc19f820a1c8c91720af652d0c88dc9664dd72aef2614f04af3b" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -946,6 +1624,8 @@ version = "1.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f" dependencies = [ + "aho-corasick", + "memchr", "regex-syntax 0.7.2", ] @@ -976,7 +1656,7 @@ version = "0.11.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ - "base64", + "base64 0.21.2", "bytes", "encoding_rs", "futures-core", @@ -1009,6 +1689,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", + "webpki-roots", "winreg 0.10.1", ] @@ -1031,12 +1712,26 @@ dependencies = [ "cc", "libc", "once_cell", - "spin", - "untrusted", + "spin 0.5.2", + "untrusted 0.7.1", "web-sys", "winapi", ] +[[package]] +name = "ring" +version = "0.17.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "684d5e6e18f669ccebf64a92236bb7db9a34f07be010e3627368182027180866" +dependencies = [ + "cc", + "getrandom", + "libc", + "spin 0.9.8", + "untrusted 0.9.0", + "windows-sys 0.48.0", +] + [[package]] name = "rustix" version = "0.37.20" @@ -1058,7 +1753,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e32ca28af694bc1bbf399c33a516dbdf1c90090b8ab23c2bc24f834aa2247f5f" dependencies = [ "log", - "ring", + "ring 0.16.20", "rustls-webpki", "sct", ] @@ -1081,7 +1776,7 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" dependencies = [ - "base64", + "base64 0.21.2", ] [[package]] @@ -1090,8 +1785,8 @@ version = "0.100.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" dependencies = [ - "ring", - "untrusted", + "ring 0.16.20", + "untrusted 0.7.1", ] [[package]] @@ -1106,6 +1801,15 @@ version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.21" @@ -1121,14 +1825,20 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "scratch" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3cf7c11c38cb994f3d40e8a8cde3bbd1f72a435e4c49e85d6553d8312306152" + [[package]] name = "sct" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" dependencies = [ - "ring", - "untrusted", + "ring 0.16.20", + "untrusted 0.7.1", ] [[package]] @@ -1156,22 +1866,22 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.164" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.164" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.32", ] [[package]] @@ -1194,6 +1904,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12022b835073e5b11e90a14f86838ceb1c8fb0325b72416845c487ac0fa95e80" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -1206,6 +1925,35 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64cd236ccc1b7a29e7e2739f27c0b2dd199804abc4290e32f59f3b68d6405c23" +dependencies = [ + "base64 0.21.2", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.1.0", + "serde", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93634eb5f75a2323b16de4748022ac4297f9e76b6dced2be287a099f41b5e788" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.32", +] + [[package]] name = "sha2" version = "0.10.7" @@ -1226,6 +1974,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shell-words" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -1276,6 +2030,24 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + [[package]] name = "syn" version = "1.0.109" @@ -1289,9 +2061,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.22" +version = "2.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2efbeae7acf4eabd6bcdcbd11c92f45231ddda7539edc7806bd1a04a03b24616" +checksum = "239814284fd6f1a4ffe4ca893952cdd93c224b6a1571c9a9eadd670295c0c9e2" dependencies = [ "proc-macro2", "quote", @@ -1304,6 +2076,29 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "tempfile" +version = "3.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" +dependencies = [ + "autocfg", + "cfg-if", + "fastrand", + "redox_syscall", + "rustix", + "windows-sys 0.48.0", +] + +[[package]] +name = "termcolor" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449" +dependencies = [ + "winapi-util", +] + [[package]] name = "terminal_size" version = "0.2.6" @@ -1331,7 +2126,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.32", ] [[package]] @@ -1344,6 +2139,35 @@ dependencies = [ "once_cell", ] +[[package]] +name = "time" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" +dependencies = [ + "deranged", + "itoa", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" +dependencies = [ + "time-core", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -1373,9 +2197,21 @@ dependencies = [ "pin-project-lite", "signal-hook-registry", "socket2 0.4.9", + "tokio-macros", "windows-sys 0.48.0", ] +[[package]] +name = "tokio-macros" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.32", +] + [[package]] name = "tokio-rustls" version = "0.24.1" @@ -1411,6 +2247,40 @@ dependencies = [ "tracing", ] +[[package]] +name = "toml" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.19.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" +dependencies = [ + "indexmap 2.1.0", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower" version = "0.4.13" @@ -1477,7 +2347,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.32", ] [[package]] @@ -1528,7 +2398,7 @@ dependencies = [ "async-trait", "cfg-if", "data-encoding", - "enum-as-inner", + "enum-as-inner 0.5.1", "futures-channel", "futures-io", "futures-util", @@ -1603,6 +2473,18 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-width" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" + +[[package]] +name = "unicode-xid" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" + [[package]] name = "untrusted" version = "0.7.1" @@ -1610,16 +2492,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] -name = "url" -version = "2.4.0" +name = "untrusted" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", - "idna 0.4.0", + "idna 0.5.0", "percent-encoding", + "serde", ] +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + [[package]] name = "valuable" version = "0.1.0" @@ -1632,6 +2527,16 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "walkdir" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -1668,7 +2573,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.32", "wasm-bindgen-shared", ] @@ -1702,7 +2607,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.32", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -1736,6 +2641,25 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" +dependencies = [ + "ring 0.17.6", + "untrusted 0.9.0", +] + +[[package]] +name = "webpki-roots" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" +dependencies = [ + "webpki", +] + [[package]] name = "widestring" version = "1.0.2" @@ -1758,12 +2682,30 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +dependencies = [ + "winapi", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af6041b3f84485c21b57acdc0fee4f4f0c93f426053dc05fa5d6fc262537bbff" +dependencies = [ + "windows-targets 0.48.0", +] + [[package]] name = "windows-sys" version = "0.42.0" @@ -1779,13 +2721,37 @@ dependencies = [ "windows_x86_64_msvc 0.42.2", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.0", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", ] [[package]] @@ -1887,6 +2853,15 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +[[package]] +name = "winnow" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829846f3e3db426d4cee4510841b71a8e58aa2a76b1132579487ae430ccd9c7b" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.10.1" @@ -1905,3 +2880,15 @@ dependencies = [ "cfg-if", "windows-sys 0.48.0", ] + +[[package]] +name = "xdg" +version = "2.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213b7324336b53d2414b2db8537e56544d981803139155afa84f76eeebb7a546" + +[[package]] +name = "zeroize" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" diff --git a/crane.nix b/crane.nix index 399f55f..c07adf8 100644 --- a/crane.nix +++ b/crane.nix @@ -6,6 +6,7 @@ , rust-bin , nix-gitignore , supportedSystems +, nix-flake }: let @@ -32,7 +33,7 @@ let else import pkgs.path { inherit system crossSystem; - overlays = [ ]; + overlays = [ nix-flake.overlays.default ]; }; rustTargetSpec = rust.toRustTargetSpec pkgsCross.pkgsStatic.stdenv.hostPlatform; @@ -66,10 +67,13 @@ let cargoTargets = lib.mapAttrsToList (_: p: p.rustTargetSpec) crossPlatforms; cargoCrossEnvs = lib.foldl (acc: p: acc // p.env) { } (builtins.attrValues crossPlatforms); - makeBuildInputs = pkgs: with pkgs; [ ] + makeBuildInputs = pkgs: + [ pkgs.nix + pkgs.boost # needed for clippy + ] ++ lib.optionals pkgs.stdenv.isDarwin [ - darwin.apple_sdk.frameworks.Security - (libiconv.override { enableStatic = true; enableShared = false; }) + pkgs.darwin.apple_sdk.frameworks.Security + (pkgs.libiconv.override { enableStatic = true; enableShared = false; }) ]; buildFor = system: @@ -87,6 +91,8 @@ let inherit (crateName) pname version; inherit src; + nativeBuildInputs = [ pkgs.pkg-config ]; + buildInputs = makeBuildInputs pkgs; cargoExtraArgs = "--target ${crossPlatform.rustTargetSpec}"; diff --git a/flake.lock b/flake.lock index 09a5415..8c25828 100644 --- a/flake.lock +++ b/flake.lock @@ -38,6 +38,22 @@ "url": "https://flakehub.com/f/edolstra/flake-compat/1.0.1.tar.gz" } }, + "flake-compat_2": { + "flake": false, + "locked": { + "lastModified": 1673956053, + "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, "flake-utils": { "inputs": { "systems": "systems" @@ -74,25 +90,94 @@ "type": "github" } }, - "nixpkgs": { + "lowdown-src": { + "flake": false, "locked": { - "lastModified": 1696604326, - "narHash": "sha256-YXUNI0kLEcI5g8lqGMb0nh67fY9f2YoJsILafh6zlMo=", - "rev": "87828a0e03d1418e848d3dd3f3014a632e4a4f64", - "revCount": 533189, + "lastModified": 1633514407, + "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=", + "owner": "kristapsdz", + "repo": "lowdown", + "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8", + "type": "github" + }, + "original": { + "owner": "kristapsdz", + "repo": "lowdown", + "type": "github" + } + }, + "nix": { + "inputs": { + "flake-compat": "flake-compat_2", + "lowdown-src": "lowdown-src", + "nixpkgs": "nixpkgs", + "nixpkgs-regression": "nixpkgs-regression" + }, + "locked": { + "lastModified": 1701122567, + "narHash": "sha256-iA8DqS+W2fWTfR+nNJSvMHqQ+4NpYMRT3b+2zS6JTvE=", + "rev": "50f8f1c8bc019a4c0fd098b9ac674b94cfc6af0d", + "revCount": 15434, "type": "tarball", - "url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.1.533189%2Brev-87828a0e03d1418e848d3dd3f3014a632e4a4f64/018b0dc8-e84f-7c59-b5d6-16849c3b2074/source.tar.gz" + "url": "https://api.flakehub.com/f/pinned/NixOS/nix/2.19.2/018c1be0-1b88-7682-b3bf-948ec82d0a0b/source.tar.gz" }, "original": { "type": "tarball", - "url": "https://flakehub.com/f/NixOS/nixpkgs/0.1.533189.tar.gz" + "url": "https://flakehub.com/f/NixOS/nix/2.19.tar.gz" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1698876495, + "narHash": "sha256-nsQo2/mkDUFeAjuu92p0dEqhRvHHiENhkKVIV1y0/Oo=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "9eb24edd6a0027fed010ccfe300a9734d029983c", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "release-23.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-regression": { + "locked": { + "lastModified": 1643052045, + "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1701615100, + "narHash": "sha256-7VI84NGBvlCTduw2aHLVB62NvCiZUlALLqBe5v684Aw=", + "rev": "e9f06adb793d1cca5384907b3b8a4071d5d7cb19", + "revCount": 492472, + "type": "tarball", + "url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.2305.492472%2Brev-e9f06adb793d1cca5384907b3b8a4071d5d7cb19/018c3e8e-bc66-7f34-9054-4564bf44b6f8/source.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://flakehub.com/f/NixOS/nixpkgs/0.2305.tar.gz" } }, "root": { "inputs": { "crane": "crane", "flake-compat": "flake-compat", - "nixpkgs": "nixpkgs", + "nix": "nix", + "nixpkgs": "nixpkgs_2", "rust-overlay": "rust-overlay_2" } }, diff --git a/flake.nix b/flake.nix index fbc8285..ddf39b7 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "GitHub Actions-powered Nix binary cache"; inputs = { - nixpkgs.url = "https://flakehub.com/f/NixOS/nixpkgs/0.1.533189.tar.gz"; + nixpkgs.url = "https://flakehub.com/f/NixOS/nixpkgs/0.2305.tar.gz"; rust-overlay = { url = "github:oxalica/rust-overlay"; @@ -16,11 +16,13 @@ }; flake-compat.url = "https://flakehub.com/f/edolstra/flake-compat/1.0.1.tar.gz"; + + nix.url = "https://flakehub.com/f/NixOS/nix/2.19.tar.gz"; }; - outputs = { self, nixpkgs, ... }@inputs: + outputs = { self, nixpkgs, nix, ... }@inputs: let - overlays = [ inputs.rust-overlay.overlays.default ]; + overlays = [ inputs.rust-overlay.overlays.default nix.overlays.default ]; supportedSystems = [ "aarch64-linux" "x86_64-linux" @@ -32,13 +34,35 @@ cranePkgs = pkgs.callPackage ./crane.nix { inherit supportedSystems; inherit (inputs) crane; + nix-flake = nix; }; inherit (pkgs) lib; }); in { packages = forEachSupportedSystem ({ pkgs, cranePkgs, ... }: rec { - inherit (cranePkgs) magic-nix-cache; + magic-nix-cache = (pkgs.pkgsStatic.callPackage ./package.nix { + rustPlatform = pkgs.pkgsStatic.rustPackages_1_70.rustPlatform; + nix = pkgs.pkgsStatic.nix.overrideAttrs (old: { + patches = (old.patches or []) ++ [ ./nix.patch ]; + }); + }).overrideAttrs (old: { + nativeBuildInputs = (old.nativeBuildInputs or []) ++ [ + pkgs.nukeReferences + ]; + + # Read by pkg_config crate (do some autodetection in build.rs?) + PKG_CONFIG_ALL_STATIC = "1"; + + "NIX_CFLAGS_LINK_${pkgs.pkgsStatic.stdenv.cc.suffixSalt}" = "-lc"; + RUSTFLAGS = "-C relocation-model=static"; + + postFixup = (old.postFixup or "") + '' + rm -f $out/nix-support/propagated-build-inputs + nuke-refs $out/bin/magic-nix-cache + ''; + }); + #inherit (cranePkgs) magic-nix-cache; default = magic-nix-cache; }); diff --git a/magic-nix-cache/Cargo.toml b/magic-nix-cache/Cargo.toml index 989880d..14c30ae 100644 --- a/magic-nix-cache/Cargo.toml +++ b/magic-nix-cache/Cargo.toml @@ -22,12 +22,20 @@ daemonize = "0.5.0" is_ci = "1.1.1" sha2 = { version = "0.10.6", default-features = false } reqwest = { version = "0.11.17", default-features = false, features = ["blocking", "rustls-tls-native-roots", "trust-dns"] } +netrc-rs = "0.1.2" +jwt = { version = "0.16" } +attic = { git = "ssh://git@github.com/DeterminateSystems/attic-priv", branch = "main" } +#attic = { path = "../../attic-priv/attic" } +attic-client = { git = "ssh://git@github.com/DeterminateSystems/attic-priv", branch = "main" } +#attic-client = { path = "../../attic-priv/client" } +indicatif = "0.17" [dependencies.tokio] version = "1.28.0" default-features = false features = [ "fs", + "macros", "process", "rt", "rt-multi-thread", diff --git a/magic-nix-cache/src/api.rs b/magic-nix-cache/src/api.rs index 12303df..3a8ea2f 100644 --- a/magic-nix-cache/src/api.rs +++ b/magic-nix-cache/src/api.rs @@ -36,7 +36,7 @@ async fn workflow_start(Extension(state): Extension) -> Result>(); - tracing::info!("Pushing {} new paths", new_paths.len()); - let store_uri = make_store_uri(&state.self_endpoint); - upload_paths(new_paths.clone(), &store_uri).await?; + if state.api.is_some() { + tracing::info!("Pushing {} new paths to GHA cache", new_paths.len()); + let store_uri = make_store_uri(&state.self_endpoint); + upload_paths(new_paths.clone(), &store_uri).await?; + } + + if let Some(attic_state) = &state.flakehub_state { + tracing::info!("Pushing {} new paths to Attic", new_paths.len()); + + let new_paths = new_paths + .iter() + .map(|path| state.store.follow_store_path(path).unwrap()) + .collect(); + + crate::flakehub::push(attic_state, state.store.clone(), new_paths).await?; + } let sender = state.shutdown_sender.lock().await.take().unwrap(); sender.send(()).unwrap(); diff --git a/magic-nix-cache/src/binary_cache.rs b/magic-nix-cache/src/binary_cache.rs index 5283684..316548e 100644 --- a/magic-nix-cache/src/binary_cache.rs +++ b/magic-nix-cache/src/binary_cache.rs @@ -61,9 +61,11 @@ async fn get_narinfo( return pull_through(&state, &path); } - if let Some(url) = state.api.get_file_url(&[&key]).await? { - state.metrics.narinfos_served.incr(); - return Ok(Redirect::temporary(&url)); + if let Some(api) = &state.api { + if let Some(url) = api.get_file_url(&[&key]).await? { + state.metrics.narinfos_served.incr(); + return Ok(Redirect::temporary(&url)); + } } let mut negative_cache = state.narinfo_nagative_cache.write().await; @@ -88,13 +90,15 @@ async fn put_narinfo( return Err(Error::BadRequest); } + let api = state.api.as_ref().ok_or(Error::GHADisabled)?; + let store_path_hash = components[0].to_string(); let key = format!("{}.narinfo", store_path_hash); - let allocation = state.api.allocate_file_with_random_suffix(&key).await?; + let allocation = api.allocate_file_with_random_suffix(&key).await?; let stream = StreamReader::new( body.map(|r| r.map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))), ); - state.api.upload_file(allocation, stream).await?; + api.upload_file(allocation, stream).await?; state.metrics.narinfos_uploaded.incr(); state @@ -107,7 +111,13 @@ async fn put_narinfo( } async fn get_nar(Extension(state): Extension, Path(path): Path) -> Result { - if let Some(url) = state.api.get_file_url(&[&path]).await? { + if let Some(url) = state + .api + .as_ref() + .ok_or(Error::GHADisabled)? + .get_file_url(&[&path]) + .await? + { state.metrics.nars_served.incr(); return Ok(Redirect::temporary(&url)); } @@ -124,11 +134,13 @@ async fn put_nar( Path(path): Path, body: BodyStream, ) -> Result<()> { - let allocation = state.api.allocate_file_with_random_suffix(&path).await?; + let api = state.api.as_ref().ok_or(Error::GHADisabled)?; + + let allocation = api.allocate_file_with_random_suffix(&path).await?; let stream = StreamReader::new( body.map(|r| r.map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))), ); - state.api.upload_file(allocation, stream).await?; + api.upload_file(allocation, stream).await?; state.metrics.nars_uploaded.incr(); Ok(()) diff --git a/magic-nix-cache/src/error.rs b/magic-nix-cache/src/error.rs index 9b99701..574b9d9 100644 --- a/magic-nix-cache/src/error.rs +++ b/magic-nix-cache/src/error.rs @@ -24,6 +24,9 @@ pub enum Error { #[error("Failed to upload paths")] FailedToUpload, + + #[error("GHA cache is disabled")] + GHADisabled, } impl IntoResponse for Error { diff --git a/magic-nix-cache/src/flakehub.rs b/magic-nix-cache/src/flakehub.rs new file mode 100644 index 0000000..c935ee7 --- /dev/null +++ b/magic-nix-cache/src/flakehub.rs @@ -0,0 +1,278 @@ +use crate::error::Result; +use attic::api::v1::cache_config::{CreateCacheRequest, KeypairConfig}; +use attic::cache::CacheSliceIdentifier; +use attic::nix_store::{NixStore, StorePath}; +use attic_client::{ + api::{ApiClient, ApiError}, + config::ServerConfig, + push::{PushConfig, Pusher}, +}; +use serde::Deserialize; +use std::env; +use std::path::Path; +use std::str::FromStr; +use std::sync::Arc; +use tokio::fs::File; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +const JWT_PREFIX: &str = "flakehub1_"; +const USER_AGENT: &str = "magic-nix-cache"; + +pub struct State { + cache: CacheSliceIdentifier, + + pub substituter: String, + + api: ApiClient, +} + +pub async fn init_cache( + flakehub_api_server: &str, + flakehub_api_server_netrc: &Path, + flakehub_cache_server: &str, +) -> Result { + // Parse netrc to get the credentials for api.flakehub.com. + let netrc = { + let mut netrc_file = File::open(flakehub_api_server_netrc).await?; + let mut netrc_contents = String::new(); + netrc_file.read_to_string(&mut netrc_contents).await?; + netrc_rs::Netrc::parse(netrc_contents, false).unwrap() + }; + + let netrc_entry = { + netrc + .machines + .iter() + .find(|machine| { + machine.name.as_ref().unwrap() + == &reqwest::Url::parse(flakehub_api_server) + .unwrap() + .host() + .unwrap() + .to_string() + }) + .unwrap() + .to_owned() + }; + + let flakehub_cache_server_hostname = reqwest::Url::parse(flakehub_cache_server) + .unwrap() + .host() + .unwrap() + .to_string(); + + // Append an entry for the FlakeHub cache server to netrc. + if !netrc + .machines + .iter() + .any(|machine| machine.name.as_ref().unwrap() == &flakehub_cache_server_hostname) + { + let mut netrc_file = tokio::fs::OpenOptions::new() + .create(false) + .append(true) + .open(flakehub_api_server_netrc) + .await + .unwrap(); + netrc_file + .write_all( + format!( + "\nmachine {} password {}\n\n", + flakehub_cache_server_hostname, + netrc_entry.password.as_ref().unwrap(), + ) + .as_bytes(), + ) + .await + .unwrap(); + } + + // Get the cache we're supposed to use. + let expected_cache_name = { + let github_repo = env::var("GITHUB_REPOSITORY") + .expect("GITHUB_REPOSITORY environment variable is not set"); + + let url = format!("{}/project/{}", flakehub_api_server, github_repo,); + + let response = reqwest::Client::new() + .get(&url) + .header("User-Agent", USER_AGENT) + .basic_auth( + netrc_entry.login.as_ref().unwrap(), + netrc_entry.password.as_ref(), + ) + .send() + .await + .unwrap(); + + if response.status().is_success() { + #[derive(Deserialize)] + struct ProjectInfo { + organization_uuid_v7: String, + project_uuid_v7: String, + } + + let project_info = response.json::().await.unwrap(); + + let expected_cache_name = format!( + "{}:{}", + project_info.organization_uuid_v7, project_info.project_uuid_v7, + ); + + tracing::info!("Want to use cache {:?}.", expected_cache_name); + + Some(expected_cache_name) + } else { + tracing::error!( + "Failed to get project info from {}: {}", + url, + response.status() + ); + None + } + }; + + // Get a token for creating and pushing to the FlakeHub binary cache. + let (known_caches, token) = { + let url = format!("{}/token/create/cache", flakehub_api_server); + + let request = reqwest::Client::new() + .post(&url) + .header("User-Agent", USER_AGENT) + .basic_auth( + netrc_entry.login.as_ref().unwrap(), + netrc_entry.password.as_ref(), + ); + + let response = request.send().await.unwrap(); + + if !response.status().is_success() { + panic!( + "Failed to get FlakeHub binary cache creation token from {}: {}", + url, + response.status() + ); + } + + #[derive(Deserialize)] + struct Response { + token: String, + } + + let token = response.json::().await.unwrap().token; + + // Parse the JWT to get the list of caches to which we have access. + let jwt = token.strip_prefix(JWT_PREFIX).unwrap(); + let jwt_parsed: jwt::Token, _> = + jwt::Token::parse_unverified(jwt).unwrap(); + let known_caches = jwt_parsed + .claims() + .get("https://cache.flakehub.com/v1") + .unwrap() + .get("caches") + .unwrap() + .as_object() + .unwrap(); + + (known_caches.to_owned(), token) + }; + + // Use the expected cache if we have access to it, otherwise use + // the oldest cache to which we have access. + let cache_name = { + if expected_cache_name + .as_ref() + .map_or(false, |x| known_caches.get(x).is_some()) + { + expected_cache_name.unwrap().to_owned() + } else { + let mut keys: Vec<_> = known_caches.keys().collect(); + keys.sort(); + keys.first() + .expect("FlakeHub did not return any cache for the calling user.") + .to_string() + } + }; + + let cache = CacheSliceIdentifier::from_str(&cache_name).unwrap(); + + tracing::info!("Using cache {}.", cache); + + // Create the cache. + let api = ApiClient::from_server_config(ServerConfig { + endpoint: flakehub_cache_server.to_owned(), + token: Some(token.to_owned()), + }) + .unwrap(); + + let request = CreateCacheRequest { + keypair: KeypairConfig::Generate, + is_public: false, + priority: 39, + store_dir: "/nix/store".to_owned(), + upstream_cache_key_names: vec!["cache.nixos.org-1".to_owned()], // FIXME: do we want this? + }; + + if let Err(err) = api.create_cache(&cache, request).await { + match err.downcast_ref::() { + Some(ApiError::Structured(x)) if x.error == "CacheAlreadyExists" => { + tracing::info!("Cache {} already exists.", cache_name); + } + _ => { + panic!("{:?}", err); + } + } + } else { + tracing::info!("Created cache {} on {}.", cache_name, flakehub_cache_server); + } + + Ok(State { + cache, + substituter: flakehub_cache_server.to_owned(), + api, + }) +} + +pub async fn push(state: &State, store: Arc, store_paths: Vec) -> Result<()> { + let cache_config = state.api.get_cache_config(&state.cache).await.unwrap(); + + let push_config = PushConfig { + num_workers: 5, // FIXME: use number of CPUs? + force_preamble: false, + }; + + let mp = indicatif::MultiProgress::new(); + + let pusher = Pusher::new( + store.clone(), + state.api.clone(), + state.cache.to_owned(), + cache_config, + mp, + push_config, + ); + + let plan = pusher.plan(store_paths, false, false).await.unwrap(); + + for (_, path_info) in plan.store_path_map { + pusher.queue(path_info).await.unwrap(); + } + + let results = pusher.wait().await; + + for (path, res) in &results { + if let Err(err) = res { + tracing::error!( + "Upload of {} failed: {}", + store.get_full_path(path).display(), + err + ); + } + } + + tracing::info!( + "Uploaded {} paths.", + results.iter().filter(|(_path, res)| res.is_ok()).count() + ); + + Ok(()) +} diff --git a/magic-nix-cache/src/main.rs b/magic-nix-cache/src/main.rs index d998465..076cdf5 100644 --- a/magic-nix-cache/src/main.rs +++ b/magic-nix-cache/src/main.rs @@ -16,16 +16,19 @@ mod api; mod binary_cache; mod error; +mod flakehub; mod telemetry; mod util; use std::collections::HashSet; -use std::fs::{self, File}; +use std::fs::{self, create_dir_all, File, OpenOptions}; +use std::io::Write; use std::net::SocketAddr; use std::os::fd::OwnedFd; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::Arc; +use ::attic::nix_store::NixStore; use axum::{extract::Extension, routing::get, Router}; use clap::Parser; use daemonize::Daemonize; @@ -84,13 +87,36 @@ struct Args { /// This is for use in the GitHub Action only. #[arg(long, hide = true)] daemon_dir: Option, + + /// The FlakeHub API server. + #[arg(long)] + flakehub_api_server: Option, + + /// The path of the `netrc` file that contains the FlakeHub JWT token. + #[arg(long)] + flakehub_api_server_netrc: Option, + + /// The FlakeHub binary cache server. + #[arg(long)] + flakehub_cache_server: Option, + + /// The location of `nix.conf`. + #[arg(long)] + nix_conf: PathBuf, + + /// Whether to use the GHA cache. + #[arg(long)] + use_gha_cache: bool, + + /// Whether to use the FlakeHub binary cache. + #[arg(long)] + use_flakehub: bool, } /// The global server state. -#[derive(Debug)] struct StateInner { /// The GitHub Actions Cache API. - api: Api, + api: Option, /// The upstream cache. upstream: Option, @@ -111,6 +137,12 @@ struct StateInner { /// Metrics for sending to perf at shutdown metrics: telemetry::TelemetryReport, + + /// Connection to the local Nix store. + store: Arc, + + /// FlakeHub cache state. + flakehub_state: Option, } fn main() { @@ -118,17 +150,97 @@ fn main() { let args = Args::parse(); - let credentials = if let Some(credentials_file) = &args.credentials_file { - tracing::info!("Loading credentials from {:?}", credentials_file); - let bytes = fs::read(credentials_file).expect("Failed to read credentials file"); + create_dir_all(Path::new(&args.nix_conf).parent().unwrap()) + .expect("Creating parent directories of nix.conf"); - serde_json::from_slice(&bytes).expect("Failed to deserialize credentials file") + let mut nix_conf = OpenOptions::new() + .create(true) + .append(true) + .open(args.nix_conf) + .expect("Opening nix.conf"); + + let store = Arc::new(NixStore::connect().expect("Connecting to the Nix store")); + + let flakehub_state = if args.use_flakehub { + let flakehub_cache_server = args + .flakehub_cache_server + .expect("--flakehub-cache-server is required"); + let flakehub_api_server_netrc = args + .flakehub_api_server_netrc + .expect("--flakehub-api-server-netrc is required"); + + let rt = Runtime::new().unwrap(); + + match rt.block_on(async { + flakehub::init_cache( + &args + .flakehub_api_server + .expect("--flakehub-api-server is required"), + &flakehub_api_server_netrc, + &flakehub_cache_server, + ) + .await + }) { + Ok(state) => { + nix_conf + .write_all( + format!( + "extra-substituters = {}?trusted=1\nnetrc-file = {}\n", + &flakehub_cache_server, + flakehub_api_server_netrc.display() + ) + .as_bytes(), + ) + .expect("Writing to nix.conf"); + + tracing::info!("Attic cache is enabled."); + Some(state) + } + Err(err) => { + tracing::error!("Attic cache initialization failed: {}", err); + None + } + } } else { - tracing::info!("Loading credentials from environment"); - Credentials::load_from_env() - .expect("Failed to load credentials from environment (see README.md)") + tracing::info!("Attic cache is disabled."); + None }; + let api = if args.use_gha_cache { + let credentials = if let Some(credentials_file) = &args.credentials_file { + tracing::info!("Loading credentials from {:?}", credentials_file); + let bytes = fs::read(credentials_file).expect("Failed to read credentials file"); + + serde_json::from_slice(&bytes).expect("Failed to deserialize credentials file") + } else { + tracing::info!("Loading credentials from environment"); + Credentials::load_from_env() + .expect("Failed to load credentials from environment (see README.md)") + }; + + let mut api = Api::new(credentials).expect("Failed to initialize GitHub Actions Cache API"); + + if let Some(cache_version) = &args.cache_version { + api.mutate_version(cache_version.as_bytes()); + } + + nix_conf + .write_all(format!("extra-substituters = http://{}?trusted=1&compression=zstd¶llel-compression=true&priority=1\n", args.listen).as_bytes()) + .expect("Writing to nix.conf"); + + tracing::info!("GitHub Action cache is enabled."); + Some(api) + } else { + tracing::info!("GitHub Action cache is disabled."); + None + }; + + nix_conf + .write_all("fallback = true\n".as_bytes()) + .expect("Writing to nix.conf"); + + drop(nix_conf); + let diagnostic_endpoint = match args.diagnostic_endpoint.as_str() { "" => { tracing::info!("Diagnostics disabled."); @@ -137,12 +249,6 @@ fn main() { url => Some(url), }; - let mut api = Api::new(credentials).expect("Failed to initialize GitHub Actions Cache API"); - - if let Some(cache_version) = &args.cache_version { - api.mutate_version(cache_version.as_bytes()); - } - let (shutdown_sender, shutdown_receiver) = oneshot::channel(); let state = Arc::new(StateInner { @@ -153,6 +259,8 @@ fn main() { narinfo_nagative_cache: RwLock::new(HashSet::new()), self_endpoint: args.listen.to_owned(), metrics: telemetry::TelemetryReport::new(), + store, + flakehub_state, }); let app = Router::new() @@ -187,12 +295,13 @@ fn main() { .with_graceful_shutdown(async move { shutdown_receiver.await.ok(); tracing::info!("Shutting down"); + + if let Some(diagnostic_endpoint) = diagnostic_endpoint { + state.metrics.send(diagnostic_endpoint).await; + } }) .await; - if let Some(diagnostic_endpoint) = diagnostic_endpoint { - state.metrics.send(diagnostic_endpoint); - } ret.unwrap() }); } @@ -220,7 +329,9 @@ async fn dump_api_stats( request: axum::http::Request, next: axum::middleware::Next, ) -> axum::response::Response { - state.api.dump_stats(); + if let Some(api) = &state.api { + api.dump_stats(); + } next.run(request).await } diff --git a/magic-nix-cache/src/telemetry.rs b/magic-nix-cache/src/telemetry.rs index a17ba3e..a9c375e 100644 --- a/magic-nix-cache/src/telemetry.rs +++ b/magic-nix-cache/src/telemetry.rs @@ -1,7 +1,6 @@ use std::env; use std::time::SystemTime; -use is_ci; use sha2::{Digest, Sha256}; /// A telemetry report to measure the effectiveness of the Magic Nix Cache @@ -34,11 +33,11 @@ pub struct TelemetryReport { #[derive(Debug, Default, serde::Serialize)] pub struct Metric(std::sync::atomic::AtomicUsize); impl Metric { - pub fn incr(&self) -> () { + pub fn incr(&self) { self.0.fetch_add(1, std::sync::atomic::Ordering::Relaxed); } - pub fn set(&self, val: usize) -> () { + pub fn set(&self, val: usize) { self.0.store(val, std::sync::atomic::Ordering::Relaxed); } } @@ -57,7 +56,7 @@ impl TelemetryReport { } } - pub fn send(&self, endpoint: &str) { + pub async fn send(&self, endpoint: &str) { if let Some(start_time) = self.start_time { self.elapsed_seconds.set( SystemTime::now() @@ -70,12 +69,13 @@ impl TelemetryReport { } if let Ok(serialized) = serde_json::to_string_pretty(&self) { - let _ = reqwest::blocking::Client::new() + let _ = reqwest::Client::new() .post(endpoint) .body(serialized) .header("Content-Type", "application/json") .timeout(std::time::Duration::from_millis(3000)) - .send(); + .send() + .await; } } } diff --git a/magic-nix-cache/src/util.rs b/magic-nix-cache/src/util.rs index 3036be5..049303b 100644 --- a/magic-nix-cache/src/util.rs +++ b/magic-nix-cache/src/util.rs @@ -3,13 +3,15 @@ use std::collections::HashSet; use std::path::{Path, PathBuf}; +use attic::nix_store::NixStore; use tokio::{fs, process::Command}; use crate::error::{Error, Result}; /// Returns the list of store paths that are currently present. -pub async fn get_store_paths() -> Result> { - let store_dir = Path::new("/nix/store"); +pub async fn get_store_paths(store: &NixStore) -> Result> { + // FIXME: use the Nix API. + let store_dir = store.store_dir(); let mut listing = fs::read_dir(store_dir).await?; let mut paths = HashSet::new(); while let Some(entry) = listing.next_entry().await? { @@ -18,7 +20,7 @@ pub async fn get_store_paths() -> Result> { if let Some(extension) = file_name.extension() { match extension.to_str() { - None | Some("drv") | Some("lock") => { + None | Some("drv") | Some("lock") | Some("chroot") => { // Malformed or not interesting continue; } diff --git a/nix.patch b/nix.patch new file mode 100644 index 0000000..60f2a5f --- /dev/null +++ b/nix.patch @@ -0,0 +1,48 @@ +diff --git a/mk/libraries.mk b/mk/libraries.mk +index 6541775f329..5118b957608 100644 +--- a/mk/libraries.mk ++++ b/mk/libraries.mk +@@ -130,7 +130,15 @@ define build-library + + $(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS) + +- $(1)_INSTALL_PATH := $$(libdir)/$$($(1)_NAME).a ++ $(1)_INSTALL_PATH := $(DESTDIR)$$($(1)_INSTALL_DIR)/$$($(1)_NAME).a ++ ++ $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR))) ++ ++ $$($(1)_INSTALL_PATH): $$($(1)_OBJS) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ ++ +$$(trace-ld) $(LD) -Ur -o $$(_d)/$$($(1)_NAME).o $$^ ++ $$(trace-ar) $(AR) crs $$@ $$(_d)/$$($(1)_NAME).o ++ ++ install: $$($(1)_INSTALL_PATH) + + endif + +diff --git a/src/libstore/local.mk b/src/libstore/local.mk +index 8f28bec6c1d..0d41e3c2cac 100644 +--- a/src/libstore/local.mk ++++ b/src/libstore/local.mk +@@ -69,6 +69,13 @@ $(d)/build.cc: + + clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh + ++$(d)/nix-store.pc: $(d)/nix-store.pc.in ++ $(trace-gen) rm -f $@ && ./config.status --quiet --file=$@ ++ifeq ($(BUILD_SHARED_LIBS), 1) ++ sed -i 's|@LIBS_PRIVATE@||' $@ ++else ++ sed -i 's|@LIBS_PRIVATE@|Libs.private: $(libstore_LDFLAGS) $(libstore_LDFLAGS_PROPAGATED) $(foreach lib, $(libstore_LIBS), $($(lib)_LDFLAGS))|' $@ ++endif + $(eval $(call install-file-in, $(d)/nix-store.pc, $(libdir)/pkgconfig, 0644)) + + $(foreach i, $(wildcard src/libstore/builtins/*.hh), \ +diff --git a/src/libstore/nix-store.pc.in b/src/libstore/nix-store.pc.in +index 6d67b1e0380..738991d307b 100644 +--- a/src/libstore/nix-store.pc.in ++++ b/src/libstore/nix-store.pc.in +@@ -7,3 +7,4 @@ Description: Nix Package Manager + Version: @PACKAGE_VERSION@ + Libs: -L${libdir} -lnixstore -lnixutil + Cflags: -I${includedir}/nix -std=c++2a ++@LIBS_PRIVATE@ diff --git a/package.nix b/package.nix new file mode 100644 index 0000000..0f22583 --- /dev/null +++ b/package.nix @@ -0,0 +1,41 @@ +{ lib, stdenv, rustPlatform +, pkg-config +, installShellFiles +, nix +, boost +, darwin +}: + +let + ignoredPaths = [ ".github" "target" "book" ]; + +in rustPlatform.buildRustPackage rec { + pname = "magic-nix-cache"; + version = "0.1.0"; + + src = lib.cleanSourceWith { + filter = name: type: !(type == "directory" && builtins.elem (baseNameOf name) ignoredPaths); + src = lib.cleanSource ./.; + }; + + nativeBuildInputs = [ + pkg-config + installShellFiles + ]; + + buildInputs = [ + nix boost + ] ++ lib.optionals stdenv.isDarwin (with darwin.apple_sdk.frameworks; [ + SystemConfiguration + ]); + + cargoLock = { + lockFile = ./Cargo.lock; + allowBuiltinFetchGit = true; + }; + + ATTIC_DISTRIBUTOR = "attic"; + + # Recursive Nix is not stable yet + doCheck = false; +}