Merge pull request #82 from DeterminateSystems/colemickens/axum-0.7

tree: upgrade axum, hyper, etc
This commit is contained in:
Cole Mickens 2024-08-09 11:46:40 -07:00 committed by GitHub
commit 2506ee0164
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 702 additions and 757 deletions

948
Cargo.lock generated

File diff suppressed because it is too large Load diff

122
crane.nix
View file

@ -1,122 +0,0 @@
{ stdenv
, pkgs
, lib
, crane
, rust
, rust-bin
, nix-gitignore
, supportedSystems
, nix-flake
}:
let
inherit (stdenv.hostPlatform) system;
nightlyVersion = "2024-03-28";
rustNightly = (pkgs.rust-bin.nightly.${nightlyVersion}.default.override {
extensions = [ "rust-src" "rust-analyzer-preview" ];
targets = cargoTargets;
}).overrideAttrs (old: {
# Remove the propagated libiconv since we want to add our static version
depsTargetTargetPropagated = lib.filter (d: d.pname != "libiconv")
(lib.flatten (old.depsTargetTargetPropagated or [ ]));
});
# For easy cross-compilation in devShells
# We are just composing the pkgsCross.*.stdenv.cc together
crossPlatforms =
let
makeCrossPlatform = crossSystem:
let
pkgsCross =
if crossSystem == system then pkgs
else
import pkgs.path {
inherit system crossSystem;
overlays = [ nix-flake.overlays.default ];
};
rustTargetSpec = rust.toRustTargetSpec pkgsCross.pkgsStatic.stdenv.hostPlatform;
rustTargetSpecUnderscored = builtins.replaceStrings [ "-" ] [ "_" ] rustTargetSpec;
cargoLinkerEnv = lib.strings.toUpper "CARGO_TARGET_${rustTargetSpecUnderscored}_LINKER";
cargoCcEnv = "CC_${rustTargetSpecUnderscored}"; # for ring
ccbin = "${pkgsCross.stdenv.cc}/bin/${pkgsCross.stdenv.cc.targetPrefix}cc";
in
{
name = crossSystem;
value = {
inherit rustTargetSpec;
cc = pkgsCross.stdenv.cc;
pkgs = pkgsCross;
buildInputs = makeBuildInputs pkgsCross;
env = {
"${cargoLinkerEnv}" = ccbin;
"${cargoCcEnv}" = ccbin;
};
};
};
systems = lib.filter (s: s == system || lib.hasInfix "linux" s) supportedSystems
# Cross from aarch64-darwin -> x86_64-darwin doesn't work yet
# Hopefully the situation will improve with the SDK bumps
++ lib.optional (system == "x86_64-darwin") "aarch64-darwin";
in
builtins.listToAttrs (map makeCrossPlatform systems);
cargoTargets = lib.mapAttrsToList (_: p: p.rustTargetSpec) crossPlatforms;
cargoCrossEnvs = lib.foldl (acc: p: acc // p.env) { } (builtins.attrValues crossPlatforms);
makeBuildInputs = pkgs:
[ pkgs.nix
pkgs.boost # needed for clippy
]
++ lib.optionals pkgs.stdenv.isDarwin [
pkgs.darwin.apple_sdk.frameworks.Security
(pkgs.libiconv.override { enableStatic = true; enableShared = false; })
];
buildFor = system:
let
crossPlatform = crossPlatforms.${system};
inherit (crossPlatform) pkgs;
craneLib = (crane.mkLib pkgs).overrideToolchain rustNightly;
crateName = craneLib.crateNameFromCargoToml {
cargoToml = ./magic-nix-cache/Cargo.toml;
};
src = nix-gitignore.gitignoreSource [ ] ./.;
commonArgs = {
inherit (crateName) pname version;
inherit src;
nativeBuildInputs = [ pkgs.pkg-config ];
buildInputs = makeBuildInputs pkgs;
cargoExtraArgs = "--target ${crossPlatform.rustTargetSpec}";
cargoVendorDir = craneLib.vendorMultipleCargoDeps {
inherit (craneLib.findCargoFiles src) cargoConfigs;
cargoLockList = [
./Cargo.lock
"${rustNightly.passthru.availableComponents.rust-src}/lib/rustlib/src/rust/Cargo.lock"
];
};
} // crossPlatform.env;
crate = craneLib.buildPackage (commonArgs // {
cargoArtifacts = craneLib.buildDepsOnly commonArgs;
# The resulting executable must be standalone
allowedRequisites = [ ];
});
in
crate;
in
{
inherit crossPlatforms cargoTargets cargoCrossEnvs rustNightly;
magic-nix-cache = buildFor system;
}

View file

@ -19,6 +19,26 @@
"url": "https://flakehub.com/f/ipetkov/crane/0.16.3.tar.gz"
}
},
"fenix": {
"inputs": {
"nixpkgs": [
"nixpkgs"
],
"rust-analyzer-src": "rust-analyzer-src"
},
"locked": {
"lastModified": 1722493751,
"narHash": "sha256-l7/yMehbrL5d4AI8E2hKtNlT50BlUAau4EKTgPg9KcY=",
"rev": "60ab4a085ef6ee40f2ef7921ca4061084dd8cf26",
"revCount": 1955,
"type": "tarball",
"url": "https://api.flakehub.com/f/pinned/nix-community/fenix/0.1.1955%2Brev-60ab4a085ef6ee40f2ef7921ca4061084dd8cf26/01910d03-2462-7e48-b72e-439d1152bd11/source.tar.gz"
},
"original": {
"type": "tarball",
"url": "https://flakehub.com/f/nix-community/fenix/0.1.1727.tar.gz"
}
},
"flake-compat": {
"locked": {
"lastModified": 1696426674,
@ -85,24 +105,6 @@
"type": "github"
}
},
"flake-utils_2": {
"inputs": {
"systems": "systems"
},
"locked": {
"lastModified": 1705309234,
"narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"libgit2": {
"flake": false,
"locked": {
@ -138,7 +140,7 @@
},
"original": {
"type": "tarball",
"url": "https://flakehub.com/f/NixOS/nix/2.20.tar.gz"
"url": "https://flakehub.com/f/NixOS/nix/%3D2.22.1.tar.gz"
}
},
"nixpkgs": {
@ -222,45 +224,26 @@
"root": {
"inputs": {
"crane": "crane",
"fenix": "fenix",
"flake-compat": "flake-compat",
"nix": "nix",
"nixpkgs": "nixpkgs_2",
"rust-overlay": "rust-overlay"
"nixpkgs": "nixpkgs_2"
}
},
"rust-overlay": {
"inputs": {
"flake-utils": "flake-utils_2",
"nixpkgs": [
"nixpkgs"
]
},
"rust-analyzer-src": {
"flake": false,
"locked": {
"lastModified": 1716862669,
"narHash": "sha256-7oTPM9lcdwiI1cpRC313B+lHawocgpY5F07N+Rbm5Uk=",
"owner": "oxalica",
"repo": "rust-overlay",
"rev": "47b2d15658b37716393b2463a019000dbd6ce4bc",
"lastModified": 1722449213,
"narHash": "sha256-1na4m2PNH99syz2g/WQ+Hr3RfY7k4H8NBnmkr5dFDXw=",
"owner": "rust-lang",
"repo": "rust-analyzer",
"rev": "c8e41d95061543715b30880932ec3dc24c42d7ae",
"type": "github"
},
"original": {
"owner": "oxalica",
"repo": "rust-overlay",
"type": "github"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"owner": "rust-lang",
"ref": "nightly",
"repo": "rust-analyzer",
"type": "github"
}
}

194
flake.nix
View file

@ -4,8 +4,8 @@
inputs = {
nixpkgs.url = "https://flakehub.com/f/NixOS/nixpkgs/0.2311.tar.gz";
rust-overlay = {
url = "github:oxalica/rust-overlay";
fenix = {
url = "https://flakehub.com/f/nix-community/fenix/0.1.1727.tar.gz";
inputs.nixpkgs.follows = "nixpkgs";
};
@ -16,32 +16,84 @@
flake-compat.url = "https://flakehub.com/f/edolstra/flake-compat/1.0.1.tar.gz";
nix.url = "https://flakehub.com/f/NixOS/nix/2.20.tar.gz";
nix.url = "https://flakehub.com/f/NixOS/nix/=2.22.1.tar.gz";
};
outputs = { self, nixpkgs, nix, ... }@inputs:
outputs = { self, nixpkgs, fenix, crane, ... }@inputs:
let
overlays = [ inputs.rust-overlay.overlays.default nix.overlays.default ];
supportedSystems = [
"aarch64-linux"
"x86_64-linux"
"aarch64-darwin"
"x86_64-darwin"
];
forEachSupportedSystem = f: nixpkgs.lib.genAttrs supportedSystems (system: f rec {
pkgs = import nixpkgs { inherit overlays system; };
cranePkgs = pkgs.callPackage ./crane.nix {
inherit supportedSystems;
inherit (inputs) crane;
nix-flake = nix;
pkgs = import nixpkgs {
inherit system;
overlays = [
inputs.nix.overlays.default
self.overlays.default
];
};
inherit (pkgs) lib;
inherit system;
});
fenixToolchain = system: with fenix.packages.${system};
combine ([
stable.clippy
stable.rustc
stable.cargo
stable.rustfmt
stable.rust-src
] ++ nixpkgs.lib.optionals (system == "x86_64-linux") [
targets.x86_64-unknown-linux-musl.stable.rust-std
] ++ nixpkgs.lib.optionals (system == "aarch64-linux") [
targets.aarch64-unknown-linux-musl.stable.rust-std
]);
in
{
packages = forEachSupportedSystem ({ pkgs, cranePkgs, ... }: rec {
magic-nix-cache = pkgs.callPackage ./package.nix { };
#inherit (cranePkgs) magic-nix-cache;
overlays.default = final: prev:
let
toolchain = fenixToolchain final.hostPlatform.system;
craneLib = (crane.mkLib final).overrideToolchain toolchain;
crateName = craneLib.crateNameFromCargoToml {
cargoToml = ./magic-nix-cache/Cargo.toml;
};
commonArgs = {
inherit (crateName) pname version;
src = self;
nativeBuildInputs = with final; [
pkg-config
];
buildInputs = [
final.nix
final.boost
] ++ final.lib.optionals final.stdenv.isDarwin [
final.darwin.apple_sdk.frameworks.SystemConfiguration
(final.libiconv.override { enableStatic = true; enableShared = false; })
];
NIX_CFLAGS_LINK = final.lib.optionalString final.stdenv.isDarwin "-lc++abi";
};
cargoArtifacts = craneLib.buildDepsOnly commonArgs;
in
rec {
magic-nix-cache = craneLib.buildPackage (commonArgs // {
inherit cargoArtifacts;
});
default = magic-nix-cache;
};
packages = forEachSupportedSystem ({ pkgs, ... }: rec {
magic-nix-cache = pkgs.magic-nix-cache;
default = magic-nix-cache;
veryLongChain =
@ -75,12 +127,18 @@
createChain 200 startFile;
});
devShells = forEachSupportedSystem ({ pkgs, cranePkgs, lib }: {
devShells = forEachSupportedSystem ({ system, pkgs, lib }:
let
toolchain = fenixToolchain system;
in
{
default = pkgs.mkShell {
inputsFrom = [ cranePkgs.magic-nix-cache ];
packages = with pkgs; [
toolchain
nix # for linking attic
boost # for linking attic
bashInteractive
cranePkgs.rustNightly
pkg-config
cargo-bloat
@ -90,108 +148,14 @@
bacon
age
] ++ lib.optionals pkgs.stdenv.isDarwin (with pkgs.darwin.apple_sdk.frameworks; [
SystemConfiguration
]);
] ++ lib.optionals pkgs.stdenv.isDarwin [
libiconv
darwin.apple_sdk.frameworks.SystemConfiguration
];
NIX_CFLAGS_LINK = lib.optionalString pkgs.stdenv.isDarwin "-lc++abi";
RUST_SRC_PATH = "${toolchain}/lib/rustlib/src/rust/library";
};
/*
cross = pkgs.mkShell ({
inputsFrom = [ cranePkgs.magic-nix-cache ];
packages = with pkgs; [
bashInteractive
cranePkgs.rustNightly
cargo-bloat
cargo-edit
cargo-udeps
cargo-watch
age
];
shellHook =
let
crossSystems = lib.filter (s: s != pkgs.system) (builtins.attrNames cranePkgs.crossPlatforms);
in
''
# Returns compiler environment variables for a platform
#
# getTargetFlags "suffixSalt" "nativeBuildInputs" "buildInputs"
getTargetFlags() {
# Here we only call the setup-hooks of nativeBuildInputs.
#
# What's off-limits for us:
#
# - findInputs
# - activatePackage
# - Other functions in stdenv setup that depend on the private accumulator variables
(
suffixSalt="$1"
nativeBuildInputs="$2"
buildInputs="$3"
# Offsets for the nativeBuildInput (e.g., gcc)
hostOffset=-1
targetOffset=0
# In stdenv, the hooks are first accumulated before being called.
# Here we call them immediately
addEnvHooks() {
local depHostOffset="$1"
# For simplicity, we only call the hook on buildInputs
for pkg in $buildInputs; do
depTargetOffset=1
$2 $pkg
done
}
unset _PATH
unset NIX_CFLAGS_COMPILE
unset NIX_LDFLAGS
# For simplicity, we only call the setup-hooks of nativeBuildInputs
for nbi in $nativeBuildInputs; do
addToSearchPath _PATH "$nbi/bin"
if [ -e "$nbi/nix-support/setup-hook" ]; then
source "$nbi/nix-support/setup-hook"
fi
done
echo "export NIX_CFLAGS_COMPILE_''${suffixSalt}='$NIX_CFLAGS_COMPILE'"
echo "export NIX_LDFLAGS_''${suffixSalt}='$NIX_LDFLAGS'"
echo "export PATH=$PATH''${_PATH+:$_PATH}"
)
}
target_flags=$(mktemp)
${lib.concatMapStrings (system: let
crossPlatform = cranePkgs.crossPlatforms.${system};
in ''
getTargetFlags \
"${crossPlatform.cc.suffixSalt}" \
"${crossPlatform.cc} ${crossPlatform.cc.bintools}" \
"${builtins.concatStringsSep " " (crossPlatform.buildInputs ++ crossPlatform.pkgs.stdenv.defaultBuildInputs)}" >$target_flags
. $target_flags
'') crossSystems}
rm $target_flags
# Suffix flags for current system as well
export NIX_CFLAGS_COMPILE_${pkgs.stdenv.cc.suffixSalt}="$NIX_CFLAGS_COMPILE"
export NIX_LDFLAGS_${pkgs.stdenv.cc.suffixSalt}="$NIX_LDFLAGS"
unset NIX_CFLAGS_COMPILE
unset NIX_LDFLAGS
'';
} // cranePkgs.cargoCrossEnvs);
keygen = pkgs.mkShellNoCC {
packages = with pkgs; [
age
];
};
*/
});
};
}

View file

@ -11,7 +11,7 @@ derivative = { version = "2.2.0", default-features = false }
futures = { version = "0.3.28", default-features = false, features = ["alloc"] }
hex = "0.4.3"
rand = { version = "0.8.5", default-features = false, features = ["std", "std_rng"] }
reqwest = { version = "0.11.17", default-features = false, features = ["json", "rustls-tls-native-roots", "stream", "trust-dns"] }
reqwest = { version = "0.12.5", default-features = false, features = ["json", "rustls-tls-native-roots", "stream", "trust-dns"] }
serde = { version = "1.0.162", default-features = false, features = ["derive"] }
serde_json = { version = "1.0.96", default-features = false }
sha2 = { version = "0.10.6", default-features = false }

View file

@ -7,11 +7,12 @@ license = "Apache-2.0"
[dependencies]
gha-cache = { path = "../gha-cache" }
axum = { version = "0.6.18", default-features = false, features = [
axum = { version = "0.7.5", default-features = false, features = [
"json",
"tokio",
"http2",
"macros"
] }
axum-macros = "0.3.7"
clap = { version = "4.2.7", default-features = false, features = [
"std",
"derive",
@ -26,19 +27,20 @@ tracing-subscriber = { version = "0.3.17", default-features = false, features =
"tracing-log",
"smallvec",
] }
tower-http = { version = "0.4.0", features = ["trace"] }
tower-http = { version = "0.5.2", features = ["trace"] }
serde = { version = "1.0.162", features = ["derive"] }
serde_json = { version = "1.0.96", default-features = false }
thiserror = "1.0.40"
tokio-stream = { version = "0.1.14", default-features = false }
tokio-util = { version = "0.7.8", features = ["io", "compat"] }
tokio-stream = { version = "0.1.15", default-features = false }
tokio-util = { version = "0.7.11", features = ["io", "compat"] }
daemonize = "0.5.0"
is_ci = "1.1.1"
sha2 = { version = "0.10.6", default-features = false }
reqwest = { version = "0.11.17", default-features = false, features = [
reqwest = { version = "0.12.5", default-features = false, features = [
"blocking",
"rustls-tls-native-roots",
"trust-dns",
"json"
] }
netrc-rs = "0.1.2"
attic = { git = "https://github.com/DeterminateSystems/attic", branch = "fixups-for-magic-nix-cache" }
@ -51,6 +53,10 @@ uuid = { version = "1.4.0", features = ["serde", "v7", "rand", "std"] }
futures = "0.3"
async-compression = "0.4"
tracing-appender = "0.2.3"
http = "1.0"
http-body-util = "0.1"
hyper = { version = "1.0.0", features = ["full"] }
hyper-util = { version = "0.1", features = ["tokio", "server-auto", "http1"] }
[dependencies.tokio]
version = "1.28.0"

View file

@ -4,7 +4,6 @@
use attic::nix_store::StorePath;
use axum::{extract::Extension, routing::post, Json, Router};
use axum_macros::debug_handler;
use serde::{Deserialize, Serialize};
use super::State;
@ -30,7 +29,6 @@ pub fn get_router() -> Router {
}
/// Record existing paths.
#[debug_handler]
async fn workflow_start(Extension(state): Extension<State>) -> Result<Json<WorkflowStartResponse>> {
tracing::info!("Workflow started");
let reply = if let Some(original_paths) = &state.original_paths {

View file

@ -1,14 +1,12 @@
//! Binary Cache API.
use std::io;
use axum::{
extract::{BodyStream, Extension, Path},
extract::{Extension, Path},
response::Redirect,
routing::{get, put},
Router,
};
use tokio_stream::StreamExt;
use futures::StreamExt as _;
use tokio_util::io::StreamReader;
use super::State;
@ -79,7 +77,7 @@ async fn get_narinfo(
async fn put_narinfo(
Extension(state): Extension<State>,
Path(path): Path<String>,
body: BodyStream,
body: axum::body::Body,
) -> Result<()> {
let components: Vec<&str> = path.splitn(2, '.').collect();
@ -96,9 +94,13 @@ async fn put_narinfo(
let store_path_hash = components[0].to_string();
let key = format!("{}.narinfo", store_path_hash);
let allocation = gha_cache.api.allocate_file_with_random_suffix(&key).await?;
let body_stream = body.into_data_stream();
let stream = StreamReader::new(
body.map(|r| r.map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))),
body_stream
.map(|r| r.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))),
);
gha_cache.api.upload_file(allocation, stream).await?;
state.metrics.narinfos_uploaded.incr();
@ -135,7 +137,7 @@ async fn get_nar(Extension(state): Extension<State>, Path(path): Path<String>) -
async fn put_nar(
Extension(state): Extension<State>,
Path(path): Path<String>,
body: BodyStream,
body: axum::body::Body,
) -> Result<()> {
let gha_cache = state.gha_cache.as_ref().ok_or(Error::GHADisabled)?;
@ -143,9 +145,13 @@ async fn put_nar(
.api
.allocate_file_with_random_suffix(&path)
.await?;
let body_stream = body.into_data_stream();
let stream = StreamReader::new(
body.map(|r| r.map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))),
body_stream
.map(|r| r.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))),
);
gha_cache.api.upload_file(allocation, stream).await?;
state.metrics.nars_uploaded.incr();

View file

@ -430,8 +430,8 @@ async fn main_cli() -> Result<()> {
tracing::debug!("Created startup notification file at {startup_notification_file_path:?}");
}
let ret = axum::Server::bind(&args.listen)
.serve(app.into_make_service())
let listener = tokio::net::TcpListener::bind(&args.listen).await?;
let ret = axum::serve(listener, app.into_make_service())
.with_graceful_shutdown(async move {
shutdown_receiver.await.ok();
tracing::info!("Shutting down");
@ -568,10 +568,10 @@ fn init_logging() -> Result<LogGuard> {
}
#[cfg(debug_assertions)]
async fn dump_api_stats<B>(
async fn dump_api_stats(
Extension(state): Extension<State>,
request: axum::http::Request<B>,
next: axum::middleware::Next<B>,
request: axum::http::Request<axum::body::Body>,
next: axum::middleware::Next,
) -> axum::response::Response {
if let Some(gha_cache) = &state.gha_cache {
gha_cache.api.dump_stats();

View file

@ -1,58 +0,0 @@
{ lib
, stdenv
, rustPlatform
, pkg-config
, installShellFiles
, nix
, boost
, darwin
, rust-analyzer
, clippy
, rustfmt
}:
let
ignoredPaths = [ ".github" "target" "book" ];
version = (builtins.fromTOML (builtins.readFile ./magic-nix-cache/Cargo.toml)).package.version;
in
rustPlatform.buildRustPackage rec {
pname = "magic-nix-cache";
inherit version;
src = lib.cleanSourceWith {
filter = name: type: !(type == "directory" && builtins.elem (baseNameOf name) ignoredPaths);
src = lib.cleanSource ./.;
};
nativeBuildInputs = [
pkg-config
installShellFiles
rust-analyzer
clippy
rustfmt
];
buildInputs = [
nix
boost
] ++ lib.optionals stdenv.isDarwin (with darwin.apple_sdk.frameworks; [
SystemConfiguration
]);
cargoLock = {
lockFile = ./Cargo.lock;
allowBuiltinFetchGit = true;
};
ATTIC_DISTRIBUTOR = "attic";
# Hack to fix linking on macOS.
NIX_CFLAGS_LINK = lib.optionalString stdenv.isDarwin "-lc++abi";
# Recursive Nix is not stable yet
doCheck = false;
postFixup = ''
rm -f $out/nix-support/propagated-build-inputs
'';
}