From 9781bb8b6e01a73ff9d03a6f86b557cf5d126c45 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 10 Jan 2024 20:19:04 +0100 Subject: [PATCH] Improve error handling in FlakeHub cache setup Also update the token endpoint. --- Cargo.lock | 19 +++++++ magic-nix-cache/Cargo.toml | 1 + magic-nix-cache/src/error.rs | 26 +++++++++- magic-nix-cache/src/flakehub.rs | 90 +++++++++++++++------------------ magic-nix-cache/src/main.rs | 10 ++-- 5 files changed, 91 insertions(+), 55 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 428aad3..b77aa3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -125,6 +125,12 @@ dependencies = [ "syn 2.0.32", ] +[[package]] +name = "atomic" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59bdb34bc650a32731b31bd8f0829cc15d24a708ee31559e0bb34f2bc320cba" + [[package]] name = "attic" version = "0.2.0" @@ -1330,6 +1336,7 @@ dependencies = [ "tower-http", "tracing", "tracing-subscriber", + "uuid", ] [[package]] @@ -2530,6 +2537,18 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "uuid" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" +dependencies = [ + "atomic", + "getrandom", + "rand", + "serde", +] + [[package]] name = "valuable" version = "0.1.0" diff --git a/magic-nix-cache/Cargo.toml b/magic-nix-cache/Cargo.toml index a96d40a..c9a0adf 100644 --- a/magic-nix-cache/Cargo.toml +++ b/magic-nix-cache/Cargo.toml @@ -31,6 +31,7 @@ attic-client = { git = "ssh://git@github.com/DeterminateSystems/attic-priv", bra indicatif = "0.17" anyhow = "1.0.71" tempfile = "3.9" +uuid = { version = "1.4.0", features = ["serde", "v7", "rand", "std"] } [dependencies.tokio] version = "1.28.0" diff --git a/magic-nix-cache/src/error.rs b/magic-nix-cache/src/error.rs index 686cfdd..e804908 100644 --- a/magic-nix-cache/src/error.rs +++ b/magic-nix-cache/src/error.rs @@ -29,7 +29,31 @@ pub enum Error { GHADisabled, #[error("FlakeHub cache error: {0}")] - FlakeHub(anyhow::Error), + FlakeHub(#[from] anyhow::Error), + + #[error("FlakeHub HTTP error: {0}")] + FlakeHubHttp(#[from] reqwest::Error), + + #[error("Got HTTP response {1} getting FlakeHub binary cache creation token from {0}: {2}")] + CacheCreation(reqwest::Url, reqwest::StatusCode, String), + + #[error("netrc parse error: {0}")] + Netrc(netrc_rs::Error), + + #[error("Cannot find netrc credentials for {0}")] + MissingCreds(String), + + #[error("Received bad JWT")] + BadJWT, + + #[error("Received bad JWT token: {0}")] + JWTParsing(#[from] jwt::Error), + + #[error("Attic error: {0}")] + Attic(#[from] attic::AtticError), + + #[error("Bad URL")] + BadURL(reqwest::Url), } impl IntoResponse for Error { diff --git a/magic-nix-cache/src/flakehub.rs b/magic-nix-cache/src/flakehub.rs index 104b6a1..79c7e75 100644 --- a/magic-nix-cache/src/flakehub.rs +++ b/magic-nix-cache/src/flakehub.rs @@ -8,6 +8,7 @@ use attic_client::{ config::ServerConfig, push::{PushConfig, Pusher}, }; +use reqwest::Url; use serde::Deserialize; use std::env; use std::path::Path; @@ -15,20 +16,21 @@ use std::str::FromStr; use std::sync::Arc; use tokio::fs::File; use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use uuid::Uuid; const JWT_PREFIX: &str = "flakehub1_"; const USER_AGENT: &str = "magic-nix-cache"; pub struct State { - pub substituter: String, + pub substituter: Url, pub push_session: PushSession, } pub async fn init_cache( - flakehub_api_server: &str, + flakehub_api_server: &Url, flakehub_api_server_netrc: &Path, - flakehub_cache_server: &str, + flakehub_cache_server: &Url, store: Arc, ) -> Result { // Parse netrc to get the credentials for api.flakehub.com. @@ -36,7 +38,7 @@ pub async fn init_cache( let mut netrc_file = File::open(flakehub_api_server_netrc).await?; let mut netrc_contents = String::new(); netrc_file.read_to_string(&mut netrc_contents).await?; - netrc_rs::Netrc::parse(netrc_contents, false).unwrap() + netrc_rs::Netrc::parse(netrc_contents, false).map_err(Error::Netrc)? }; let netrc_entry = { @@ -44,35 +46,28 @@ pub async fn init_cache( .machines .iter() .find(|machine| { - machine.name.as_ref().unwrap() - == &reqwest::Url::parse(flakehub_api_server) - .unwrap() - .host() - .unwrap() - .to_string() + machine.name.as_ref() == flakehub_api_server.host().map(|x| x.to_string()).as_ref() }) - .unwrap() + .ok_or_else(|| Error::MissingCreds(flakehub_api_server.to_string()))? .to_owned() }; - let flakehub_cache_server_hostname = reqwest::Url::parse(flakehub_cache_server) - .unwrap() + let flakehub_cache_server_hostname = flakehub_cache_server .host() - .unwrap() + .ok_or_else(|| Error::BadURL(flakehub_cache_server.to_owned()))? .to_string(); // Append an entry for the FlakeHub cache server to netrc. if !netrc .machines .iter() - .any(|machine| machine.name.as_ref().unwrap() == &flakehub_cache_server_hostname) + .any(|machine| machine.name.as_ref() == Some(&flakehub_cache_server_hostname)) { let mut netrc_file = tokio::fs::OpenOptions::new() .create(false) .append(true) .open(flakehub_api_server_netrc) - .await - .unwrap(); + .await?; netrc_file .write_all( format!( @@ -82,8 +77,7 @@ pub async fn init_cache( ) .as_bytes(), ) - .await - .unwrap(); + .await?; } // Get the cache we're supposed to use. @@ -91,27 +85,28 @@ pub async fn init_cache( let github_repo = env::var("GITHUB_REPOSITORY") .expect("GITHUB_REPOSITORY environment variable is not set"); - let url = format!("{}/project/{}", flakehub_api_server, github_repo,); + let url = flakehub_api_server + .join(&format!("project/{}", github_repo)) + .unwrap(); let response = reqwest::Client::new() - .get(&url) + .get(url.to_owned()) .header("User-Agent", USER_AGENT) .basic_auth( netrc_entry.login.as_ref().unwrap(), netrc_entry.password.as_ref(), ) .send() - .await - .unwrap(); + .await?; if response.status().is_success() { #[derive(Deserialize)] struct ProjectInfo { - organization_uuid_v7: String, - project_uuid_v7: String, + organization_uuid_v7: Uuid, + project_uuid_v7: Uuid, } - let project_info = response.json::().await.unwrap(); + let project_info = response.json::().await?; let expected_cache_name = format!( "{}:{}", @@ -133,24 +128,24 @@ pub async fn init_cache( // Get a token for creating and pushing to the FlakeHub binary cache. let (known_caches, token) = { - let url = format!("{}/token/create/cache", flakehub_api_server); + let url = flakehub_api_server.join("cache/token").unwrap(); let request = reqwest::Client::new() - .post(&url) + .post(url.to_owned()) .header("User-Agent", USER_AGENT) .basic_auth( netrc_entry.login.as_ref().unwrap(), netrc_entry.password.as_ref(), ); - let response = request.send().await.unwrap(); + let response = request.send().await?; if !response.status().is_success() { - panic!( - "Failed to get FlakeHub binary cache creation token from {}: {}", + return Err(Error::CacheCreation( url, - response.status() - ); + response.status(), + response.text().await?, + )); } #[derive(Deserialize)] @@ -158,20 +153,20 @@ pub async fn init_cache( token: String, } - let token = response.json::().await.unwrap().token; + let token = response.json::().await?.token; // Parse the JWT to get the list of caches to which we have access. - let jwt = token.strip_prefix(JWT_PREFIX).unwrap(); + let jwt = token.strip_prefix(JWT_PREFIX).ok_or(Error::BadJWT)?; let jwt_parsed: jwt::Token, _> = - jwt::Token::parse_unverified(jwt).unwrap(); + jwt::Token::parse_unverified(jwt)?; let known_caches = jwt_parsed .claims() .get("https://cache.flakehub.com/v1") - .unwrap() + .ok_or(Error::BadJWT)? .get("caches") - .unwrap() + .ok_or(Error::BadJWT)? .as_object() - .unwrap(); + .ok_or(Error::BadJWT)?; (known_caches.to_owned(), token) }; @@ -193,16 +188,16 @@ pub async fn init_cache( } }; - let cache = CacheSliceIdentifier::from_str(&cache_name).unwrap(); + let cache = CacheSliceIdentifier::from_str(&cache_name)?; tracing::info!("Using cache {}.", cache); // Create the cache. let api = ApiClient::from_server_config(ServerConfig { - endpoint: flakehub_cache_server.to_owned(), + endpoint: flakehub_cache_server.to_string(), + //token: netrc_entry.password.as_ref().cloned(), token: Some(token.to_owned()), - }) - .unwrap(); + })?; let request = CreateCacheRequest { keypair: KeypairConfig::Generate, @@ -218,14 +213,14 @@ pub async fn init_cache( tracing::info!("Cache {} already exists.", cache_name); } _ => { - panic!("{:?}", err); + return Err(Error::FlakeHub(err)); } } } else { tracing::info!("Created cache {} on {}.", cache_name, flakehub_cache_server); } - let cache_config = api.get_cache_config(&cache).await.unwrap(); + let cache_config = api.get_cache_config(&cache).await?; let push_config = PushConfig { num_workers: 5, // FIXME: use number of CPUs? @@ -254,10 +249,7 @@ pub async fn init_cache( } pub async fn enqueue_paths(state: &State, store_paths: Vec) -> Result<()> { - state - .push_session - .queue_many(store_paths) - .map_err(Error::FlakeHub)?; + state.push_session.queue_many(store_paths)?; Ok(()) } diff --git a/magic-nix-cache/src/main.rs b/magic-nix-cache/src/main.rs index d4d6cff..c30c7bc 100644 --- a/magic-nix-cache/src/main.rs +++ b/magic-nix-cache/src/main.rs @@ -81,7 +81,7 @@ struct Args { /// The FlakeHub API server. #[arg(long)] - flakehub_api_server: Option, + flakehub_api_server: Option, /// The path of the `netrc` file that contains the FlakeHub JWT token. #[arg(long)] @@ -89,7 +89,7 @@ struct Args { /// The FlakeHub binary cache server. #[arg(long)] - flakehub_cache_server: Option, + flakehub_cache_server: Option, /// The location of `nix.conf`. #[arg(long)] @@ -186,16 +186,16 @@ async fn main_cli() { ) .expect("Writing to nix.conf"); - tracing::info!("Attic cache is enabled."); + tracing::info!("FlakeHub cache is enabled."); Some(state) } Err(err) => { - tracing::error!("Attic cache initialization failed: {}", err); + tracing::error!("FlakeHub cache initialization failed: {}", err); None } } } else { - tracing::info!("Attic cache is disabled."); + tracing::info!("FlakeHub cache is disabled."); None };