send and wait for dnixd flush signal

Cargo.lock

@@ -4,13 +4,19 @@ version = 4
 
 [[package]]
 name = "addr2line"
-version = "0.24.2"
+version = "0.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1"
+checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb"
 dependencies = [
  "gimli",
 ]
 
+[[package]]
+name = "adler"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
+
 [[package]]
 name = "adler2"
 version = "2.0.0"
@@ -863,17 +869,17 @@ dependencies = [
 
 [[package]]
 name = "backtrace"
-version = "0.3.74"
+version = "0.3.71"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a"
+checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d"
 dependencies = [
  "addr2line",
+ "cc",
  "cfg-if",
  "libc",
- "miniz_oxide",
+ "miniz_oxide 0.7.4",
  "object",
  "rustc-demangle",
- "windows-targets 0.52.6",
 ]
 
 [[package]]
@@ -1182,6 +1188,33 @@ dependencies = [
  "unicode-width 0.1.14",
 ]
 
+[[package]]
+name = "color-eyre"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55146f5e46f237f7423d74111267d4597b59b0dad0ffaf7303bce9945d843ad5"
+dependencies = [
+ "backtrace",
+ "color-spantrace",
+ "eyre",
+ "indenter",
+ "once_cell",
+ "owo-colors",
+ "tracing-error",
+]
+
+[[package]]
+name = "color-spantrace"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd6be1b2a7e382e2b98b43b2adcca6bb0e465af0bdd38123873ae61eb17a72c2"
+dependencies = [
+ "once_cell",
+ "owo-colors",
+ "tracing-core",
+ "tracing-error",
+]
+
 [[package]]
 name = "colorchoice"
 version = "1.0.3"
@@ -1760,6 +1793,16 @@ dependencies = [
  "pin-project-lite",
 ]
 
+[[package]]
+name = "eyre"
+version = "0.6.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec"
+dependencies = [
+ "indenter",
+ "once_cell",
+]
+
 [[package]]
 name = "fastcdc"
 version = "3.1.0"
@@ -1811,7 +1854,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c"
 dependencies = [
  "crc32fast",
- "miniz_oxide",
+ "miniz_oxide 0.8.4",
 ]
 
 [[package]]
@@ -2011,9 +2054,9 @@ dependencies = [
 
 [[package]]
 name = "gimli"
-version = "0.31.1"
+version = "0.28.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"
+checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
 
 [[package]]
 name = "glob"
@@ -2568,6 +2611,12 @@ dependencies = [
  "icu_properties",
 ]
 
+[[package]]
+name = "indenter"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683"
+
 [[package]]
 name = "indexmap"
 version = "1.9.3"
@@ -2857,6 +2906,7 @@ dependencies = [
  "attic-server",
  "axum 0.7.9",
  "clap",
+ "color-eyre",
  "daemonize",
  "futures",
  "gha-cache",
@@ -2933,6 +2983,15 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
 
+[[package]]
+name = "miniz_oxide"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08"
+dependencies = [
+ "adler",
+]
+
 [[package]]
 name = "miniz_oxide"
 version = "0.8.4"
@@ -3084,9 +3143,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
 
 [[package]]
 name = "object"
-version = "0.36.7"
+version = "0.32.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87"
+checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441"
 dependencies = [
  "memchr",
 ]
@@ -3148,6 +3207,12 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
 
+[[package]]
+name = "owo-colors"
+version = "3.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f"
+
 [[package]]
 name = "p256"
 version = "0.11.1"

magic-nix-cache/Cargo.toml

@@ -58,6 +58,7 @@ http-body-util = "0.1"
 hyper = { version = "1.0.0", features = ["full"] }
 hyper-util = { version = "0.1", features = ["tokio", "server-auto", "http1"] }
 xdg = { version = "2.5.2" }
+color-eyre = { version = "0.6.3" }
 
 [dependencies.tokio]
 version = "1.28.0"

magic-nix-cache/src/api.rs

@@ -94,6 +94,10 @@ async fn workflow_finish(
         }
     };
 
+    // maybe here send the request to Dnixd to Flush
+    // save uuid from response
+    // then wait on receiver until we get that same uuid back
+
     if let Some(gha_cache) = &state.gha_cache {
         tracing::info!("Waiting for GitHub action cache uploads to finish");
         gha_cache.shutdown().await?;

magic-nix-cache/src/github.rs

@@ -0,0 +1,52 @@
+use serde::{Deserialize, Serialize};
+
+const GITHUB_ACTOR_TYPE_USER: &str = "User";
+const GITHUB_ACTOR_TYPE_ORGANIZATION: &str = "Organization";
+
+#[derive(Serialize, Deserialize)]
+pub struct WorkflowData {
+    event: WorkflowDataEvent,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct WorkflowDataEvent {
+    repository: WorkflowDataEventRepo,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct WorkflowDataEventRepo {
+    owner: WorkflowDataEventRepoOwner,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct WorkflowDataEventRepoOwner {
+    login: String,
+    #[serde(rename = "type")]
+    kind: String,
+}
+
+pub(crate) fn get_actions_event_data() -> color_eyre::Result<WorkflowData> {
+    let github_context = std::env::var("GITHUB_CONTEXT")?;
+    let workflow_data: WorkflowData = serde_json::from_str::<WorkflowData>(&github_context)?;
+
+    Ok(workflow_data)
+}
+
+pub(crate) fn print_unauthenticated_error() {
+    let mut msg = "::error title=FlakeHub registration required.::Unable to authenticate to FlakeHub. Individuals must register at FlakeHub.com; Organizations must create an organization at FlakeHub.com.".to_string();
+    if let Ok(workflow_data) = get_actions_event_data() {
+        let owner = workflow_data.event.repository.owner;
+        if owner.kind == GITHUB_ACTOR_TYPE_USER {
+            msg = format!(
+                "::error title=FlakeHub registration required.::Please create an account for {} on FlakeHub.com to publish flakes.",
+                &owner.login
+            );
+        } else if owner.kind == GITHUB_ACTOR_TYPE_ORGANIZATION {
+            msg = format!(
+                "::error title=FlakeHub registration required.::Please create an organization for {} on FlakeHub.com to publish flakes.",
+                &owner.login
+            );
+        }
+    };
+    println!("{}", msg);
+}

magic-nix-cache/src/main.rs

@@ -18,6 +18,7 @@ mod env;
 mod error;
 mod flakehub;
 mod gha;
+mod github;
 mod pbh;
 mod telemetry;
 mod util;
@@ -36,6 +37,7 @@ use clap::Parser;
 use serde::{Deserialize, Serialize};
 use tokio::fs::File;
 use tokio::io::AsyncWriteExt;
+use tokio::sync::mpsc::UnboundedReceiver;
 use tokio::sync::{oneshot, Mutex, RwLock};
 use tracing_subscriber::filter::EnvFilter;
 use tracing_subscriber::layer::SubscriberExt;
@@ -226,6 +228,9 @@ struct StateInner {
 
     /// The paths in the Nix store when Magic Nix Cache started, if store diffing is enabled.
     original_paths: Option<Mutex<HashSet<PathBuf>>>,
+
+    /// The receiver side of the channel we use to get flush events from dnixd back to the workflow_shutdown handler
+    dnixd_flush_receiver: Option<UnboundedReceiver<uuid::Uuid>>,
 }
 
 #[derive(Debug, Clone)]
@@ -366,8 +371,11 @@ async fn main_cli() -> Result<()> {
                 Some(state)
             }
             Err(err) => {
-                tracing::error!("FlakeHub cache initialization failed: {}. Unable to authenticate to FlakeHub. Individuals must register at FlakeHub.com; Organizations must create an organization at FlakeHub.com.", err);
+                tracing::error!(
-                println!("::error title={{FlakeHub: Unauthenticated}}::{{Unable to authenticate to FlakeHub. Individuals must register at FlakeHub.com; Organizations must create an organization at FlakeHub.com.}}");
+                    "FlakeHub: cache initialized failed: Unauthenticated: {}",
+                    err
+                );
+                github::print_unauthenticated_error();
                 None
             }
         }
@@ -418,6 +426,15 @@ async fn main_cli() -> Result<()> {
 
     let (shutdown_sender, shutdown_receiver) = oneshot::channel();
 
+    let dnixd_flush_channel = if dnixd_available == Dnixd::Available {
+        Some(tokio::sync::mpsc::unbounded_channel())
+    } else {
+        None
+    };
+    // ????
+    let dnixd_flush_sender = dnixd_flush_channel.as_ref().map(|c| c.0.clone());
+    let dnixd_flush_receiver = dnixd_flush_channel.map(|c| c.1);
+
     let original_paths = args.diff_store.then_some(Mutex::new(HashSet::new()));
     let state = Arc::new(StateInner {
         gha_cache,
@@ -429,11 +446,12 @@ async fn main_cli() -> Result<()> {
         flakehub_state: RwLock::new(flakehub_state),
         logfile: guard.logfile,
         original_paths,
+        dnixd_flush_receiver,
     });
 
     if dnixd_available == Dnixd::Available {
         tracing::info!("Subscribing to Determinate Nixd build events.");
-        crate::pbh::subscribe_uds_post_build_hook(dnixd_uds_socket_path, state.clone()).await?;
+        crate::pbh::subscribe_uds_post_build_hook(dnixd_uds_socket_path, state.clone(), dnixd_flush_sender).await?;
     } else {
         tracing::info!("Patching nix.conf to use a post-build-hook.");
         crate::pbh::setup_legacy_post_build_hook(&args.listen, &mut nix_conf).await?;

magic-nix-cache/src/pbh.rs

@@ -21,6 +21,7 @@ use crate::State;
 pub async fn subscribe_uds_post_build_hook(
     dnixd_uds_socket_path: PathBuf,
     state: State,
+    dnixd_flush_sender: Option<tokio::sync::mpsc::UnboundedSender<uuid::Uuid>>,
 ) -> Result<()> {
     tokio::spawn(async move {
         let dnixd_uds_socket_path = &dnixd_uds_socket_path;
@@ -79,6 +80,9 @@ pub async fn subscribe_uds_post_build_hook(
                     tracing::debug!("built-paths subscription: ignoring non-data frame");
                     continue;
                 };
+
+                // TODO: check for flush event of flush type - send it to the sender
+
                 let Ok(event): core::result::Result<BuiltPathResponseEventV1, _> =
                     serde_json::from_slice(event_str)
                 else {