Add a 5 min default timeout for deadlocks (#16342)

When a process is running and another calls `uv cache clean` or `uv
cache prune` we currently deadlock - sometimes until the CI timeout
(astral-sh/setup-uv#588). To avoid this, we
add a default 5 min timeout waiting for a lock. 5 min balances allowing
in-progress builds to finish, especially with larger native
dependencies, while also giving timely errors for deadlocks on (remote)
systems.

Commit 1 is a refactoring.

This branch also fixes a problem with the logging where acquired and
released resources currently mismatch:

```
DEBUG Acquired lock for `https://github.com/tqdm/tqdm`
DEBUG Using existing Git source `https://github.com/tqdm/tqdm`
DEBUG Released lock at `C:\Users\Konsti\AppData\Local\uv\cache\git-v0\locks\16bb813afef8edd2`
```

Author: konstin

Cargo.lock

@@ -178,7 +178,7 @@ tempfile = { version = "3.14.0" }
 textwrap = { version = "0.16.1" }
 thiserror = { version = "2.0.0" }
 astral-tl = { version = "0.7.9" }
-tokio = { version = "1.40.0", features = ["fs", "io-util", "macros", "process", "rt", "signal", "sync"] }
+tokio = { version = "1.40.0", features = ["fs", "io-util", "macros", "process", "rt", "signal", "sync", "time"] }
 tokio-stream = { version = "0.1.16" }
 tokio-util = { version = "0.7.12", features = ["compat", "io"] }
 toml = { version = "0.9.2", features = ["fast_hash"] }

Cargo.toml

@@ -23,7 +23,7 @@ use crate::{
     index::{AuthPolicy, Indexes},
     realm::Realm,
 };
-use crate::{Index, TextCredentialStore, TomlCredentialError};
+use crate::{Index, TextCredentialStore};
 
 /// Cached check for whether we're running in Dependabot.
 static IS_DEPENDABOT: LazyLock<bool> =
@@ -65,49 +65,55 @@ impl NetrcMode {
 
 /// Strategy for loading text-based credential files.
 enum TextStoreMode {
-    Automatic(LazyLock<Option<TextCredentialStore>>),
+    Automatic(tokio::sync::OnceCell<Option<TextCredentialStore>>),
     Enabled(TextCredentialStore),
     Disabled,
 }
 
 impl Default for TextStoreMode {
     fn default() -> Self {
-        // TODO(zanieb): Reconsider this pattern. We're just mirroring the [`NetrcMode`]
-        // implementation for now.
-        Self::Automatic(LazyLock::new(|| {
-            let path = TextCredentialStore::default_file()
-                .inspect_err(|err| {
-                    warn!("Failed to determine credentials file path: {}", err);
-                })
-                .ok()?;
-
-            match TextCredentialStore::read(&path) {
-                Ok((store, _lock)) => {
-                    debug!("Loaded credential file {}", path.display());
-                    Some(store)
-                }
-                Err(TomlCredentialError::Io(err)) if err.kind() == std::io::ErrorKind::NotFound => {
-                    debug!("No credentials file found at {}", path.display());
-                    None
-                }
-                Err(err) => {
-                    warn!(
-                        "Failed to load credentials from {}: {}",
-                        path.display(),
-                        err
-                    );
-                    None
-                }
-            }
-        }))
+        Self::Automatic(tokio::sync::OnceCell::new())
     }
 }
 
 impl TextStoreMode {
+    async fn load_default_store() -> Option<TextCredentialStore> {
+        let path = TextCredentialStore::default_file()
+            .inspect_err(|err| {
+                warn!("Failed to determine credentials file path: {}", err);
+            })
+            .ok()?;
+
+        match TextCredentialStore::read(&path).await {
+            Ok((store, _lock)) => {
+                debug!("Loaded credential file {}", path.display());
+                Some(store)
+            }
+            Err(err)
+                if err
+                    .as_io_error()
+                    .is_some_and(|err| err.kind() == std::io::ErrorKind::NotFound) =>
+            {
+                debug!("No credentials file found at {}", path.display());
+                None
+            }
+            Err(err) => {
+                warn!(
+                    "Failed to load credentials from {}: {}",
+                    path.display(),
+                    err
+                );
+                None
+            }
+        }
+    }
+
     /// Get the parsed credential store, if enabled.
-    fn get(&self) -> Option<&TextCredentialStore> {
+    async fn get(&self) -> Option<&TextCredentialStore> {
         match self {
-            Self::Automatic(lock) => lock.as_ref(),
+            // TODO(zanieb): Reconsider this pattern. We're just mirroring the [`NetrcMode`]
+            // implementation for now.
+            Self::Automatic(lock) => lock.get_or_init(Self::load_default_store).await.as_ref(),
             Self::Enabled(store) => Some(store),
             Self::Disabled => None,
         }
@@ -736,7 +742,7 @@ impl AuthMiddleware {
             Some(credentials)
 
         // Text credential store support.
-        } else if let Some(credentials) = self.text_store.get().and_then(|text_store| {
+        } else if let Some(credentials) = self.text_store.get().await.and_then(|text_store| {
             debug!("Checking text store for credentials for {url}");
             text_store.get_credentials(url, credentials.as_ref().and_then(|credentials| credentials.username())).cloned()
         }) {

crates/uv-auth/src/middleware.rs

@@ -5,7 +5,7 @@ use fs_err as fs;
 use rustc_hash::FxHashMap;
 use serde::{Deserialize, Serialize};
 use thiserror::Error;
-use uv_fs::{LockedFile, with_added_extension};
+use uv_fs::{LockedFile, LockedFileError, LockedFileMode, with_added_extension};
 use uv_preview::{Preview, PreviewFeatures};
 use uv_redacted::DisplaySafeUrl;
 
@@ -28,20 +28,24 @@ pub enum AuthBackend {
 }
 
 impl AuthBackend {
-    pub fn from_settings(preview: Preview) -> Result<Self, TomlCredentialError> {
+    pub async fn from_settings(preview: Preview) -> Result<Self, TomlCredentialError> {
         // If preview is enabled, we'll use the system-native store
         if preview.is_enabled(PreviewFeatures::NATIVE_AUTH) {
             return Ok(Self::System(KeyringProvider::native()));
         }
 
         // Otherwise, we'll use the plaintext credential store
         let path = TextCredentialStore::default_file()?;
-        match TextCredentialStore::read(&path) {
+        match TextCredentialStore::read(&path).await {
             Ok((store, lock)) => Ok(Self::TextStore(store, lock)),
-            Err(TomlCredentialError::Io(err)) if err.kind() == std::io::ErrorKind::NotFound => {
+            Err(err)
+                if err
+                    .as_io_error()
+                    .is_some_and(|err| err.kind() == std::io::ErrorKind::NotFound) =>
+            {
                 Ok(Self::TextStore(
                     TextCredentialStore::default(),
-                    TextCredentialStore::lock(&path)?,
+                    TextCredentialStore::lock(&path).await?,
                 ))
             }
             Err(err) => Err(err),
@@ -69,6 +73,8 @@ pub enum AuthScheme {
 pub enum TomlCredentialError {
     #[error(transparent)]
     Io(#[from] std::io::Error),
+    #[error(transparent)]
+    LockedFile(#[from] LockedFileError),
     #[error("Failed to parse TOML credential file: {0}")]
     ParseError(#[from] toml::de::Error),
     #[error("Failed to serialize credentials to TOML")]
@@ -83,6 +89,21 @@ pub enum TomlCredentialError {
     TokenNotUnicode(#[from] std::string::FromUtf8Error),
 }
 
+impl TomlCredentialError {
+    pub fn as_io_error(&self) -> Option<&std::io::Error> {
+        match self {
+            Self::Io(err) => Some(err),
+            Self::LockedFile(err) => err.as_io_error(),
+            Self::ParseError(_)
+            | Self::SerializeError(_)
+            | Self::BasicAuthError(_)
+            | Self::BearerAuthError(_)
+            | Self::CredentialsDirError
+            | Self::TokenNotUnicode(_) => None,
+        }
+    }
+}
+
 #[derive(Debug, Error)]
 pub enum BasicAuthError {
     #[error("`username` is required with `scheme = basic`")]
@@ -233,12 +254,12 @@ impl TextCredentialStore {
     }
 
     /// Acquire a lock on the credentials file at the given path.
-    pub fn lock(path: &Path) -> Result<LockedFile, TomlCredentialError> {
+    pub async fn lock(path: &Path) -> Result<LockedFile, TomlCredentialError> {
         if let Some(parent) = path.parent() {
             fs::create_dir_all(parent)?;
         }
         let lock = with_added_extension(path, ".lock");
-        Ok(LockedFile::acquire_blocking(lock, "credentials store")?)
+        Ok(LockedFile::acquire(lock, LockedFileMode::Exclusive, "credentials store").await?)
     }
 
     /// Read credentials from a file.
@@ -269,8 +290,8 @@ impl TextCredentialStore {
     /// Returns [`TextCredentialStore`] and a [`LockedFile`] to hold if mutating the store.
     ///
     /// If the store will not be written to following the read, the lock can be dropped.
-    pub fn read<P: AsRef<Path>>(path: P) -> Result<(Self, LockedFile), TomlCredentialError> {
-        let lock = Self::lock(path.as_ref())?;
+    pub async fn read<P: AsRef<Path>>(path: P) -> Result<(Self, LockedFile), TomlCredentialError> {
+        let lock = Self::lock(path.as_ref()).await?;
         let store = Self::from_file(path)?;
         Ok((store, lock))
     }
@@ -450,8 +471,8 @@ mod tests {
         assert!(store.get_credentials(&url, None).is_none());
     }
 
-    #[test]
-    fn test_file_operations() {
+    #[tokio::test]
+    async fn test_file_operations() {
         let mut temp_file = NamedTempFile::new().unwrap();
         writeln!(
             temp_file,
@@ -487,7 +508,7 @@ password = "pass2"
         store
             .write(
                 temp_output.path(),
-                TextCredentialStore::lock(temp_file.path()).unwrap(),
+                TextCredentialStore::lock(temp_file.path()).await.unwrap(),
             )
             .unwrap();
 

crates/uv-auth/src/store.rs

@@ -59,7 +59,10 @@ fn setup(manifest: Manifest) -> impl Fn(bool) {
         .build()
         .unwrap();
 
-    let cache = Cache::from_path("../../.cache").init().unwrap();
+    let cache = Cache::from_path("../../.cache")
+        .init_no_wait()
+        .expect("No cache contention when running benchmarks")
+        .unwrap();
     let interpreter = PythonEnvironment::from_root("../../.venv", &cache)
         .unwrap()
         .into_interpreter();

crates/uv-bench/benches/uv.rs

@@ -20,9 +20,11 @@ uv-cache = { workspace = true }
 uv-client = { workspace = true }
 uv-distribution-filename = { workspace = true }
 uv-extract = { workspace = true }
+uv-fs = { workspace = true }
 uv-pep440 = { workspace = true }
 uv-platform = { workspace = true }
 uv-redacted = { workspace = true }
+
 fs-err = { workspace = true, features = ["tokio"] }
 futures = { workspace = true }
 reqwest = { workspace = true }

crates/uv-bin-install/Cargo.toml

@@ -22,6 +22,7 @@ use uv_distribution_filename::SourceDistExtension;
 use uv_cache::{Cache, CacheBucket, CacheEntry};
 use uv_client::{BaseClient, is_transient_network_error};
 use uv_extract::{Error as ExtractError, stream};
+use uv_fs::LockedFileError;
 use uv_pep440::Version;
 use uv_platform::Platform;
 use uv_redacted::DisplaySafeUrl;
@@ -135,6 +136,9 @@ pub enum Error {
     #[error(transparent)]
     Io(#[from] std::io::Error),
 
+    #[error(transparent)]
+    LockedFile(#[from] LockedFileError),
+
     #[error("Failed to detect platform")]
     Platform(#[from] uv_platform::Error),
 

crates/uv-bin-install/src/lib.rs

@@ -36,7 +36,7 @@ use uv_distribution_types::{
     ConfigSettings, ExtraBuildRequirement, ExtraBuildRequires, IndexLocations, Requirement,
     Resolution,
 };
-use uv_fs::LockedFile;
+use uv_fs::{LockedFile, LockedFileMode};
 use uv_fs::{PythonExt, Simplified};
 use uv_normalize::PackageName;
 use uv_pep440::Version;
@@ -493,12 +493,16 @@ impl SourceBuild {
                 "uv-setuptools-{}.lock",
                 cache_digest(&canonical_source_path)
             ));
-            source_tree_lock = LockedFile::acquire(lock_path, self.source_tree.to_string_lossy())
-                .await
-                .inspect_err(|err| {
-                    warn!("Failed to acquire build lock: {err}");
-                })
-                .ok();
+            source_tree_lock = LockedFile::acquire(
+                lock_path,
+                LockedFileMode::Exclusive,
+                self.source_tree.to_string_lossy(),
+            )
+            .await
+            .inspect_err(|err| {
+                warn!("Failed to acquire build lock: {err}");
+            })
+            .ok();
         }
         Ok(source_tree_lock)
     }