Initial persistent caching prototype

.github/workflows/test.yml

@@ -53,8 +53,11 @@ jobs:
         run: cargo fmt -- --check
       - name: Clippy
         run: cargo clippy --workspace --all-targets -- -D warnings
+      # TODO: Use something like cargo-hack for more robust feature configuration testing.
+      - name: Clippy / all-features
+        run: cargo clippy --workspace --all-targets --features persistence -- -D warnings
       - name: Test
-        run: cargo nextest run --workspace --all-targets --no-fail-fast
+        run: cargo nextest run --workspace --all-targets --features persistence --no-fail-fast
       - name: Test Manual Registration / no-default-features
         run: cargo nextest run --workspace --tests --no-fail-fast --no-default-features --features macros
       - name: Test docs

Cargo.toml

@@ -23,6 +23,7 @@ parking_lot = "0.12"
 portable-atomic = "1"
 rustc-hash = "2"
 smallvec = "1"
+thin-vec = { version = "0.2.14", features = ["serde"] }
 tracing = { version = "0.1", default-features = false, features = ["std"] }
 
 # Automatic ingredient registration.
@@ -33,18 +34,23 @@ rayon = { version = "1.10.0", optional = true }
 
 # Stuff we want Update impls for by default
 compact_str = { version = "0.9", optional = true }
-thin-vec = "0.2.14"
 
 shuttle = { version = "0.8.1", optional = true }
 
+# Persistent caching
+erased-serde = { version = "0.4.6", optional = true }
+serde = { version = "1.0.219", features = ["derive"], optional = true }
+
 [features]
 default = ["salsa_unstable", "rayon", "macros", "inventory", "accumulator"]
 inventory = ["dep:inventory"]
+persistence = ["dep:serde", "dep:erased-serde", "salsa-macros/persistence"]
 shuttle = ["dep:shuttle"]
 accumulator = ["salsa-macro-rules/accumulator"]
+macros = ["dep:salsa-macros"]
+
 # FIXME: remove `salsa_unstable` before 1.0.
 salsa_unstable = []
-macros = ["dep:salsa-macros"]
 
 # This interlocks the `salsa-macros` and `salsa` versions together
 # preventing scenarios where they could diverge in a given project
@@ -68,6 +74,7 @@ expect-test = "1.5.1"
 rustversion = "1.0"
 test-log = { version = "0.2.18", features = ["trace"] }
 trybuild = "1.0"
+serde_json = "1.0.140"
 
 [target.'cfg(all(not(target_os = "windows"), not(target_os = "openbsd"), any(target_arch = "x86_64", target_arch = "aarch64", target_arch = "powerpc64")))'.dev-dependencies]
 tikv-jemallocator = "0.6.0"

components/salsa-macro-rules/src/setup_input_struct.rs

@@ -53,6 +53,15 @@ macro_rules! setup_input_struct {
         // The function used to implement `C::heap_size`.
         heap_size_fn: $($heap_size_fn:path)?,
 
+        // If `true`, `serialize_fn` and `deserialize_fn` have been provided.
+        persist: $persist:tt,
+
+        // The path to the `serialize` function for the value's fields.
+        serialize_fn: $($serialize_fn:path)?,
+
+        // The path to the `serialize` function for the value's fields.
+        deserialize_fn: $($deserialize_fn:path)?,
+
         // Annoyingly macro-rules hygiene does not extend to items defined in the macro.
         // We have the procedural macro generate names for those items that are
         // not used elsewhere in the user's code.
@@ -93,6 +102,9 @@ macro_rules! setup_input_struct {
                 };
                 const DEBUG_NAME: &'static str = stringify!($Struct);
                 const FIELD_DEBUG_NAMES: &'static [&'static str] = &[$(stringify!($field_id)),*];
+
+                const PERSIST: bool = $persist;
+
                 type Singleton = $zalsa::macro_if! {if $is_singleton {$zalsa::input::Singleton} else {$zalsa::input::NotSingleton}};
 
                 type Struct = $Struct;
@@ -107,6 +119,32 @@ macro_rules! setup_input_struct {
                         Some($heap_size_fn(value))
                     }
                 )?
+
+                fn serialize<S: $zalsa::serde::Serializer>(
+                    fields: &Self::Fields,
+                    serializer: S,
+                ) -> Result<S::Ok, S::Error> {
+                    $zalsa::macro_if! {
+                        if $persist {
+                            $($serialize_fn(fields, serializer))?
+                        } else {
+                            panic!("attempted to serialize value not marked with `persist` attribute")
+                        }
+                    }
+                }
+
+                fn deserialize<'de, D: $zalsa::serde::Deserializer<'de>>(
+                    deserializer: D,
+                ) -> Result<Self::Fields, D::Error> {
+                    $zalsa::macro_if! {
+                        if $persist {
+                            $($deserialize_fn(deserializer))?
+                        } else {
+                            panic!("attempted to deserialize value not marked with `persist` attribute")
+                        }
+                    }
+                }
+
             }
 
             impl $Configuration {
@@ -174,6 +212,13 @@ macro_rules! setup_input_struct {
                     aux.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().into()
                 }
 
+                fn entries(
+                    zalsa: &$zalsa::Zalsa
+                ) -> impl Iterator<Item = $zalsa::DatabaseKeyIndex> + '_ {
+                    let ingredient_index = zalsa.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>();
+                    <$Configuration>::ingredient_(zalsa).entries(zalsa).map(|(key, _)| key)
+                }
+
                 #[inline]
                 fn cast(id: $zalsa::Id, type_id: $zalsa::TypeId) -> $zalsa::Option<Self> {
                     if type_id == $zalsa::TypeId::of::<$Struct>() {
@@ -194,6 +239,26 @@ macro_rules! setup_input_struct {
                 }
             }
 
+            $zalsa::macro_if! { $persist =>
+                impl $zalsa::serde::Serialize for $Struct {
+                    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+                    where
+                        S: $zalsa::serde::Serializer,
+                    {
+                        $zalsa::serde::Serialize::serialize(&$zalsa::AsId::as_id(self), serializer)
+                    }
+                }
+
+                impl<'de> $zalsa::serde::Deserialize<'de> for $Struct {
+                    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+                    where
+                        D: $zalsa::serde::Deserializer<'de>,
+                    {
+                        let id = $zalsa::Id::deserialize(deserializer)?;
+                        Ok($zalsa::FromId::from_id(id))
+                    }
+                }
+            }
             impl $Struct {
                 #[inline]
                 pub fn $new_fn<$Db>(db: &$Db, $($required_field_id: $required_field_ty),*) -> Self

components/salsa-macro-rules/src/setup_interned_struct.rs

@@ -69,6 +69,15 @@ macro_rules! setup_interned_struct {
         // The function used to implement `C::heap_size`.
         heap_size_fn: $($heap_size_fn:path)?,
 
+        // If `true`, `serialize_fn` and `deserialize_fn` have been provided.
+        persist: $persist:tt,
+
+        // The path to the `serialize` function for the value's fields.
+        serialize_fn: $($serialize_fn:path)?,
+
+        // The path to the `serialize` function for the value's fields.
+        deserialize_fn: $($deserialize_fn:path)?,
+
         // Annoyingly macro-rules hygiene does not extend to items defined in the macro.
         // We have the procedural macro generate names for those items that are
         // not used elsewhere in the user's code.
@@ -144,9 +153,12 @@ macro_rules! setup_interned_struct {
                     line: line!(),
                 };
                 const DEBUG_NAME: &'static str = stringify!($Struct);
+                const PERSIST: bool = $persist;
+
                 $(
                     const REVISIONS: ::core::num::NonZeroUsize = ::core::num::NonZeroUsize::new($revisions).unwrap();
                 )?
+
                 type Fields<'a> = $StructDataIdent<'a>;
                 type Struct<'db> = $Struct< $($db_lt_arg)? >;
 
@@ -155,11 +167,35 @@ macro_rules! setup_interned_struct {
                         Some($heap_size_fn(value))
                     }
                 )?
+
+                fn serialize<S: $zalsa::serde::Serializer>(
+                    fields: &Self::Fields<'_>,
+                    serializer: S,
+                ) -> Result<S::Ok, S::Error> {
+                    $zalsa::macro_if! {
+                        if $persist {
+                            $($serialize_fn(fields, serializer))?
+                        } else {
+                            panic!("attempted to serialize value not marked with `persist` attribute")
+                        }
+                    }
+                }
+
+                fn deserialize<'de, D: $zalsa::serde::Deserializer<'de>>(
+                    deserializer: D,
+                ) -> Result<Self::Fields<'static>, D::Error> {
+                    $zalsa::macro_if! {
+                        if $persist {
+                            $($deserialize_fn(deserializer))?
+                        } else {
+                            panic!("attempted to deserialize value not marked with `persist` attribute")
+                        }
+                    }
+                }
             }
 
             impl $Configuration {
-                pub fn ingredient(zalsa: &$zalsa::Zalsa) -> &$zalsa_struct::IngredientImpl<Self>
-                {
+                pub fn ingredient(zalsa: &$zalsa::Zalsa) -> &$zalsa_struct::IngredientImpl<Self> {
                     static CACHE: $zalsa::IngredientCache<$zalsa_struct::IngredientImpl<$Configuration>> =
                         $zalsa::IngredientCache::new();
 
@@ -204,6 +240,13 @@ macro_rules! setup_interned_struct {
                     aux.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().into()
                 }
 
+                fn entries(
+                    zalsa: &$zalsa::Zalsa
+                ) -> impl Iterator<Item = $zalsa::DatabaseKeyIndex> + '_ {
+                    let ingredient_index = zalsa.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>();
+                    <$Configuration>::ingredient(zalsa).entries(zalsa).map(|(key, _)| key)
+                }
+
                 #[inline]
                 fn cast(id: $zalsa::Id, type_id: $zalsa::TypeId) -> $zalsa::Option<Self> {
                     if type_id == $zalsa::TypeId::of::<$Struct>() {
@@ -224,6 +267,28 @@ macro_rules! setup_interned_struct {
                 }
             }
 
+            $zalsa::macro_if! { $persist =>
+                impl<$($db_lt_arg)?> $zalsa::serde::Serialize for $Struct<$($db_lt_arg)?> {
+                    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+                    where
+                        S: $zalsa::serde::Serializer,
+                    {
+                        $zalsa::serde::Serialize::serialize(&$zalsa::AsId::as_id(self), serializer)
+                    }
+                }
+
+                impl<'de, $($db_lt_arg)?> $zalsa::serde::Deserialize<'de> for $Struct<$($db_lt_arg)?> {
+                    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+                    where
+                        D: $zalsa::serde::Deserializer<'de>,
+                    {
+                        let id = $zalsa::Id::deserialize(deserializer)?;
+                        Ok($zalsa::FromId::from_id(id))
+                    }
+                }
+            }
+
+
             unsafe impl< $($db_lt_arg)? > $zalsa::Update for $Struct< $($db_lt_arg)? > {
                 unsafe fn maybe_update(old_pointer: *mut Self, new_value: Self) -> bool {
                     if unsafe { *old_pointer } != new_value {