Feat/disk cache (#354)

- Bump rust version to `nightly-2025-07-02` because some of the nightly features we were using have been stabilized. - Introduce feature `disk_cache` which enables caching to disk. Each time an artifact is retrieved from the cache it will be read and deserialized. On a cache miss the artifact will be created, serialized and stored to disk. - Introduce feature `mem_cache` which enables caching to memory. All cached artifacts are kept in memory after they are created. The mem cache implementation avoids cloning of artifacts by extending their lifetime to `'static`. This is `unsafe` code, but I argue that this usage is safe. - Add a `build.rs` - When the feature `disk_cache` is enabled, the `build.rs` will inject env variables to the process with the git commit information, which is used to index the cached artifacts - Replace all previous cached artifacts from `LazyStatic` methods that call the cache API - Derive `Serialize, Deserialize` for all `*Target` types so that they can be serialized for caching to disk - Add finer level of caching: now we cache the `CircuitData` and `VerifierData` independently. The reason for this is that `CircuitData` is a very big artifact which is not needed for verification. So by only accessing `VerifierData` in verification we don't pay a big overhead for reading from disk and deserializing - Add missing artifacts to the cache: like the `CircuitData` for the `MainPod` indexed by `Params` - Add helper types to serialize and deserialize `CircuitData`, `CommonData` and `VerifierData` with the set of gates and generators used in the recursive MainPod circuit - Tweak the ids of our custom gates so that they remain unique when their generic parameters change - Bugfix: several tests were using the standard `vd_set` but were using MainPod circuits with non-default parameters. This was working before because there was a bug: the MainPod circuit was reporting that the used verifier data was the standard one instead of picking the one corresponding to it's own Params. Summary of breaking changes: - One and only one of the features `mem_cache` or `disk_cache` need to be enabled. By default it's `mem_cache` - To enable the `disk_cache` you need to disable the default features like this: `--no-default-features --features=backend_plonky2,zk,disk_cache` - Removed `DEFAULT_PARAMS`, instead use `Params::default()` - Removed `STANDARD_REC_MAIN_POD_CIRCUIT_DATA`, instead use `cache_get_standard_rec_main_pod_common_circuit_data` - The library is now using `nightly-2025-07-02`. Some rust language features are unstable in previous versions.
2025-07-24 12:15:31 +02:00 · 2025-07-24 12:15:31 +02:00 · 8429cd224d
commit 8429cd224d
parent 745d654048
35 changed files with 831 additions and 207 deletions
--- a/src/backends/plonky2/serialization.rs
+++ b/src/backends/plonky2/serialization.rs
@ -0,0 +1,252 @@
+use std::ops::Deref;
+
+use plonky2::{
+    field::extension::quintic::QuinticExtension,
+    gates::{
+        arithmetic_base::ArithmeticGate, arithmetic_extension::ArithmeticExtensionGate,
+        base_sum::BaseSumGate, constant::ConstantGate, coset_interpolation::CosetInterpolationGate,
+        exponentiation::ExponentiationGate, lookup::LookupGate, lookup_table::LookupTableGate,
+        multiplication_extension::MulExtensionGate, noop::NoopGate, poseidon::PoseidonGate,
+        poseidon_mds::PoseidonMdsGate, public_input::PublicInputGate,
+        random_access::RandomAccessGate, reducing::ReducingGate,
+        reducing_extension::ReducingExtensionGate,
+    },
+    get_gate_tag_impl, impl_gate_serializer, read_gate_impl,
+    util::serialization::GateSerializer,
+};
+use serde::{de, ser, Deserialize, Serialize};
+
+use crate::backends::plonky2::{
+    basetypes::{CircuitData, CommonCircuitData, VerifierCircuitData, C, D, F},
+    circuits::{common::LtMaskGenerator, utils::DebugGenerator},
+    primitives::ec::{
+        bits::ConditionalZeroGenerator,
+        curve::PointSquareRootGenerator,
+        field::QuotientGeneratorOEF,
+        gates::{
+            curve::ECAddHomogOffset,
+            field::NNFMulSimple,
+            generic::{GateAdapter, RecursiveGateAdapter, RecursiveGenerator},
+        },
+    },
+};
+
+#[derive(Debug)]
+pub(crate) struct Pod2GateSerializer;
+impl GateSerializer<F, D> for Pod2GateSerializer {
+    impl_gate_serializer! {
+        Pod2GateSerializer,
+        ArithmeticGate,
+        ArithmeticExtensionGate<D>,
+        BaseSumGate<2>,
+        ConstantGate,
+        CosetInterpolationGate<F, D>,
+        ExponentiationGate<F, D>,
+        LookupGate,
+        LookupTableGate,
+        MulExtensionGate<D>,
+        NoopGate,
+        PoseidonMdsGate<F, D>,
+        PoseidonGate<F, D>,
+        PublicInputGate,
+        RandomAccessGate<F, D>,
+        ReducingExtensionGate<D>,
+        ReducingGate<D>,
+        // pod2 custom gates
+        GateAdapter::<NNFMulSimple<5, QuinticExtension<F>>>,
+        RecursiveGateAdapter::<D, NNFMulSimple<5, QuinticExtension<F>>>,
+        GateAdapter::<ECAddHomogOffset>,
+        RecursiveGateAdapter::<D, ECAddHomogOffset>
+    }
+}
+
+use plonky2::{
+    gadgets::{
+        arithmetic::EqualityGenerator,
+        arithmetic_extension::QuotientGeneratorExtension,
+        range_check::LowHighGenerator,
+        split_base::BaseSumGenerator,
+        split_join::{SplitGenerator, WireSplitGenerator},
+    },
+    gates::{
+        arithmetic_base::ArithmeticBaseGenerator,
+        arithmetic_extension::ArithmeticExtensionGenerator, base_sum::BaseSplitGenerator,
+        coset_interpolation::InterpolationGenerator, exponentiation::ExponentiationGenerator,
+        lookup::LookupGenerator, lookup_table::LookupTableGenerator,
+        multiplication_extension::MulExtensionGenerator, poseidon::PoseidonGenerator,
+        poseidon_mds::PoseidonMdsGenerator, random_access::RandomAccessGenerator,
+        reducing::ReducingGenerator,
+        reducing_extension::ReducingGenerator as ReducingExtensionGenerator,
+    },
+    get_generator_tag_impl, impl_generator_serializer,
+    iop::generator::{
+        ConstantGenerator, CopyGenerator, NonzeroTestGenerator, RandomValueGenerator,
+    },
+    read_generator_impl,
+    recursion::dummy_circuit::DummyProofGenerator,
+    util::serialization::WitnessGeneratorSerializer,
+};
+
+#[derive(Debug)]
+pub(crate) struct Pod2GeneratorSerializer {}
+
+// TODO: Add pod2 custom generators
+impl WitnessGeneratorSerializer<F, D> for Pod2GeneratorSerializer {
+    impl_generator_serializer! {
+        Pod2GeneratorSerializer,
+        ArithmeticBaseGenerator<F, D>,
+        ArithmeticExtensionGenerator<F, D>,
+        BaseSplitGenerator<2>,
+        BaseSumGenerator<2>,
+        ConstantGenerator<F>,
+        CopyGenerator,
+        DummyProofGenerator<F, C, D>,
+        EqualityGenerator,
+        ExponentiationGenerator<F, D>,
+        InterpolationGenerator<F, D>,
+        LookupGenerator,
+        LookupTableGenerator,
+        LowHighGenerator,
+        MulExtensionGenerator<F, D>,
+        NonzeroTestGenerator,
+        PoseidonGenerator<F, D>,
+        PoseidonMdsGenerator<D>,
+        QuotientGeneratorExtension<D>,
+        RandomAccessGenerator<F, D>,
+        RandomValueGenerator,
+        ReducingGenerator<D>,
+        ReducingExtensionGenerator<D>,
+        SplitGenerator,
+        WireSplitGenerator,
+        // pod2 custom generators
+        DebugGenerator,
+        LtMaskGenerator,
+        QuotientGeneratorOEF<5, QuinticExtension<F>>,
+        PointSquareRootGenerator,
+        ConditionalZeroGenerator<F, D>,
+        RecursiveGenerator<D, NNFMulSimple<5, QuinticExtension<F>>>,
+        RecursiveGenerator<1, NNFMulSimple<5, QuinticExtension<F>>>,
+        RecursiveGenerator<D, ECAddHomogOffset>,
+        RecursiveGenerator<1, ECAddHomogOffset>
+    }
+}
+
+/// Helper type to serialize and deserialize the pod2 `CircuitData` using serde traits.
+#[derive(Clone)]
+pub struct CircuitDataSerializer(pub(crate) CircuitData);
+
+impl Deref for CircuitDataSerializer {
+    type Target = CircuitData;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl Serialize for CircuitDataSerializer {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        let gate_serializer = Pod2GateSerializer {};
+        let generator_serializer = Pod2GeneratorSerializer {};
+        let bytes = self
+            .0
+            .to_bytes(&gate_serializer, &generator_serializer)
+            .map_err(ser::Error::custom)?;
+        serde_bytes::ByteBuf::from(bytes).serialize(serializer)
+    }
+}
+
+impl<'de> Deserialize<'de> for CircuitDataSerializer {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let bytes = <&'de serde_bytes::Bytes>::deserialize(deserializer)?;
+        let gate_serializer = Pod2GateSerializer {};
+        let generator_serializer = Pod2GeneratorSerializer {};
+        let circuit_data = CircuitData::from_bytes(bytes, &gate_serializer, &generator_serializer)
+            .map_err(de::Error::custom)?;
+        Ok(CircuitDataSerializer(circuit_data))
+    }
+}
+
+/// Helper type to serialize and deserialize the pod2 `CommonCircuitData` using serde traits.
+#[derive(Clone)]
+pub struct CommonCircuitDataSerializer(pub(crate) CommonCircuitData);
+
+impl Deref for CommonCircuitDataSerializer {
+    type Target = CommonCircuitData;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl Serialize for CommonCircuitDataSerializer {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        let gate_serializer = Pod2GateSerializer {};
+        let bytes = self
+            .0
+            .to_bytes(&gate_serializer)
+            .map_err(ser::Error::custom)?;
+        serde_bytes::ByteBuf::from(bytes).serialize(serializer)
+    }
+}
+
+impl<'de> Deserialize<'de> for CommonCircuitDataSerializer {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let bytes = <&'de serde_bytes::Bytes>::deserialize(deserializer)?;
+        let gate_serializer = Pod2GateSerializer {};
+        let circuit_data =
+            CommonCircuitData::from_bytes(bytes, &gate_serializer).map_err(de::Error::custom)?;
+        Ok(CommonCircuitDataSerializer(circuit_data))
+    }
+}
+
+/// Helper type to serialize and deserialize the pod2 `VerifierCircuitData` using serde traits.
+#[derive(Clone)]
+pub struct VerifierCircuitDataSerializer(pub(crate) VerifierCircuitData);
+
+impl Deref for VerifierCircuitDataSerializer {
+    type Target = VerifierCircuitData;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl Serialize for VerifierCircuitDataSerializer {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        let gate_serializer = Pod2GateSerializer {};
+        let bytes = self
+            .0
+            .to_bytes(&gate_serializer)
+            .map_err(ser::Error::custom)?;
+        serde_bytes::ByteBuf::from(bytes).serialize(serializer)
+    }
+}
+
+impl<'de> Deserialize<'de> for VerifierCircuitDataSerializer {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let bytes = <&'de serde_bytes::Bytes>::deserialize(deserializer)?;
+
+        let gate_serializer = Pod2GateSerializer {};
+        let circuit_data =
+            VerifierCircuitData::from_bytes(bytes, &gate_serializer).map_err(de::Error::custom)?;
+        Ok(VerifierCircuitDataSerializer(circuit_data))
+    }
+}