ed255/pod2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Serialization for Plonky2 Signed and Main PODs (#234)

Browse source 
* WIP

* WIP

* Working serialization for both Mock and Plonky2 versions of Signed and Main Pods

* Restore useful comment about serialized_proof()

* Use plonky2 serialization for signatures and proofs

* Add schema renames for Serialized SignedPod/MainPod types

* Break out utility function for generating common circuit data

* Review feedback fixes
This commit is contained in:
Rob Knight 2025-05-19 02:22:38 -07:00 • committed by GitHub
parent def0730462
commit de9b206852
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 381 additions and 115 deletions
Download patch file Download diff file Expand all files Collapse all files

74
src/backends/plonky2/mainpod/mod.rs
View file

@ -2,6 +2,7 @@ pub mod operation;
pub mod statement;
use std::{any::Any, sync::Arc};
use base64::{prelude::BASE64_STANDARD, Engine};
use itertools::Itertools;
pub use operation::*;
use plonky2::{
@ -9,10 +10,11 @@ use plonky2::{
iop::witness::PartialWitness,
plonk::{
circuit_builder::CircuitBuilder,
circuit_data::CircuitConfig,
circuit_data::{CircuitConfig, CommonCircuitData},
config::Hasher,
proof::{Proof, ProofWithPublicInputs},
},
util::serialization::{Buffer, Read},
};
pub use statement::*;
@ -219,7 +221,7 @@ fn fill_pad<T: Clone>(v: &mut Vec<T>, pad_value: T, len: usize) {
}
}
fn pad_statement(params: &Params, s: &mut Statement) {
pub fn pad_statement(params: &Params, s: &mut Statement) {
fill_pad(&mut s.1, StatementArg::None, params.max_statement_args)
}
@ -454,12 +456,14 @@ impl PodProver for Prover {
}
}
pub type MainPodProof = Proof<F, C, D>;
#[derive(Clone, Debug)]
pub struct MainPod {
params: Params,
id: PodId,
public_statements: Vec<Statement>,
proof: Proof<F, C, D>,
proof: MainPodProof,
}
/// Convert a Statement into middleware::Statement and replace references to SELF by `self_id`.
@ -481,6 +485,23 @@ pub(crate) fn normalize_statement(statement: &Statement, self_id: PodId) -> midd
.unwrap()
}
// This is a helper function to get the CommonCircuitData necessary to decode
// a serialized proof. At some point in the future, this data may be available
// as a constant or with static initialization, but in the meantime we can
// generate it on-demand.
fn get_common_data(params: &Params) -> Result<CommonCircuitData<F, D>, Error> {
let config = CircuitConfig::standard_recursion_config();
let mut builder = CircuitBuilder::<F, D>::new(config);
let _main_pod = MainPodVerifyCircuit {
params: params.clone(),
}
.eval(&mut builder)
.map_err(|e| Error::custom(format!("Failed to evaluate MainPodVerifyCircuit: {}", e)))?;
let data = builder.build::<C>();
Ok(data.common)
}
impl MainPod {
fn _verify(&self) -> Result<()> {
// 2. get the id out of the public statements
@ -505,6 +526,48 @@ impl MainPod {
})
.map_err(|e| Error::custom(format!("MainPod proof verification failure: {:?}", e)))
}
pub fn proof(&self) -> MainPodProof {
self.proof.clone()
}
pub fn params(&self) -> &Params {
&self.params
}
pub(crate) fn new(
proof: MainPodProof,
public_statements: Vec<Statement>,
id: PodId,
params: Params,
) -> Self {
Self {
params,
id,
public_statements,
proof,
}
}
pub fn decode_proof(proof: &str, params: &Params) -> Result<MainPodProof, Error> {
let decoded = BASE64_STANDARD.decode(proof).map_err(|e| {
Error::custom(format!(
"Failed to decode proof from base64: {}. Value: {}",
e, proof
))
})?;
let mut buf = Buffer::new(&decoded);
let common = get_common_data(params)?;
let proof = buf.read_proof(&common).map_err(|e| {
Error::custom(format!(
"Failed to read proof from buffer: {}. Value: {}",
e, proof
))
})?;
Ok(proof)
}
}
impl Pod for MainPod {
@ -526,7 +589,10 @@ impl Pod for MainPod {
}
fn serialized_proof(&self) -> String {
todo!()
let mut buffer = Vec::new();
use plonky2::util::serialization::Write;
buffer.write_proof(&self.proof).unwrap();
BASE64_STANDARD.encode(buffer)
}
}

4
src/backends/plonky2/mock/mainpod.rs
View file

@ -271,6 +271,10 @@ impl MockMainPod {
}
Ok(())
}
pub fn params(&self) -> &Params {
&self.params
}
}
impl Pod for MockMainPod {

6
src/backends/plonky2/mock/signedpod.rs
View file

@ -59,6 +59,10 @@ impl MockSignedPod {
pub(crate) fn new(id: PodId, signature: String, kvs: HashMap<Key, Value>) -> Self {
Self { id, signature, kvs }
}
pub fn signature(&self) -> String {
self.signature.clone()
}
}
impl MockSignedPod {
@ -131,7 +135,7 @@ impl Pod for MockSignedPod {
}
fn serialized_proof(&self) -> String {
self.signature.to_string()
serde_json::to_string(&self.signature).unwrap()
}
}

4
src/backends/plonky2/primitives/signature/mod.rs
View file

@ -21,6 +21,7 @@ use plonky2::{
pub mod circuit;
pub use circuit::*;
use serde::{Deserialize, Serialize};
use crate::{
backends::plonky2::{
@ -57,7 +58,8 @@ pub struct SecretKey(pub(crate) RawValue);
#[derive(Clone, Debug)]
pub struct PublicKey(pub RawValue);
#[derive(Clone, Debug)]
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(transparent)]
pub struct Signature(pub(crate) Proof);
/// Implements the key generation and the computation of proof-based signatures.

29
src/backends/plonky2/signedpod.rs
View file

@ -1,13 +1,15 @@
use std::collections::HashMap;
use base64::{prelude::BASE64_STANDARD, Engine};
use itertools::Itertools;
use plonky2::util::serialization::Buffer;
use crate::{
backends::plonky2::{
error::{Error, Result},
primitives::{
merkletree::MerkleTree,
signature::{PublicKey, SecretKey, Signature},
signature::{PublicKey, SecretKey, Signature, VP},
},
},
constants::MAX_DEPTH,
@ -92,6 +94,29 @@ impl SignedPod {
Ok(())
}
pub fn decode_signature(signature: &str) -> Result<Signature, Error> {
use plonky2::util::serialization::Read;
let decoded = BASE64_STANDARD.decode(signature).map_err(|e| {
Error::custom(format!(
"Failed to decode signature from base64: {}. Value: {}",
e, signature
))
})?;
let mut buf = Buffer::new(&decoded);
let proof = buf.read_proof(&VP.0.common).map_err(|e| {
Error::custom(format!(
"Failed to read signature from buffer: {}. Value: {}",
e, signature
))
})?;
let sig = Signature(proof);
Ok(sig)
}
}
impl Pod for SignedPod {
@ -122,7 +147,7 @@ impl Pod for SignedPod {
let mut buffer = Vec::new();
use plonky2::util::serialization::Write;
buffer.write_proof(&self.signature.0).unwrap();
hex::encode(buffer)
BASE64_STANDARD.encode(buffer)
}
}

16
src/frontend/mod.rs
View file

@ -5,6 +5,7 @@ use std::{collections::HashMap, convert::From, fmt};
use itertools::Itertools;
use serde::{Deserialize, Serialize};
use serialization::{SerializedMainPod, SerializedSignedPod};
use crate::middleware::{
self, check_st_tmpl, hash_str, hash_values, AnchoredKey, Hash, Key, MainPodInputs,
@ -19,15 +20,6 @@ mod serialization;
pub use custom::*;
pub use error::*;
pub use operation::*;
use serialization::*;
/// This type is just for presentation purposes.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub enum PodClass {
#[default]
Signed,
Main,
}
#[derive(Clone, Debug)]
pub struct SignedPodBuilder {
@ -68,7 +60,7 @@ impl SignedPodBuilder {
/// SignedPod is a wrapper on top of backend::SignedPod, which additionally stores the
/// string<-->hash relation of the keys.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(try_from = "SignedPodHelper", into = "SignedPodHelper")]
#[serde(from = "SerializedSignedPod", into = "SerializedSignedPod")]
pub struct SignedPod {
pub pod: Box<dyn middleware::Pod>,
// We store a copy of the key values for quick access
@ -617,16 +609,18 @@ impl MainPodBuilder {
Ok(MainPod {
pod,
params: self.params.clone(),
public_statements,
})
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(try_from = "MainPodHelper", into = "MainPodHelper")]
#[serde(try_from = "SerializedMainPod", into = "SerializedMainPod")]
pub struct MainPod {
pub pod: Box<dyn middleware::Pod>,
pub public_statements: Vec<Statement>,
pub params: Params,
}
impl fmt::Display for MainPod {

356
src/frontend/serialization.rs
View file

@ -1,109 +1,216 @@
use std::collections::HashMap;
use std::{any::Any, collections::HashMap};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use super::Error;
use crate::{
backends::plonky2::mock::{mainpod::MockMainPod, signedpod::MockSignedPod},
frontend::{Error, MainPod, SignedPod, Statement},
middleware::{containers::Dictionary, Key, PodId, Value},
backends::plonky2::{
mainpod::{pad_statement, MainPod as Plonky2MainPod, Statement as BackendStatement},
mock::{mainpod::MockMainPod, signedpod::MockSignedPod},
signedpod::SignedPod as Plonky2SignedPod,
},
frontend::{MainPod, SignedPod},
middleware::{
self, containers::Dictionary, serialization::ordered_map, AnchoredKey, Key, Params, PodId,
Statement, StatementArg, Value, SELF,
},
};
#[derive(Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "camelCase")]
#[schemars(title = "SignedPod")]
pub struct SignedPodHelper {
entries: HashMap<Key, Value>,
proof: String,
pod_class: String,
pod_type: String,
}
impl TryFrom<SignedPodHelper> for SignedPod {
type Error = Error;
fn try_from(helper: SignedPodHelper) -> Result<SignedPod, Self::Error> {
if helper.pod_class != "Signed" {
return Err(Error::custom("pod_class is not Signed"));
}
if helper.pod_type != "Mock" {
return Err(Error::custom("pod_type is not Mock"));
}
let dict = Dictionary::new(helper.entries.clone())?.clone();
let pod = MockSignedPod::new(PodId(dict.commitment()), helper.proof, dict.kvs().clone());
Ok(SignedPod {
pod: Box::new(pod),
kvs: helper.entries,
})
}
}
impl From<SignedPod> for SignedPodHelper {
fn from(pod: SignedPod) -> Self {
SignedPodHelper {
entries: pod.kvs,
proof: pod.pod.serialized_proof(),
pod_class: "Signed".to_string(),
pod_type: "Mock".to_string(),
}
}
pub enum SignedPodType {
Signed,
MockSigned,
}
#[derive(Serialize, Deserialize, JsonSchema)]
#[schemars(title = "MainPod")]
#[serde(rename_all = "camelCase")]
pub struct MainPodHelper {
#[schemars(rename = "SignedPod")]
pub struct SerializedSignedPod {
id: PodId,
#[serde(serialize_with = "ordered_map")]
entries: HashMap<Key, Value>,
proof: String,
pod_type: SignedPodType,
}
#[derive(Serialize, Deserialize, JsonSchema)]
pub enum MainPodType {
Main,
MockMain,
}
#[derive(Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "camelCase")]
#[schemars(rename = "MainPod")]
pub struct SerializedMainPod {
id: PodId,
public_statements: Vec<Statement>,
proof: String,
pod_class: String,
pod_type: String,
params: Params,
pod_type: MainPodType,
}
impl TryFrom<MainPodHelper> for MainPod {
type Error = Error; // or you can create a custom error type
fn try_from(helper: MainPodHelper) -> Result<Self, Self::Error> {
if helper.pod_class != "Main" {
return Err(Error::custom("pod_class is not Main"));
}
if helper.pod_type != "Mock" {
return Err(Error::custom("pod_type is not Mock"));
}
let pod = MockMainPod::deserialize(helper.proof)
.map_err(|e| Error::custom(format!("Failed to deserialize proof: {}", e)))?;
Ok(MainPod {
pod: Box::new(pod),
public_statements: helper.public_statements,
})
}
}
impl From<MainPod> for MainPodHelper {
fn from(pod: MainPod) -> Self {
MainPodHelper {
public_statements: pod.public_statements,
impl From<SignedPod> for SerializedSignedPod {
fn from(pod: SignedPod) -> Self {
SerializedSignedPod {
id: pod.id(),
entries: pod.kvs,
proof: pod.pod.serialized_proof(),
pod_class: "Main".to_string(),
pod_type: "Mock".to_string(),
pod_type: if (&*pod.pod as &dyn Any)
.downcast_ref::<Plonky2SignedPod>()
.is_some()
{
SignedPodType::Signed
} else if (&*pod.pod as &dyn Any)
.downcast_ref::<MockSignedPod>()
.is_some()
{
SignedPodType::MockSigned
} else {
unreachable!()
},
}
}
}
impl From<SerializedSignedPod> for SignedPod {
fn from(serialized: SerializedSignedPod) -> Self {
match serialized.pod_type {
SignedPodType::Signed => SignedPod {
pod: Box::new(Plonky2SignedPod {
id: serialized.id,
signature: Plonky2SignedPod::decode_signature(&serialized.proof).unwrap(),
dict: Dictionary::new(serialized.entries.clone()).unwrap(),
}),
kvs: serialized.entries,
},
SignedPodType::MockSigned => SignedPod {
pod: Box::new(MockSignedPod::new(
serialized.id,
serde_json::from_str(&serialized.proof).unwrap(),
serialized.entries.clone(),
)),
kvs: serialized.entries,
},
}
}
}
impl From<MainPod> for SerializedMainPod {
fn from(pod: MainPod) -> Self {
SerializedMainPod {
id: pod.id(),
proof: pod.pod.serialized_proof(),
params: pod.params.clone(),
pod_type: if (&*pod.pod as &dyn Any)
.downcast_ref::<Plonky2MainPod>()
.is_some()
{
MainPodType::Main
} else if (&*pod.pod as &dyn Any)
.downcast_ref::<MockMainPod>()
.is_some()
{
MainPodType::MockMain
} else {
unreachable!()
},
public_statements: pod.public_statements.clone(),
}
}
}
impl TryFrom<SerializedMainPod> for MainPod {
type Error = Error;
fn try_from(serialized: SerializedMainPod) -> Result<Self, Self::Error> {
match serialized.pod_type {
MainPodType::Main => Ok(MainPod {
pod: Box::new(Plonky2MainPod::new(
Plonky2MainPod::decode_proof(&serialized.proof, &serialized.params).map_err(
|e| {
Error::custom(format!(
"Failed to deserialize MainPod proof: {}. Value: {}",
e, serialized.proof
))
},
)?,
middleware_statements_to_backend(
serialized.public_statements.clone(),
&serialized.params,
serialized.id,
),
serialized.id,
serialized.params.clone(),
)),
public_statements: serialized.public_statements,
params: serialized.params,
}),
MainPodType::MockMain => Ok(MainPod {
pod: Box::new(
MockMainPod::deserialize(serialized.proof.clone()).map_err(|e| {
Error::custom(format!(
"Failed to deserialize MockMainPod: {}. Value: {}",
e, serialized.proof
))
})?,
),
public_statements: serialized.public_statements,
params: serialized.params,
}),
}
}
}
// To deserialize a backend MainPod, we need to convert the middleware
// statements to backend statements, and padding the list with None statements.
fn middleware_statements_to_backend(
mid_statements: Vec<Statement>,
params: &Params,
id: PodId,
) -> Vec<BackendStatement> {
let mut statements = Vec::new();
for i in 0..(params.max_public_statements) {
let mut st: BackendStatement = mid_statements
.get(i)
.unwrap_or(&middleware::Statement::None)
.clone()
.into();
st = BackendStatement(
st.0.clone(),
st.1.iter()
.map(|sa| match &sa {
StatementArg::Key(AnchoredKey { pod_id, key }) if *pod_id == id => {
StatementArg::Key(AnchoredKey::new(SELF, key.clone()))
}
_ => sa.clone(),
})
.collect(),
);
pad_statement(params, &mut st);
statements.push(st);
}
statements
}
#[cfg(test)]
mod tests {
use std::collections::HashSet;
use std::collections::{HashMap, HashSet};
// Pretty assertions give nicer diffs between expected and actual values
use pretty_assertions::assert_eq;
use schemars::schema_for;
use super::*;
use crate::{
backends::plonky2::mock::{mainpod::MockProver, signedpod::MockSigner},
backends::plonky2::{
mainpod::Prover,
mock::{mainpod::MockProver, signedpod::MockSigner},
primitives::signature::SecretKey,
signedpod::Signer,
},
examples::{
eth_dos_pod_builder, eth_friend_signed_pod_builder, zu_kyc_pod_builder,
zu_kyc_sign_pod_builders,
@ -112,7 +219,7 @@ mod tests {
middleware::{
self,
containers::{Array, Set},
Params, TypedValue,
Params, RawValue, TypedValue,
},
};
@ -167,8 +274,7 @@ mod tests {
}
}
fn build_signed_pod() -> Result<SignedPod> {
let mut signer = MockSigner { pk: "test".into() };
fn signed_pod_builder() -> SignedPodBuilder {
let mut builder = SignedPodBuilder::new(&Params::default());
builder.insert("name", "test");
builder.insert("age", 30);
@ -195,25 +301,46 @@ mod tests {
]))
.unwrap(),
);
let pod = builder.sign(&mut signer).unwrap();
Ok(pod)
builder
}
#[test]
fn test_signed_pod_serialization() {
let pod = build_signed_pod().unwrap();
let builder = signed_pod_builder();
let mut signer = Signer(SecretKey(RawValue::from(1)));
let pod = builder.sign(&mut signer).unwrap();
let serialized = serde_json::to_string_pretty(&pod).unwrap();
println!("serialized: {}", serialized);
let deserialized: SignedPod = serde_json::from_str(&serialized).unwrap();
println!(
"deserialized: {}",
serde_json::to_string_pretty(&deserialized).unwrap()
);
assert_eq!(pod.kvs, deserialized.kvs);
assert_eq!(pod.verify().is_ok(), deserialized.verify().is_ok());
assert_eq!(pod.id(), deserialized.id())
}
fn build_zukyc_pod() -> Result<MainPod> {
#[test]
fn test_mock_signed_pod_serialization() {
let builder = signed_pod_builder();
let mut signer = MockSigner { pk: "test".into() };
let pod = builder.sign(&mut signer).unwrap();
let serialized = serde_json::to_string_pretty(&pod).unwrap();
println!("serialized: {}", serialized);
let deserialized: SignedPod = serde_json::from_str(&serialized).unwrap();
println!(
"deserialized: {}",
serde_json::to_string_pretty(&deserialized).unwrap()
);
assert_eq!(pod.kvs, deserialized.kvs);
assert_eq!(pod.verify().is_ok(), deserialized.verify().is_ok());
assert_eq!(pod.id(), deserialized.id())
}
fn build_mock_zukyc_pod() -> Result<MainPod> {
let params = middleware::Params::default();
let (gov_id_builder, pay_stub_builder, sanction_list_builder) =
@ -238,9 +365,34 @@ mod tests {
Ok(kyc_pod)
}
fn build_plonky2_zukyc_pod() -> Result<MainPod> {
let params = middleware::Params {
// Currently the circuit uses random access that only supports vectors of length 64.
// With max_input_main_pods=3 we need random access to a vector of length 73.
max_input_main_pods: 1,
..Default::default()
};
let (gov_id_builder, pay_stub_builder, sanction_list_builder) =
zu_kyc_sign_pod_builders(&params);
let mut signer = Signer(SecretKey(RawValue::from(1)));
let gov_id_pod = gov_id_builder.sign(&mut signer)?;
let mut signer = Signer(SecretKey(RawValue::from(2)));
let pay_stub_pod = pay_stub_builder.sign(&mut signer)?;
let mut signer = Signer(SecretKey(RawValue::from(3)));
let sanction_list_pod = sanction_list_builder.sign(&mut signer)?;
let kyc_builder =
zu_kyc_pod_builder(&params, &gov_id_pod, &pay_stub_pod, &sanction_list_pod)?;
let mut prover = Prover {};
let kyc_pod = kyc_builder.prove(&mut prover, &params)?;
Ok(kyc_pod)
}
#[test]
fn test_main_pod_serialization() -> Result<()> {
let kyc_pod = build_zukyc_pod()?;
fn test_mock_main_pod_serialization() -> Result<()> {
let kyc_pod = build_mock_zukyc_pod()?;
let serialized = serde_json::to_string_pretty(&kyc_pod).unwrap();
println!("serialized: {}", serialized);
let deserialized: MainPod = serde_json::from_str(&serialized).unwrap();
@ -252,6 +404,19 @@ mod tests {
Ok(())
}
#[test]
fn test_plonky2_main_pod_serialization() -> Result<()> {
let kyc_pod = build_plonky2_zukyc_pod()?;
let serialized = serde_json::to_string_pretty(&kyc_pod).unwrap();
let deserialized: MainPod = serde_json::from_str(&serialized).unwrap();
assert_eq!(kyc_pod.public_statements, deserialized.public_statements);
assert_eq!(kyc_pod.pod.id(), deserialized.pod.id());
assert_eq!(kyc_pod.pod.verify()?, deserialized.pod.verify()?);
Ok(())
}
fn build_ethdos_pod() -> Result<MainPod> {
let params = Params {
max_input_signed_pods: 3,
@ -295,14 +460,17 @@ mod tests {
#[test]
// This tests that we can generate JSON Schemas for the MainPod and
// SignedPod types, and that we can validate real Signed and Main Pods
// against the schemas.
// SignedPod types, and that we can validate Signed and Main Pods
// against the schemas. Since both Mock and Plonky2 PODs have the same
// public interface, we can assume that the schema works for both.
fn test_schema() {
let mainpod_schema = schema_for!(MainPodHelper);
let signedpod_schema = schema_for!(SignedPodHelper);
let mainpod_schema = schema_for!(SerializedMainPod);
let signedpod_schema = schema_for!(SerializedSignedPod);
let kyc_pod = build_zukyc_pod().unwrap();
let signed_pod = build_signed_pod().unwrap();
let kyc_pod = build_mock_zukyc_pod().unwrap();
let signed_pod = signed_pod_builder()
.sign(&mut MockSigner { pk: "test".into() })
.unwrap();
let ethdos_pod = build_ethdos_pod().unwrap();
let mainpod_schema_value = serde_json::to_value(&mainpod_schema).unwrap();
let signedpod_schema_value = serde_json::to_value(&signedpod_schema).unwrap();

7
src/middleware/mod.rs
View file

@ -553,7 +553,7 @@ impl ToFields for PodId {
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub enum PodType {
None = 0,
MockSigned = 1,
@ -574,7 +574,7 @@ impl fmt::Display for PodType {
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct Params {
pub max_input_signed_pods: usize,
@ -690,6 +690,7 @@ pub trait Pod: fmt::Debug + DynClone + Any {
})
.collect()
}
// Front-end Pods keep references to middleware Pods. Most of the
// middleware data can be derived directly from front-end data, but the
// "proof" data is only created at the point of proving/signing, and
@ -698,6 +699,8 @@ pub trait Pod: fmt::Debug + DynClone + Any {
// the implementation details of the middleware, this method allows the
// middleware to provide some serialized data that can be used to
// reconstruct the proof.
// It is an important principle that this data is opaque to the front-end
// and any third-party code.
fn serialized_proof(&self) -> String;
}