ed255/pod2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Implement generic serialization/deserialization (#260)

Browse source 
* complete general serialization

* bump default params temporarily

* disable recursion in great_boy_pod example
This commit is contained in:
Eduard S. 2025-06-10 12:17:30 +02:00 • committed by GitHub
parent c66506c048
commit 621f8be6b5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 392 additions and 253 deletions
Download patch file Download diff file Expand all files Collapse all files

40
src/backends/plonky2/emptypod.rs
View file

@ -1,6 +1,5 @@
use std::{collections::HashMap, sync::Mutex};
use base64::{prelude::BASE64_STANDARD, Engine};
use itertools::Itertools;
use plonky2::{
hash::hash_types::HashOutTarget,
@ -11,6 +10,7 @@ use plonky2::{
proof::ProofWithPublicInputs,
},
};
use serde::{Deserialize, Serialize};
use crate::{
backends::plonky2::{
@ -19,10 +19,11 @@ use crate::{
common::{Flattenable, StatementTarget},
mainpod::{CalculateIdGadget, PI_OFFSET_ID},
},
deserialize_proof,
error::{Error, Result},
mainpod::{self, calculate_id},
recursion::pad_circuit,
LazyLock, DEFAULT_PARAMS, STANDARD_REC_MAIN_POD_CIRCUIT_DATA,
serialize_proof, LazyLock, DEFAULT_PARAMS, STANDARD_REC_MAIN_POD_CIRCUIT_DATA,
},
middleware::{
self, AnchoredKey, DynError, Hash, Params, Pod, PodId, PodType, RecursivePod, Statement,
@ -154,6 +155,28 @@ impl EmptyPod {
})
.map_err(|e| Error::custom(format!("EmptyPod proof verification failure: {:?}", e)))
}
pub(crate) fn deserialize(
params: Params,
id: PodId,
vds_root: Hash,
data: serde_json::Value,
) -> Result<Box<dyn RecursivePod>> {
let data: Data = serde_json::from_value(data)?;
let circuit_data = &*STANDARD_REC_MAIN_POD_CIRCUIT_DATA;
let proof = deserialize_proof(&circuit_data.common, &data.proof)?;
Ok(Box::new(Self {
params,
id,
vds_root,
proof,
}))
}
}
#[derive(Serialize, Deserialize)]
struct Data {
proof: String,
}
impl Pod for EmptyPod {
@ -167,16 +190,19 @@ impl Pod for EmptyPod {
fn id(&self) -> PodId {
self.id
}
fn pod_type(&self) -> (usize, &'static str) {
(PodType::Empty as usize, "Empty")
}
fn pub_self_statements(&self) -> Vec<middleware::Statement> {
vec![type_statement()]
}
fn serialized_proof(&self) -> String {
let mut buffer = Vec::new();
use plonky2::util::serialization::Write;
buffer.write_proof(&self.proof).unwrap();
BASE64_STANDARD.encode(buffer)
fn serialize_data(&self) -> serde_json::Value {
serde_json::to_value(Data {
proof: serialize_proof(&self.proof),
})
.expect("serialization to json")
}
}

2
src/backends/plonky2/error.rs
View file

@ -43,6 +43,8 @@ pub enum Error {
Plonky2ProofFail(anyhow::Error),
#[error("base64::DecodeError: {0}")]
Base64Decode(#[from] base64::DecodeError),
#[error("serde_json::Error: {0}")]
SerdeJson(#[from] serde_json::Error),
#[error(transparent)]
Tree(#[from] crate::backends::plonky2::primitives::merkletree::error::TreeError),
#[error(transparent)]

80
src/backends/plonky2/mainpod/mod.rs
View file

@ -2,25 +2,26 @@ pub mod operation;
pub mod statement;
use std::{any::Any, iter, sync::Arc};
use base64::{prelude::BASE64_STANDARD, Engine};
use itertools::Itertools;
pub use operation::*;
use plonky2::{
hash::poseidon::PoseidonHash,
plonk::{circuit_data::CommonCircuitData, config::Hasher},
util::serialization::{Buffer, Read},
};
use serde::{Deserialize, Serialize};
pub use statement::*;
use crate::{
backends::plonky2::{
basetypes::{Proof, ProofWithPublicInputs, VerifierOnlyCircuitData, D},
circuits::mainpod::{CustomPredicateVerification, MainPodVerifyInput, MainPodVerifyTarget},
deserialize_proof,
emptypod::EmptyPod,
error::{Error, Result},
mock::emptypod::MockEmptyPod,
primitives::merkletree::MerkleClaimAndProof,
recursion::{RecursiveCircuit, RecursiveParams},
serialize_proof,
signedpod::SignedPod,
STANDARD_REC_MAIN_POD_CIRCUIT_DATA,
},
@ -241,13 +242,14 @@ fn pad_operation_args(params: &Params, args: &mut Vec<OperationArg>) {
fill_pad(args, OperationArg::None, params.max_operation_args)
}
/// Returns the statements from the given MainPodInputs, padding to the
/// respective max lengths defined at the given Params.
/// Returns the statements from the given MainPodInputs, padding to the respective max lengths
/// defined at the given Params. Also returns a copy of the dynamic-length public statements from
/// the list of statements.
pub(crate) fn layout_statements(
params: &Params,
mock: bool,
inputs: &MainPodInputs,
) -> Result<Vec<Statement>> {
) -> Result<(Vec<Statement>, Vec<Statement>)> {
let mut statements = Vec::new();
// Statement at index 0 is always None to be used for padding operation arguments in custom
@ -334,7 +336,11 @@ pub(crate) fn layout_statements(
statements.push(st);
}
Ok(statements)
let offset_public_statements = statements.len() - params.max_public_statements;
let public_statements = statements
[offset_public_statements..offset_public_statements + 1 + inputs.public_statements.len()]
.to_vec();
Ok((statements, public_statements))
}
pub(crate) fn process_private_statements_operations(
@ -469,7 +475,7 @@ impl Prover {
&custom_predicate_batches,
)?;
let statements = layout_statements(params, false, &inputs)?;
let (statements, public_statements) = layout_statements(params, false, &inputs)?;
let operations = process_private_statements_operations(
params,
&statements,
@ -479,8 +485,6 @@ impl Prover {
)?;
let operations = process_public_statements_operations(params, &statements, operations)?;
let public_statements =
statements[statements.len() - params.max_public_statements..].to_vec();
// get the id out of the public statements
let id: PodId = PodId(calculate_id(&public_statements, params));
@ -558,6 +562,12 @@ fn get_common_data(params: &Params) -> Result<CommonCircuitData<F, D>, Error> {
Ok(circuit_data.common.clone())
}
#[derive(Serialize, Deserialize)]
struct Data {
public_statements: Vec<Statement>,
proof: String,
}
impl MainPod {
fn _verify(&self) -> Result<()> {
// 2. get the id out of the public statements
@ -602,40 +612,22 @@ impl MainPod {
&self.params
}
pub(crate) fn new(
proof: Proof,
public_statements: Vec<Statement>,
pub(crate) fn deserialize(
params: Params,
id: PodId,
vds_root: Hash,
params: Params,
) -> Self {
Self {
data: serde_json::Value,
) -> Result<Box<dyn RecursivePod>> {
let data: Data = serde_json::from_value(data)?;
let common = get_common_data(&params)?;
let proof = deserialize_proof(&common, &data.proof)?;
Ok(Box::new(Self {
params,
id,
vds_root,
public_statements,
proof,
}
}
pub fn decode_proof(proof: &str, params: &Params) -> Result<Proof, Error> {
let decoded = BASE64_STANDARD.decode(proof).map_err(|e| {
Error::custom(format!(
"Failed to decode proof from base64: {}. Value: {}",
e, proof
))
})?;
let mut buf = Buffer::new(&decoded);
let common = get_common_data(params)?;
let proof = buf.read_proof(&common).map_err(|e| {
Error::custom(format!(
"Failed to read proof from buffer: {}. Value: {}",
e, proof
))
})?;
Ok(proof)
public_statements: data.public_statements,
}))
}
}
@ -650,6 +642,9 @@ impl Pod for MainPod {
fn id(&self) -> PodId {
self.id
}
fn pod_type(&self) -> (usize, &'static str) {
(PodType::Main as usize, "Main")
}
fn pub_self_statements(&self) -> Vec<middleware::Statement> {
self.public_statements
@ -659,11 +654,12 @@ impl Pod for MainPod {
.collect()
}
fn serialized_proof(&self) -> String {
let mut buffer = Vec::new();
use plonky2::util::serialization::Write;
buffer.write_proof(&self.proof).unwrap();
BASE64_STANDARD.encode(buffer)
fn serialize_data(&self) -> serde_json::Value {
serde_json::to_value(Data {
proof: serialize_proof(&self.proof),
public_statements: self.public_statements.clone(),
})
.expect("serialization to json")
}
}

15
src/backends/plonky2/mock/emptypod.rs
View file

@ -46,6 +46,14 @@ impl MockEmptyPod {
}
Ok(())
}
pub(crate) fn deserialize(
params: Params,
id: PodId,
_vds_root: Hash,
_data: serde_json::Value,
) -> Result<Box<dyn RecursivePod>> {
Ok(Box::new(Self { params, id }))
}
}
impl Pod for MockEmptyPod {
@ -58,12 +66,15 @@ impl Pod for MockEmptyPod {
fn id(&self) -> PodId {
self.id
}
fn pod_type(&self) -> (usize, &'static str) {
(PodType::MockEmpty as usize, "MockEmpty")
}
fn pub_self_statements(&self) -> Vec<Statement> {
vec![type_statement()]
}
fn serialized_proof(&self) -> String {
todo!()
fn serialize_data(&self) -> serde_json::Value {
serde_json::Value::Null
}
}

62
src/backends/plonky2/mock/mainpod.rs
View file

@ -4,7 +4,6 @@
use std::fmt;
use base64::{prelude::BASE64_STANDARD, Engine};
use serde::{Deserialize, Serialize};
use crate::{
@ -20,7 +19,7 @@ use crate::{
},
middleware::{
self, hash_str, AnchoredKey, DynError, Hash, MainPodInputs, NativePredicate, Params, Pod,
PodId, PodProver, Predicate, RecursivePod, StatementArg, KEY_TYPE, SELF,
PodId, PodProver, PodType, Predicate, RecursivePod, StatementArg, KEY_TYPE, SELF,
},
};
@ -40,6 +39,7 @@ impl PodProver for MockProver {
pub struct MockMainPod {
params: Params,
id: PodId,
vds_root: Hash,
// input_signed_pods: Vec<Box<dyn Pod>>,
// input_main_pods: Vec<Box<dyn Pod>>,
// New statements introduced by this pod
@ -51,6 +51,7 @@ pub struct MockMainPod {
// All Merkle proofs
// TODO: Use a backend-specific representation
merkle_proofs: Vec<MerkleClaimAndProof>,
// TODO: Add input pods
}
impl fmt::Display for MockMainPod {
@ -124,6 +125,14 @@ fn fmt_statement_index(
Ok(())
}
#[derive(Serialize, Deserialize)]
struct Data {
public_statements: Vec<Statement>,
operations: Vec<Operation>,
statements: Vec<Statement>,
merkle_proofs: Vec<MerkleClaimAndProof>,
}
/// Inputs are sorted as:
/// - SignedPods
/// - MainPods
@ -148,7 +157,7 @@ impl MockMainPod {
pub fn new(params: &Params, inputs: MainPodInputs) -> Result<Self> {
// TODO: Insert a new public statement of ValueOf with `key=KEY_TYPE,
// value=PodType::MockMainPod`
let statements = layout_statements(params, true, &inputs)?;
let (statements, public_statements) = layout_statements(params, true, &inputs)?;
// Extract Merkle proofs and pad.
let merkle_proofs = extract_merkle_proofs(params, inputs.operations)?;
@ -161,15 +170,13 @@ impl MockMainPod {
)?;
let operations = process_public_statements_operations(params, &statements, operations)?;
let public_statements =
statements[statements.len() - params.max_public_statements..].to_vec();
// get the id out of the public statements
let id: PodId = PodId(calculate_id(&public_statements, params));
Ok(Self {
params: params.clone(),
id,
vds_root: inputs.vds_root,
// input_signed_pods,
// input_main_pods,
// input_statements,
@ -183,13 +190,27 @@ impl MockMainPod {
// MockMainPods include some internal private state which is necessary
// for verification. In non-mock Pods, this state will not be necessary,
// as the public statements can be verified using a ZK proof.
pub(crate) fn deserialize(serialized: String) -> Result<Self> {
let proof = String::from_utf8(BASE64_STANDARD.decode(&serialized)?)
.map_err(|e| anyhow::anyhow!("Invalid base64 encoding: {}", e))?;
let pod: MockMainPod = serde_json::from_str(&proof)
.map_err(|e| anyhow::anyhow!("Failed to parse proof: {}", e))?;
Ok(pod)
pub(crate) fn deserialize(
params: Params,
id: PodId,
vds_root: Hash,
data: serde_json::Value,
) -> Result<Box<dyn RecursivePod>> {
let Data {
public_statements,
operations,
statements,
merkle_proofs,
} = serde_json::from_value(data)?;
Ok(Box::new(Self {
params,
id,
vds_root,
public_statements,
operations,
statements,
merkle_proofs,
}))
}
fn _verify(&self) -> Result<()> {
@ -289,6 +310,9 @@ impl Pod for MockMainPod {
fn id(&self) -> PodId {
self.id
}
fn pod_type(&self) -> (usize, &'static str) {
(PodType::MockMain as usize, "MockMain")
}
fn pub_self_statements(&self) -> Vec<middleware::Statement> {
self.public_statements
.iter()
@ -297,8 +321,14 @@ impl Pod for MockMainPod {
.collect()
}
fn serialized_proof(&self) -> String {
BASE64_STANDARD.encode(serde_json::to_string(self).unwrap())
fn serialize_data(&self) -> serde_json::Value {
serde_json::to_value(Data {
public_statements: self.public_statements.clone(),
operations: self.operations.clone(),
statements: self.statements.clone(),
merkle_proofs: self.merkle_proofs.clone(),
})
.expect("serialization to json")
}
}
@ -310,7 +340,7 @@ impl RecursivePod for MockMainPod {
panic!("MockMainPod can't be verified in a recursive MainPod circuit");
}
fn vds_root(&self) -> Hash {
panic!("MockMainPod can't be verified in a recursive MainPod circuit");
self.vds_root
}
}

43
src/backends/plonky2/mock/signedpod.rs
View file

@ -1,6 +1,7 @@
use std::collections::HashMap;
use itertools::Itertools;
use serde::{Deserialize, Serialize};
use crate::{
backends::plonky2::{
@ -9,8 +10,9 @@ use crate::{
},
constants::MAX_DEPTH,
middleware::{
containers::Dictionary, hash_str, AnchoredKey, DynError, Hash, Key, Params, Pod, PodId,
PodSigner, PodType, RawValue, Statement, Value, KEY_SIGNER, KEY_TYPE, SELF,
containers::Dictionary, hash_str, serialization::ordered_map, AnchoredKey, DynError, Hash,
Key, Params, Pod, PodId, PodSigner, PodType, RawValue, Statement, Value, KEY_SIGNER,
KEY_TYPE, SELF,
},
};
@ -55,17 +57,18 @@ pub struct MockSignedPod {
kvs: HashMap<Key, Value>,
}
impl MockSignedPod {
pub(crate) fn new(id: PodId, signature: String, kvs: HashMap<Key, Value>) -> Self {
Self { id, signature, kvs }
}
pub fn signature(&self) -> String {
self.signature.clone()
}
#[derive(Serialize, Deserialize)]
struct Data {
signature: String,
#[serde(serialize_with = "ordered_map")]
kvs: HashMap<Key, Value>,
}
impl MockSignedPod {
pub fn signature(&self) -> String {
self.signature.clone()
}
fn _verify(&self) -> Result<()> {
// 1. Verify id
let mt = MerkleTree::new(
@ -108,6 +111,15 @@ impl MockSignedPod {
Ok(())
}
pub(crate) fn deserialize(id: PodId, data: serde_json::Value) -> Result<Box<dyn Pod>> {
let data: Data = serde_json::from_value(data)?;
Ok(Box::new(Self {
id,
signature: data.signature,
kvs: data.kvs,
}))
}
}
impl Pod for MockSignedPod {
@ -121,6 +133,9 @@ impl Pod for MockSignedPod {
fn id(&self) -> PodId {
self.id
}
fn pod_type(&self) -> (usize, &'static str) {
(PodType::MockSigned as usize, "MockSigned")
}
fn pub_self_statements(&self) -> Vec<Statement> {
// By convention we put the KEY_TYPE first and KEY_SIGNER second
@ -136,8 +151,12 @@ impl Pod for MockSignedPod {
.collect()
}
fn serialized_proof(&self) -> String {
serde_json::to_string(&self.signature).unwrap()
fn serialize_data(&self) -> serde_json::Value {
serde_json::to_value(Data {
signature: self.signature.clone(),
kvs: self.kvs.clone(),
})
.expect("serialization to json")
}
}

37
src/backends/plonky2/mod.rs
View file

@ -10,11 +10,13 @@ pub mod signedpod;
use std::sync::LazyLock;
use base64::{prelude::BASE64_STANDARD, Engine};
pub use error::*;
use plonky2::util::serialization::{Buffer, Read};
use crate::{
backends::plonky2::{
basetypes::CircuitData,
basetypes::{CircuitData, CommonCircuitData, Proof},
circuits::mainpod::{MainPodVerifyTarget, NUM_PUBLIC_INPUTS},
recursion::RecursiveCircuit,
},
@ -37,3 +39,36 @@ pub static STANDARD_REC_MAIN_POD_CIRCUIT_DATA: LazyLock<CircuitData> = LazyLock:
.1
)
});
pub fn serialize_bytes(bytes: &[u8]) -> String {
BASE64_STANDARD.encode(bytes)
}
pub fn deserialize_bytes(data: &str) -> Result<Vec<u8>> {
BASE64_STANDARD.decode(data).map_err(|e| {
Error::custom(format!(
"Failed to decode data from base64: {}. Value: {}",
e, data
))
})
}
pub fn deserialize_proof(common: &CommonCircuitData, proof: &str) -> Result<Proof> {
let decoded = deserialize_bytes(proof)?;
let mut buf = Buffer::new(&decoded);
let proof = buf.read_proof(common).map_err(|e| {
Error::custom(format!(
"Failed to read proof from buffer: {}. Value: {}",
e, proof
))
})?;
Ok(proof)
}
pub fn serialize_proof(proof: &Proof) -> String {
let mut buffer = Vec::new();
use plonky2::util::serialization::Write;
buffer.write_proof(proof).unwrap();
serialize_bytes(&buffer)
}

59
src/backends/plonky2/signedpod.rs
View file

@ -1,12 +1,13 @@
use std::collections::HashMap;
use base64::{prelude::BASE64_STANDARD, Engine};
use itertools::Itertools;
use num_bigint::RandBigInt;
use rand::rngs::OsRng;
use serde::{Deserialize, Serialize};
use crate::{
backends::plonky2::{
deserialize_bytes,
error::{Error, Result},
primitives::{
ec::{
@ -15,6 +16,7 @@ use crate::{
},
merkletree::MerkleTree,
},
serialize_bytes,
},
constants::MAX_DEPTH,
middleware::{
@ -68,6 +70,13 @@ pub struct SignedPod {
pub dict: Dictionary,
}
#[derive(Serialize, Deserialize)]
struct Data {
signer: String,
signature: String,
kvs: Dictionary,
}
impl SignedPod {
fn _verify(&self) -> Result<()> {
// 1. Verify type
@ -107,23 +116,31 @@ impl SignedPod {
.ok_or(Error::custom("Invalid signature!".into()))
}
pub fn decode_proof(signature: &str) -> Result<(Point, Signature), Error> {
let proof_bytes = BASE64_STANDARD.decode(signature).map_err(|e| {
Error::custom(format!(
"Failed to decode proof from base64: {}. Value: {}",
e, signature
))
})?;
pub(crate) fn deserialize(id: PodId, data: serde_json::Value) -> Result<Box<dyn Pod>> {
let data: Data = serde_json::from_value(data)?;
let signer_bytes = deserialize_bytes(&data.signer)?;
let signature_bytes = deserialize_bytes(&data.signature)?;
if proof_bytes.len() != 160 {
if signer_bytes.len() != 80 {
return Err(Error::custom(
"Invalid byte encoding of signed POD proof.".to_string(),
"Invalid byte encoding of signed POD signer.".to_string(),
));
}
if signature_bytes.len() != 80 {
return Err(Error::custom(
"Invalid byte encoding of signed POD signature.".to_string(),
));
}
let signer = Point::from_bytes(&proof_bytes[..80])?;
let signature = Signature::from_bytes(&proof_bytes[80..])?;
Ok((signer, signature))
let signer = Point::from_bytes(&signer_bytes)?;
let signature = Signature::from_bytes(&signature_bytes)?;
Ok(Box::new(Self {
id,
signature,
signer,
dict: data.kvs,
}))
}
}
@ -138,6 +155,9 @@ impl Pod for SignedPod {
fn id(&self) -> PodId {
self.id
}
fn pod_type(&self) -> (usize, &'static str) {
(PodType::Signed as usize, "Signed")
}
fn pub_self_statements(&self) -> Vec<Statement> {
// By convention we put the KEY_TYPE first and KEY_SIGNER second
@ -153,10 +173,15 @@ impl Pod for SignedPod {
.collect()
}
fn serialized_proof(&self) -> String {
// Serialise signer + signature.
let proof_bytes = [self.signer.as_bytes(), self.signature.as_bytes()].concat();
BASE64_STANDARD.encode(&proof_bytes)
fn serialize_data(&self) -> serde_json::Value {
let signer = serialize_bytes(&self.signer.as_bytes());
let signature = serialize_bytes(&self.signature.as_bytes());
serde_json::to_value(Data {
signer,
signature,
kvs: self.dict.clone(),
})
.expect("serialization to json")
}
}