Implement generic serialization/deserialization (#260)

* complete general serialization

* bump default params temporarily

* disable recursion in great_boy_pod example

src/backends/plonky2/emptypod.rs

@@ -1,6 +1,5 @@
 use std::{collections::HashMap, sync::Mutex};
 
-use base64::{prelude::BASE64_STANDARD, Engine};
 use itertools::Itertools;
 use plonky2::{
     hash::hash_types::HashOutTarget,
@@ -11,6 +10,7 @@ use plonky2::{
         proof::ProofWithPublicInputs,
     },
 };
+use serde::{Deserialize, Serialize};
 
 use crate::{
     backends::plonky2::{
@@ -19,10 +19,11 @@ use crate::{
             common::{Flattenable, StatementTarget},
             mainpod::{CalculateIdGadget, PI_OFFSET_ID},
         },
+        deserialize_proof,
         error::{Error, Result},
         mainpod::{self, calculate_id},
         recursion::pad_circuit,
-        LazyLock, DEFAULT_PARAMS, STANDARD_REC_MAIN_POD_CIRCUIT_DATA,
+        serialize_proof, LazyLock, DEFAULT_PARAMS, STANDARD_REC_MAIN_POD_CIRCUIT_DATA,
     },
     middleware::{
         self, AnchoredKey, DynError, Hash, Params, Pod, PodId, PodType, RecursivePod, Statement,
@@ -154,6 +155,28 @@ impl EmptyPod {
         })
         .map_err(|e| Error::custom(format!("EmptyPod proof verification failure: {:?}", e)))
     }
+
+    pub(crate) fn deserialize(
+        params: Params,
+        id: PodId,
+        vds_root: Hash,
+        data: serde_json::Value,
+    ) -> Result<Box<dyn RecursivePod>> {
+        let data: Data = serde_json::from_value(data)?;
+        let circuit_data = &*STANDARD_REC_MAIN_POD_CIRCUIT_DATA;
+        let proof = deserialize_proof(&circuit_data.common, &data.proof)?;
+        Ok(Box::new(Self {
+            params,
+            id,
+            vds_root,
+            proof,
+        }))
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+struct Data {
+    proof: String,
 }
 
 impl Pod for EmptyPod {
@@ -167,16 +190,19 @@ impl Pod for EmptyPod {
     fn id(&self) -> PodId {
         self.id
     }
+    fn pod_type(&self) -> (usize, &'static str) {
+        (PodType::Empty as usize, "Empty")
+    }
 
     fn pub_self_statements(&self) -> Vec<middleware::Statement> {
         vec![type_statement()]
     }
 
-    fn serialized_proof(&self) -> String {
+    fn serialize_data(&self) -> serde_json::Value {
-        let mut buffer = Vec::new();
+        serde_json::to_value(Data {
-        use plonky2::util::serialization::Write;
+            proof: serialize_proof(&self.proof),
-        buffer.write_proof(&self.proof).unwrap();
+        })
-        BASE64_STANDARD.encode(buffer)
+        .expect("serialization to json")
     }
 }
 

src/backends/plonky2/error.rs

@@ -43,6 +43,8 @@ pub enum Error {
     Plonky2ProofFail(anyhow::Error),
     #[error("base64::DecodeError: {0}")]
     Base64Decode(#[from] base64::DecodeError),
+    #[error("serde_json::Error: {0}")]
+    SerdeJson(#[from] serde_json::Error),
     #[error(transparent)]
     Tree(#[from] crate::backends::plonky2::primitives::merkletree::error::TreeError),
     #[error(transparent)]

src/backends/plonky2/mainpod/mod.rs

@@ -2,25 +2,26 @@ pub mod operation;
 pub mod statement;
 use std::{any::Any, iter, sync::Arc};
 
-use base64::{prelude::BASE64_STANDARD, Engine};
 use itertools::Itertools;
 pub use operation::*;
 use plonky2::{
     hash::poseidon::PoseidonHash,
     plonk::{circuit_data::CommonCircuitData, config::Hasher},
-    util::serialization::{Buffer, Read},
 };
+use serde::{Deserialize, Serialize};
 pub use statement::*;
 
 use crate::{
     backends::plonky2::{
         basetypes::{Proof, ProofWithPublicInputs, VerifierOnlyCircuitData, D},
         circuits::mainpod::{CustomPredicateVerification, MainPodVerifyInput, MainPodVerifyTarget},
+        deserialize_proof,
         emptypod::EmptyPod,
         error::{Error, Result},
         mock::emptypod::MockEmptyPod,
         primitives::merkletree::MerkleClaimAndProof,
         recursion::{RecursiveCircuit, RecursiveParams},
+        serialize_proof,
         signedpod::SignedPod,
         STANDARD_REC_MAIN_POD_CIRCUIT_DATA,
     },
@@ -241,13 +242,14 @@ fn pad_operation_args(params: &Params, args: &mut Vec<OperationArg>) {
     fill_pad(args, OperationArg::None, params.max_operation_args)
 }
 
-/// Returns the statements from the given MainPodInputs, padding to the
+/// Returns the statements from the given MainPodInputs, padding to the respective max lengths
-/// respective max lengths defined at the given Params.
+/// defined at the given Params.  Also returns a copy of the dynamic-length public statements from
+/// the list of statements.
 pub(crate) fn layout_statements(
     params: &Params,
     mock: bool,
     inputs: &MainPodInputs,
-) -> Result<Vec<Statement>> {
+) -> Result<(Vec<Statement>, Vec<Statement>)> {
     let mut statements = Vec::new();
 
     // Statement at index 0 is always None to be used for padding operation arguments in custom
@@ -334,7 +336,11 @@ pub(crate) fn layout_statements(
         statements.push(st);
     }
 
-    Ok(statements)
+    let offset_public_statements = statements.len() - params.max_public_statements;
+    let public_statements = statements
+        [offset_public_statements..offset_public_statements + 1 + inputs.public_statements.len()]
+        .to_vec();
+    Ok((statements, public_statements))
 }
 
 pub(crate) fn process_private_statements_operations(
@@ -469,7 +475,7 @@ impl Prover {
             &custom_predicate_batches,
         )?;
 
-        let statements = layout_statements(params, false, &inputs)?;
+        let (statements, public_statements) = layout_statements(params, false, &inputs)?;
         let operations = process_private_statements_operations(
             params,
             &statements,
@@ -479,8 +485,6 @@ impl Prover {
         )?;
         let operations = process_public_statements_operations(params, &statements, operations)?;
 
-        let public_statements =
-            statements[statements.len() - params.max_public_statements..].to_vec();
         // get the id out of the public statements
         let id: PodId = PodId(calculate_id(&public_statements, params));
 
@@ -558,6 +562,12 @@ fn get_common_data(params: &Params) -> Result<CommonCircuitData<F, D>, Error> {
     Ok(circuit_data.common.clone())
 }
 
+#[derive(Serialize, Deserialize)]
+struct Data {
+    public_statements: Vec<Statement>,
+    proof: String,
+}
+
 impl MainPod {
     fn _verify(&self) -> Result<()> {
         // 2. get the id out of the public statements
@@ -602,40 +612,22 @@ impl MainPod {
         &self.params
     }
 
-    pub(crate) fn new(
+    pub(crate) fn deserialize(
-        proof: Proof,
+        params: Params,
-        public_statements: Vec<Statement>,
         id: PodId,
         vds_root: Hash,
-        params: Params,
+        data: serde_json::Value,
-    ) -> Self {
+    ) -> Result<Box<dyn RecursivePod>> {
-        Self {
+        let data: Data = serde_json::from_value(data)?;
+        let common = get_common_data(&params)?;
+        let proof = deserialize_proof(&common, &data.proof)?;
+        Ok(Box::new(Self {
             params,
             id,
             vds_root,
-            public_statements,
             proof,
-        }
+            public_statements: data.public_statements,
-    }
+        }))
-
-    pub fn decode_proof(proof: &str, params: &Params) -> Result<Proof, Error> {
-        let decoded = BASE64_STANDARD.decode(proof).map_err(|e| {
-            Error::custom(format!(
-                "Failed to decode proof from base64: {}. Value: {}",
-                e, proof
-            ))
-        })?;
-        let mut buf = Buffer::new(&decoded);
-        let common = get_common_data(params)?;
-
-        let proof = buf.read_proof(&common).map_err(|e| {
-            Error::custom(format!(
-                "Failed to read proof from buffer: {}. Value: {}",
-                e, proof
-            ))
-        })?;
-
-        Ok(proof)
     }
 }
 
@@ -650,6 +642,9 @@ impl Pod for MainPod {
     fn id(&self) -> PodId {
         self.id
     }
+    fn pod_type(&self) -> (usize, &'static str) {
+        (PodType::Main as usize, "Main")
+    }
 
     fn pub_self_statements(&self) -> Vec<middleware::Statement> {
         self.public_statements
@@ -659,11 +654,12 @@ impl Pod for MainPod {
             .collect()
     }
 
-    fn serialized_proof(&self) -> String {
+    fn serialize_data(&self) -> serde_json::Value {
-        let mut buffer = Vec::new();
+        serde_json::to_value(Data {
-        use plonky2::util::serialization::Write;
+            proof: serialize_proof(&self.proof),
-        buffer.write_proof(&self.proof).unwrap();
+            public_statements: self.public_statements.clone(),
-        BASE64_STANDARD.encode(buffer)
+        })
+        .expect("serialization to json")
     }
 }
 

src/backends/plonky2/mock/emptypod.rs

@@ -46,6 +46,14 @@ impl MockEmptyPod {
         }
         Ok(())
     }
+    pub(crate) fn deserialize(
+        params: Params,
+        id: PodId,
+        _vds_root: Hash,
+        _data: serde_json::Value,
+    ) -> Result<Box<dyn RecursivePod>> {
+        Ok(Box::new(Self { params, id }))
+    }
 }
 
 impl Pod for MockEmptyPod {
@@ -58,12 +66,15 @@ impl Pod for MockEmptyPod {
     fn id(&self) -> PodId {
         self.id
     }
+    fn pod_type(&self) -> (usize, &'static str) {
+        (PodType::MockEmpty as usize, "MockEmpty")
+    }
     fn pub_self_statements(&self) -> Vec<Statement> {
         vec![type_statement()]
     }
 
-    fn serialized_proof(&self) -> String {
+    fn serialize_data(&self) -> serde_json::Value {
-        todo!()
+        serde_json::Value::Null
     }
 }
 

src/backends/plonky2/mock/mainpod.rs

@@ -4,7 +4,6 @@
 
 use std::fmt;
 
-use base64::{prelude::BASE64_STANDARD, Engine};
 use serde::{Deserialize, Serialize};
 
 use crate::{
@@ -20,7 +19,7 @@ use crate::{
     },
     middleware::{
         self, hash_str, AnchoredKey, DynError, Hash, MainPodInputs, NativePredicate, Params, Pod,
-        PodId, PodProver, Predicate, RecursivePod, StatementArg, KEY_TYPE, SELF,
+        PodId, PodProver, PodType, Predicate, RecursivePod, StatementArg, KEY_TYPE, SELF,
     },
 };
 
@@ -40,6 +39,7 @@ impl PodProver for MockProver {
 pub struct MockMainPod {
     params: Params,
     id: PodId,
+    vds_root: Hash,
     //   input_signed_pods: Vec<Box<dyn Pod>>,
     //   input_main_pods: Vec<Box<dyn Pod>>,
     // New statements introduced by this pod
@@ -51,6 +51,7 @@ pub struct MockMainPod {
     // All Merkle proofs
     // TODO: Use a backend-specific representation
     merkle_proofs: Vec<MerkleClaimAndProof>,
+    // TODO: Add input pods
 }
 
 impl fmt::Display for MockMainPod {
@@ -124,6 +125,14 @@ fn fmt_statement_index(
     Ok(())
 }
 
+#[derive(Serialize, Deserialize)]
+struct Data {
+    public_statements: Vec<Statement>,
+    operations: Vec<Operation>,
+    statements: Vec<Statement>,
+    merkle_proofs: Vec<MerkleClaimAndProof>,
+}
+
 /// Inputs are sorted as:
 /// - SignedPods
 /// - MainPods
@@ -148,7 +157,7 @@ impl MockMainPod {
     pub fn new(params: &Params, inputs: MainPodInputs) -> Result<Self> {
         // TODO: Insert a new public statement of ValueOf with `key=KEY_TYPE,
         // value=PodType::MockMainPod`
-        let statements = layout_statements(params, true, &inputs)?;
+        let (statements, public_statements) = layout_statements(params, true, &inputs)?;
         // Extract Merkle proofs and pad.
         let merkle_proofs = extract_merkle_proofs(params, inputs.operations)?;
 
@@ -161,15 +170,13 @@ impl MockMainPod {
         )?;
         let operations = process_public_statements_operations(params, &statements, operations)?;
 
-        let public_statements =
-            statements[statements.len() - params.max_public_statements..].to_vec();
-
         // get the id out of the public statements
         let id: PodId = PodId(calculate_id(&public_statements, params));
 
         Ok(Self {
             params: params.clone(),
             id,
+            vds_root: inputs.vds_root,
             //  input_signed_pods,
             //  input_main_pods,
             //  input_statements,
@@ -183,13 +190,27 @@ impl MockMainPod {
     // MockMainPods include some internal private state which is necessary
     // for verification. In non-mock Pods, this state will not be necessary,
     // as the public statements can be verified using a ZK proof.
-    pub(crate) fn deserialize(serialized: String) -> Result<Self> {
+    pub(crate) fn deserialize(
-        let proof = String::from_utf8(BASE64_STANDARD.decode(&serialized)?)
+        params: Params,
-            .map_err(|e| anyhow::anyhow!("Invalid base64 encoding: {}", e))?;
+        id: PodId,
-        let pod: MockMainPod = serde_json::from_str(&proof)
+        vds_root: Hash,
-            .map_err(|e| anyhow::anyhow!("Failed to parse proof: {}", e))?;
+        data: serde_json::Value,
-
+    ) -> Result<Box<dyn RecursivePod>> {
-        Ok(pod)
+        let Data {
+            public_statements,
+            operations,
+            statements,
+            merkle_proofs,
+        } = serde_json::from_value(data)?;
+        Ok(Box::new(Self {
+            params,
+            id,
+            vds_root,
+            public_statements,
+            operations,
+            statements,
+            merkle_proofs,
+        }))
     }
 
     fn _verify(&self) -> Result<()> {
@@ -289,6 +310,9 @@ impl Pod for MockMainPod {
     fn id(&self) -> PodId {
         self.id
     }
+    fn pod_type(&self) -> (usize, &'static str) {
+        (PodType::MockMain as usize, "MockMain")
+    }
     fn pub_self_statements(&self) -> Vec<middleware::Statement> {
         self.public_statements
             .iter()
@@ -297,8 +321,14 @@ impl Pod for MockMainPod {
             .collect()
     }
 
-    fn serialized_proof(&self) -> String {
+    fn serialize_data(&self) -> serde_json::Value {
-        BASE64_STANDARD.encode(serde_json::to_string(self).unwrap())
+        serde_json::to_value(Data {
+            public_statements: self.public_statements.clone(),
+            operations: self.operations.clone(),
+            statements: self.statements.clone(),
+            merkle_proofs: self.merkle_proofs.clone(),
+        })
+        .expect("serialization to json")
     }
 }
 
@@ -310,7 +340,7 @@ impl RecursivePod for MockMainPod {
         panic!("MockMainPod can't be verified in a recursive MainPod circuit");
     }
     fn vds_root(&self) -> Hash {
-        panic!("MockMainPod can't be verified in a recursive MainPod circuit");
+        self.vds_root
     }
 }
 

src/backends/plonky2/mock/signedpod.rs

@@ -1,6 +1,7 @@
 use std::collections::HashMap;
 
 use itertools::Itertools;
+use serde::{Deserialize, Serialize};
 
 use crate::{
     backends::plonky2::{
@@ -9,8 +10,9 @@ use crate::{
     },
     constants::MAX_DEPTH,
     middleware::{
-        containers::Dictionary, hash_str, AnchoredKey, DynError, Hash, Key, Params, Pod, PodId,
+        containers::Dictionary, hash_str, serialization::ordered_map, AnchoredKey, DynError, Hash,
-        PodSigner, PodType, RawValue, Statement, Value, KEY_SIGNER, KEY_TYPE, SELF,
+        Key, Params, Pod, PodId, PodSigner, PodType, RawValue, Statement, Value, KEY_SIGNER,
+        KEY_TYPE, SELF,
     },
 };
 
@@ -55,17 +57,18 @@ pub struct MockSignedPod {
     kvs: HashMap<Key, Value>,
 }
 
-impl MockSignedPod {
+#[derive(Serialize, Deserialize)]
-    pub(crate) fn new(id: PodId, signature: String, kvs: HashMap<Key, Value>) -> Self {
+struct Data {
-        Self { id, signature, kvs }
+    signature: String,
+    #[serde(serialize_with = "ordered_map")]
+    kvs: HashMap<Key, Value>,
 }
 
+impl MockSignedPod {
     pub fn signature(&self) -> String {
         self.signature.clone()
     }
-}
 
-impl MockSignedPod {
     fn _verify(&self) -> Result<()> {
         // 1. Verify id
         let mt = MerkleTree::new(
@@ -108,6 +111,15 @@ impl MockSignedPod {
 
         Ok(())
     }
+
+    pub(crate) fn deserialize(id: PodId, data: serde_json::Value) -> Result<Box<dyn Pod>> {
+        let data: Data = serde_json::from_value(data)?;
+        Ok(Box::new(Self {
+            id,
+            signature: data.signature,
+            kvs: data.kvs,
+        }))
+    }
 }
 
 impl Pod for MockSignedPod {
@@ -121,6 +133,9 @@ impl Pod for MockSignedPod {
     fn id(&self) -> PodId {
         self.id
     }
+    fn pod_type(&self) -> (usize, &'static str) {
+        (PodType::MockSigned as usize, "MockSigned")
+    }
 
     fn pub_self_statements(&self) -> Vec<Statement> {
         // By convention we put the KEY_TYPE first and KEY_SIGNER second
@@ -136,8 +151,12 @@ impl Pod for MockSignedPod {
             .collect()
     }
 
-    fn serialized_proof(&self) -> String {
+    fn serialize_data(&self) -> serde_json::Value {
-        serde_json::to_string(&self.signature).unwrap()
+        serde_json::to_value(Data {
+            signature: self.signature.clone(),
+            kvs: self.kvs.clone(),
+        })
+        .expect("serialization to json")
     }
 }
 

src/backends/plonky2/mod.rs

@@ -10,11 +10,13 @@ pub mod signedpod;
 
 use std::sync::LazyLock;
 
+use base64::{prelude::BASE64_STANDARD, Engine};
 pub use error::*;
+use plonky2::util::serialization::{Buffer, Read};
 
 use crate::{
     backends::plonky2::{
-        basetypes::CircuitData,
+        basetypes::{CircuitData, CommonCircuitData, Proof},
         circuits::mainpod::{MainPodVerifyTarget, NUM_PUBLIC_INPUTS},
         recursion::RecursiveCircuit,
     },
@@ -37,3 +39,36 @@ pub static STANDARD_REC_MAIN_POD_CIRCUIT_DATA: LazyLock<CircuitData> = LazyLock:
         .1
     )
 });
+
+pub fn serialize_bytes(bytes: &[u8]) -> String {
+    BASE64_STANDARD.encode(bytes)
+}
+
+pub fn deserialize_bytes(data: &str) -> Result<Vec<u8>> {
+    BASE64_STANDARD.decode(data).map_err(|e| {
+        Error::custom(format!(
+            "Failed to decode data from base64: {}. Value: {}",
+            e, data
+        ))
+    })
+}
+
+pub fn deserialize_proof(common: &CommonCircuitData, proof: &str) -> Result<Proof> {
+    let decoded = deserialize_bytes(proof)?;
+    let mut buf = Buffer::new(&decoded);
+    let proof = buf.read_proof(common).map_err(|e| {
+        Error::custom(format!(
+            "Failed to read proof from buffer: {}. Value: {}",
+            e, proof
+        ))
+    })?;
+
+    Ok(proof)
+}
+
+pub fn serialize_proof(proof: &Proof) -> String {
+    let mut buffer = Vec::new();
+    use plonky2::util::serialization::Write;
+    buffer.write_proof(proof).unwrap();
+    serialize_bytes(&buffer)
+}

src/backends/plonky2/signedpod.rs

@@ -1,12 +1,13 @@
 use std::collections::HashMap;
 
-use base64::{prelude::BASE64_STANDARD, Engine};
 use itertools::Itertools;
 use num_bigint::RandBigInt;
 use rand::rngs::OsRng;
+use serde::{Deserialize, Serialize};
 
 use crate::{
     backends::plonky2::{
+        deserialize_bytes,
         error::{Error, Result},
         primitives::{
             ec::{
@@ -15,6 +16,7 @@ use crate::{
             },
             merkletree::MerkleTree,
         },
+        serialize_bytes,
     },
     constants::MAX_DEPTH,
     middleware::{
@@ -68,6 +70,13 @@ pub struct SignedPod {
     pub dict: Dictionary,
 }
 
+#[derive(Serialize, Deserialize)]
+struct Data {
+    signer: String,
+    signature: String,
+    kvs: Dictionary,
+}
+
 impl SignedPod {
     fn _verify(&self) -> Result<()> {
         // 1. Verify type
@@ -107,23 +116,31 @@ impl SignedPod {
             .ok_or(Error::custom("Invalid signature!".into()))
     }
 
-    pub fn decode_proof(signature: &str) -> Result<(Point, Signature), Error> {
+    pub(crate) fn deserialize(id: PodId, data: serde_json::Value) -> Result<Box<dyn Pod>> {
-        let proof_bytes = BASE64_STANDARD.decode(signature).map_err(|e| {
+        let data: Data = serde_json::from_value(data)?;
-            Error::custom(format!(
+        let signer_bytes = deserialize_bytes(&data.signer)?;
-                "Failed to decode proof from base64: {}. Value: {}",
+        let signature_bytes = deserialize_bytes(&data.signature)?;
-                e, signature
-            ))
-        })?;
 
-        if proof_bytes.len() != 160 {
+        if signer_bytes.len() != 80 {
             return Err(Error::custom(
-                "Invalid byte encoding of signed POD proof.".to_string(),
+                "Invalid byte encoding of signed POD signer.".to_string(),
+            ));
+        }
+        if signature_bytes.len() != 80 {
+            return Err(Error::custom(
+                "Invalid byte encoding of signed POD signature.".to_string(),
             ));
         }
 
-        let signer = Point::from_bytes(&proof_bytes[..80])?;
+        let signer = Point::from_bytes(&signer_bytes)?;
-        let signature = Signature::from_bytes(&proof_bytes[80..])?;
+        let signature = Signature::from_bytes(&signature_bytes)?;
-        Ok((signer, signature))
+
+        Ok(Box::new(Self {
+            id,
+            signature,
+            signer,
+            dict: data.kvs,
+        }))
     }
 }
 
@@ -138,6 +155,9 @@ impl Pod for SignedPod {
     fn id(&self) -> PodId {
         self.id
     }
+    fn pod_type(&self) -> (usize, &'static str) {
+        (PodType::Signed as usize, "Signed")
+    }
 
     fn pub_self_statements(&self) -> Vec<Statement> {
         // By convention we put the KEY_TYPE first and KEY_SIGNER second
@@ -153,10 +173,15 @@ impl Pod for SignedPod {
             .collect()
     }
 
-    fn serialized_proof(&self) -> String {
+    fn serialize_data(&self) -> serde_json::Value {
-        // Serialise signer + signature.
+        let signer = serialize_bytes(&self.signer.as_bytes());
-        let proof_bytes = [self.signer.as_bytes(), self.signature.as_bytes()].concat();
+        let signature = serialize_bytes(&self.signature.as_bytes());
-        BASE64_STANDARD.encode(&proof_bytes)
+        serde_json::to_value(Data {
+            signer,
+            signature,
+            kvs: self.dict.clone(),
+        })
+        .expect("serialization to json")
     }
 }
 

src/examples/mod.rs

@@ -299,6 +299,7 @@ pub fn great_boy_pod_builder(
 pub fn great_boy_pod_full_flow() -> Result<(Params, MainPodBuilder)> {
     let params = Params {
         max_input_signed_pods: 6,
+        max_input_recursive_pods: 0,
         max_statements: 100,
         max_public_statements: 50,
         num_public_statements_id: 50,

src/frontend/mod.rs

@@ -61,7 +61,7 @@ impl SignedPodBuilder {
 /// SignedPod is a wrapper on top of backend::SignedPod, which additionally stores the
 /// string<-->hash relation of the keys.
 #[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(from = "SerializedSignedPod", into = "SerializedSignedPod")]
+#[serde(try_from = "SerializedSignedPod", into = "SerializedSignedPod")]
 pub struct SignedPod {
     pub pod: Box<dyn middleware::Pod>,
     // We store a copy of the key values for quick access

src/frontend/serialization.rs

@@ -1,20 +1,10 @@
-use std::{any::Any, collections::HashMap};
-
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 
 use super::Error;
 use crate::{
-    backends::plonky2::{
-        mainpod::{pad_statement, MainPod as Plonky2MainPod, Statement as BackendStatement},
-        mock::{mainpod::MockMainPod, signedpod::MockSignedPod},
-        signedpod::SignedPod as Plonky2SignedPod,
-    },
     frontend::{MainPod, SignedPod},
-    middleware::{
+    middleware::{deserialize_pod, deserialize_signed_pod, Hash, Params, PodId},
-        self, containers::Dictionary, serialization::ordered_map, AnchoredKey, Hash, Key, Params,
-        PodId, Statement, StatementArg, Value, EMPTY_HASH, SELF,
-    },
 };
 
 #[derive(Serialize, Deserialize, JsonSchema)]
@@ -27,102 +17,54 @@ pub enum SignedPodType {
 #[serde(rename_all = "camelCase")]
 #[schemars(rename = "SignedPod")]
 pub struct SerializedSignedPod {
+    pod_type: (usize, String),
     id: PodId,
-    #[serde(serialize_with = "ordered_map")]
+    data: serde_json::Value,
-    entries: HashMap<Key, Value>,
-    proof: String,
-    pod_type: SignedPodType,
-}
-
-#[derive(Serialize, Deserialize, JsonSchema)]
-pub enum MainPodType {
-    Main,
-    MockMain,
 }
 
 #[derive(Serialize, Deserialize, JsonSchema)]
 #[serde(rename_all = "camelCase")]
 #[schemars(rename = "MainPod")]
 pub struct SerializedMainPod {
+    params: Params,
+    pod_type: (usize, String),
     id: PodId,
     vds_root: Hash,
-    public_statements: Vec<Statement>,
+    data: serde_json::Value,
-    proof: String,
-    params: Params,
-    pod_type: MainPodType,
 }
 
 impl From<SignedPod> for SerializedSignedPod {
     fn from(pod: SignedPod) -> Self {
+        let (pod_type, pod_type_name_str) = pod.pod.pod_type();
+        let data = pod.pod.serialize_data();
         SerializedSignedPod {
+            pod_type: (pod_type, pod_type_name_str.to_string()),
             id: pod.id(),
-            entries: pod.kvs,
+            data,
-            proof: pod.pod.serialized_proof(),
-            pod_type: if (&*pod.pod as &dyn Any)
-                .downcast_ref::<Plonky2SignedPod>()
-                .is_some()
-            {
-                SignedPodType::Signed
-            } else if (&*pod.pod as &dyn Any)
-                .downcast_ref::<MockSignedPod>()
-                .is_some()
-            {
-                SignedPodType::MockSigned
-            } else {
-                unreachable!()
-            },
         }
     }
 }
 
-impl From<SerializedSignedPod> for SignedPod {
+impl TryFrom<SerializedSignedPod> for SignedPod {
-    fn from(serialized: SerializedSignedPod) -> Self {
+    type Error = Error;
-        match serialized.pod_type {
+
-            SignedPodType::Signed => {
+    fn try_from(serialized: SerializedSignedPod) -> Result<Self, Self::Error> {
-                let (signer, signature) =
+        let pod = deserialize_signed_pod(serialized.pod_type.0, serialized.id, serialized.data)?;
-                    Plonky2SignedPod::decode_proof(&serialized.proof).unwrap();
+        let kvs = pod.kvs().into_iter().map(|(ak, v)| (ak.key, v)).collect();
-                SignedPod {
+        Ok(Self { pod, kvs })
-                    pod: Box::new(Plonky2SignedPod {
-                        id: serialized.id,
-                        signer,
-                        signature,
-                        dict: Dictionary::new(serialized.entries.clone()).unwrap(),
-                    }),
-                    kvs: serialized.entries,
-                }
-            }
-            SignedPodType::MockSigned => SignedPod {
-                pod: Box::new(MockSignedPod::new(
-                    serialized.id,
-                    serde_json::from_str(&serialized.proof).unwrap(),
-                    serialized.entries.clone(),
-                )),
-                kvs: serialized.entries,
-            },
-        }
     }
 }
 
 impl From<MainPod> for SerializedMainPod {
     fn from(pod: MainPod) -> Self {
-        let (pod_type, vds_root) =
+        let (pod_type, pod_type_name_str) = pod.pod.pod_type();
-            if let Some(pod) = (&*pod.pod as &dyn Any).downcast_ref::<Plonky2MainPod>() {
+        let data = pod.pod.serialize_data();
-                (MainPodType::Main, pod.vds_root())
-            } else if (&*pod.pod as &dyn Any)
-                .downcast_ref::<MockMainPod>()
-                .is_some()
-            {
-                (MainPodType::MockMain, EMPTY_HASH)
-            } else {
-                unreachable!()
-            };
         SerializedMainPod {
+            pod_type: (pod_type, pod_type_name_str.to_string()),
             id: pod.id(),
-            vds_root,
+            vds_root: pod.pod.vds_root(),
-            proof: pod.pod.serialized_proof(),
             params: pod.params.clone(),
-            pod_type,
+            data,
-            public_statements: pod.public_statements.clone(),
         }
     }
 }
@@ -131,76 +73,20 @@ impl TryFrom<SerializedMainPod> for MainPod {
     type Error = Error;
 
     fn try_from(serialized: SerializedMainPod) -> Result<Self, Self::Error> {
-        match serialized.pod_type {
+        let pod = deserialize_pod(
-            MainPodType::Main => Ok(MainPod {
+            serialized.pod_type.0,
-                pod: Box::new(Plonky2MainPod::new(
+            serialized.params.clone(),
-                    Plonky2MainPod::decode_proof(&serialized.proof, &serialized.params).map_err(
-                        |e| {
-                            Error::custom(format!(
-                                "Failed to deserialize MainPod proof: {}. Value: {}",
-                                e, serialized.proof
-                            ))
-                        },
-                    )?,
-                    middleware_statements_to_backend(
-                        serialized.public_statements.clone(),
-                        &serialized.params,
-                        serialized.id,
-                    ),
             serialized.id,
             serialized.vds_root,
-                    serialized.params.clone(),
+            serialized.data,
-                )),
+        )?;
-                public_statements: serialized.public_statements,
+        let public_statements = pod.pub_statements();
+        Ok(Self {
+            pod,
+            public_statements,
             params: serialized.params,
-            }),
-            MainPodType::MockMain => Ok(MainPod {
-                pod: Box::new(
-                    MockMainPod::deserialize(serialized.proof.clone()).map_err(|e| {
-                        Error::custom(format!(
-                            "Failed to deserialize MockMainPod: {}. Value: {}",
-                            e, serialized.proof
-                        ))
-                    })?,
-                ),
-                public_statements: serialized.public_statements,
-                params: serialized.params,
-            }),
-        }
-    }
-}
-
-// To deserialize a backend MainPod, we need to convert the middleware
-// statements to backend statements, and padding the list with None statements.
-fn middleware_statements_to_backend(
-    mid_statements: Vec<Statement>,
-    params: &Params,
-    id: PodId,
-) -> Vec<BackendStatement> {
-    let mut statements = Vec::new();
-    for i in 0..(params.max_public_statements) {
-        let mut st: BackendStatement = mid_statements
-            .get(i)
-            .unwrap_or(&middleware::Statement::None)
-            .clone()
-            .into();
-
-        st = BackendStatement(
-            st.0.clone(),
-            st.1.iter()
-                .map(|sa| match &sa {
-                    StatementArg::Key(AnchoredKey { pod_id, key }) if *pod_id == id => {
-                        StatementArg::Key(AnchoredKey::new(SELF, key.clone()))
-                    }
-                    _ => sa.clone(),
         })
-                .collect(),
-        );
-        pad_statement(params, &mut st);
-        statements.push(st);
     }
-
-    statements
 }
 
 #[cfg(test)]
@@ -225,7 +111,7 @@ mod tests {
         frontend::{Result, SignedPodBuilder},
         middleware::{
             self,
-            containers::{Array, Set},
+            containers::{Array, Dictionary, Set},
             Params, TypedValue,
         },
     };

src/middleware/basetypes.rs

@@ -46,6 +46,8 @@ use super::serialization::*;
 // types would come from the plonky3 backend.
 #[cfg(feature = "backend_plonky2")]
 pub use crate::backends::plonky2::basetypes::*;
+#[cfg(feature = "backend_plonky2")]
+pub use crate::backends::plonky2::{Error as BackendError, Result as BackendResult};
 use crate::middleware::{Params, ToFields, Value};
 
 pub const HASH_SIZE: usize = 4;

src/middleware/mod.rs

@@ -2,6 +2,8 @@
 //! the backend.
 
 use std::sync::Arc;
+
+use strum_macros::FromRepr;
 mod basetypes;
 use std::{
     cmp::{Ordering, PartialEq, PartialOrd},
@@ -15,6 +17,7 @@ pub mod containers;
 mod custom;
 mod error;
 mod operation;
+mod pod_deserialization;
 pub mod serialization;
 mod statement;
 use std::{any::Any, collections::HashMap, fmt};
@@ -24,6 +27,7 @@ pub use custom::*;
 use dyn_clone::DynClone;
 pub use error::*;
 pub use operation::*;
+pub use pod_deserialization::*;
 use serialization::*;
 pub use statement::*;
 
@@ -565,7 +569,7 @@ impl ToFields for PodId {
     }
 }
 
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, FromRepr, Serialize, Deserialize, JsonSchema)]
 pub enum PodType {
     None = 0,
     MockSigned = 1,
@@ -636,7 +640,8 @@ impl Default for Params {
             max_signed_pod_values: 8,
             max_public_statements: 10,
             num_public_statements_id: 16,
-            max_statement_args: 5,
+            // TODO: Reduce to 5 or less after https://github.com/0xPARC/pod2/issues/229
+            max_statement_args: 6,
             max_operation_args: 5,
             max_custom_predicate_batches: 2,
             max_custom_predicate_verifications: 5,
@@ -735,6 +740,8 @@ pub trait Pod: fmt::Debug + DynClone + Any {
     fn params(&self) -> &Params;
     fn verify(&self) -> Result<(), Box<DynError>>;
     fn id(&self) -> PodId;
+    /// Return a uuid of the pod type and its name.  The name is only used as metadata.
+    fn pod_type(&self) -> (usize, &'static str);
     /// Statements as internally generated, where self-referencing arguments use SELF in the
     /// anchored key.  The serialization of these statements is used to calculate the id.
     fn pub_self_statements(&self) -> Vec<Statement>;
@@ -746,6 +753,9 @@ pub trait Pod: fmt::Debug + DynClone + Any {
             .map(|statement| normalize_statement(&statement, self.id()))
             .collect()
     }
+    /// Return this Pods data serialized into a json value.  This serialization can skip `params,
+    /// id, vds_root`
+    fn serialize_data(&self) -> serde_json::Value;
     /// Extract key-values from ValueOf public statements
     fn kvs(&self) -> HashMap<AnchoredKey, Value> {
         self.pub_statements()
@@ -767,7 +777,7 @@ pub trait Pod: fmt::Debug + DynClone + Any {
     // reconstruct the proof.
     // It is an important principle that this data is opaque to the front-end
     // and any third-party code.
-    fn serialized_proof(&self) -> String;
+    // fn serialized_proof(&self) -> String;
 }
 
 // impl Clone for Box<dyn Pod>
@@ -810,11 +820,14 @@ impl Pod for NonePod {
     fn id(&self) -> PodId {
         PodId(EMPTY_HASH)
     }
+    fn pod_type(&self) -> (usize, &'static str) {
+        (PodType::None as usize, "None")
+    }
     fn pub_self_statements(&self) -> Vec<Statement> {
         Vec::new()
     }
-    fn serialized_proof(&self) -> String {
+    fn serialize_data(&self) -> serde_json::Value {
-        "".to_string()
+        serde_json::Value::Null
     }
 }
 

src/middleware/pod_deserialization.rs

@@ -0,0 +1,93 @@
+use std::{
+    collections::HashMap,
+    sync::{LazyLock, Mutex},
+};
+
+use crate::middleware::{
+    BackendResult, Error, Hash, Params, Pod, PodId, PodType, RecursivePod, Result,
+};
+
+type DeserializeFn = fn(
+    params: Params,
+    id: PodId,
+    vds_root: Hash,
+    data: serde_json::Value,
+) -> BackendResult<Box<dyn RecursivePod>>;
+
+static DESERIALIZERS: LazyLock<Mutex<HashMap<usize, DeserializeFn>>> =
+    LazyLock::new(backend::deserializers_default);
+
+pub fn register_pod_deserializer(pod_type: usize, deserialize_fn: DeserializeFn) {
+    DESERIALIZERS
+        .lock()
+        .unwrap()
+        .insert(pod_type, deserialize_fn);
+}
+
+pub fn deserialize_pod(
+    pod_type: usize,
+    params: Params,
+    id: PodId,
+    vds_root: Hash,
+    data: serde_json::Value,
+) -> Result<Box<dyn RecursivePod>> {
+    let deserialize_fn: DeserializeFn =
+        *DESERIALIZERS
+            .lock()
+            .unwrap()
+            .get(&pod_type)
+            .ok_or(Error::custom(format!(
+                "pod deserializer for pod_type={} not registered.  See https://github.com/0xPARC/pod2/wiki/PodType for pod type assignments.",
+                pod_type
+            )))?;
+
+    deserialize_fn(params, id, vds_root, data)
+        .map_err(|e| Error::custom(format!("deserialize error: {:?}", e)))
+}
+
+pub fn deserialize_signed_pod(
+    pod_type: usize,
+    id: PodId,
+    data: serde_json::Value,
+) -> Result<Box<dyn Pod>> {
+    backend::deserialize_signed_pod(pod_type, id, data)
+}
+
+#[cfg(feature = "backend_plonky2")]
+mod backend {
+    use super::*;
+    use crate::backends::plonky2::{
+        emptypod::EmptyPod,
+        mainpod::MainPod,
+        mock::{emptypod::MockEmptyPod, mainpod::MockMainPod, signedpod::MockSignedPod},
+        signedpod::SignedPod,
+    };
+
+    pub(super) fn deserializers_default() -> Mutex<HashMap<usize, DeserializeFn>> {
+        let mut map: HashMap<usize, DeserializeFn> = HashMap::new();
+        map.insert(PodType::Empty as usize, EmptyPod::deserialize);
+        map.insert(PodType::Main as usize, MainPod::deserialize);
+        map.insert(PodType::MockEmpty as usize, MockEmptyPod::deserialize);
+        map.insert(PodType::MockMain as usize, MockMainPod::deserialize);
+        Mutex::new(map)
+    }
+
+    pub(super) fn deserialize_signed_pod(
+        pod_type: usize,
+        id: PodId,
+        data: serde_json::Value,
+    ) -> Result<Box<dyn Pod>> {
+        if pod_type == PodType::MockSigned as usize {
+            MockSignedPod::deserialize(id, data)
+                .map_err(|e| Error::custom(format!("deserialize error: {:?}", e)))
+        } else if pod_type == PodType::Signed as usize {
+            SignedPod::deserialize(id, data)
+                .map_err(|e| Error::custom(format!("deserialize error: {:?}", e)))
+        } else {
+            Err(Error::custom(format!(
+                "unexpected pod_type={} for deserialize_signed_pod",
+                pod_type
+            )))
+        }
+    }
+}