Complete the verification in MainMockPod (#302)

- Update the `RecursivePod` trait to return `vd_set` instead of `vds_root`
  - A native verifier requires the entire set to reason about the circuits that have been used in the recursive tree
- Implement serialization/deserialization for `VDSet`
- Remove `DynError` and use `BackendError` instead for middleware functions that wrap or define trait functions implemented in the backend.  This is based on the fact that we will only have a single backend enabled at a time, so there's no need for a `dyn Error`
  - Move the implementations of `_verify` functions to `verify` and similarly for `_prove`
- Complete the verification of a MockMainPod: the verification of input pods was missing.  The inclusion of these input pods in the serialization was also missing.  With this change a `MockMainPod` will grow after each recursion.  This was expected from the design but was not the case because of the missing recursive native verification implementation.

* apply feedback from @arnaucube

src/backends/plonky2/mock/emptypod.rs

@@ -7,8 +7,8 @@ use crate::{
         mainpod::{self, calculate_id},
     },
     middleware::{
-        AnchoredKey, DynError, Hash, Params, Pod, PodId, PodType, RecursivePod, Statement, Value,
-        KEY_TYPE, SELF,
+        AnchoredKey, Params, Pod, PodId, PodType, RecursivePod, Statement, VDSet, Value, KEY_TYPE,
+        SELF,
     },
 };
 
@@ -16,6 +16,7 @@ use crate::{
 pub struct MockEmptyPod {
     params: Params,
     id: PodId,
+    vd_set: VDSet,
 }
 
 fn type_statement() -> Statement {
@@ -26,15 +27,22 @@ fn type_statement() -> Statement {
 }
 
 impl MockEmptyPod {
-    pub fn new_boxed(params: &Params) -> Box<dyn RecursivePod> {
+    pub fn new_boxed(params: &Params, vd_set: VDSet) -> Box<dyn RecursivePod> {
         let statements = [mainpod::Statement::from(type_statement())];
         let id = PodId(calculate_id(&statements, params));
         Box::new(Self {
             params: params.clone(),
             id,
+            vd_set,
         })
     }
-    fn _verify(&self) -> Result<()> {
+}
+
+impl Pod for MockEmptyPod {
+    fn params(&self) -> &Params {
+        &self.params
+    }
+    fn verify(&self) -> Result<()> {
         let statements = self
             .pub_self_statements()
             .into_iter()
@@ -46,15 +54,6 @@ impl MockEmptyPod {
         }
         Ok(())
     }
-}
-
-impl Pod for MockEmptyPod {
-    fn params(&self) -> &Params {
-        &self.params
-    }
-    fn verify(&self) -> Result<(), Box<DynError>> {
-        Ok(self._verify()?)
-    }
     fn id(&self) -> PodId {
         self.id
     }
@@ -77,16 +76,16 @@ impl RecursivePod for MockEmptyPod {
     fn proof(&self) -> Proof {
         panic!("MockEmptyPod can't be verified in a recursive MainPod circuit");
     }
-    fn vds_root(&self) -> Hash {
-        panic!("MockEmptyPod can't be verified in a recursive MainPod circuit");
+    fn vd_set(&self) -> &VDSet {
+        &self.vd_set
     }
     fn deserialize_data(
         params: Params,
         _data: serde_json::Value,
-        _vds_root: Hash,
+        vd_set: VDSet,
         id: PodId,
-    ) -> Result<Box<dyn RecursivePod>, Box<DynError>> {
-        Ok(Box::new(Self { params, id }))
+    ) -> Result<Box<dyn RecursivePod>> {
+        Ok(Box::new(Self { params, id, vd_set }))
     }
 }
 
@@ -98,7 +97,7 @@ pub mod tests {
     fn test_mock_empty_pod() {
         let params = Params::default();
 
-        let empty_pod = MockEmptyPod::new_boxed(&params);
+        let empty_pod = MockEmptyPod::new_boxed(&params, VDSet::new(8, &[]).unwrap());
         empty_pod.verify().unwrap();
     }
 }

src/backends/plonky2/mock/mainpod.rs

@@ -2,7 +2,7 @@
 // MainPod
 //
 
-use std::fmt;
+use std::{fmt, iter};
 
 use itertools::Itertools;
 use serde::{Deserialize, Serialize};
@@ -16,12 +16,14 @@ use crate::{
             process_private_statements_operations, process_public_statements_operations, Operation,
             Statement,
         },
+        mock::{emptypod::MockEmptyPod, signedpod::MockSignedPod},
         primitives::merkletree::MerkleClaimAndProof,
+        recursion::hash_verifier_data,
     },
     middleware::{
-        self, hash_str, AnchoredKey, DynError, Hash, MainPodInputs, NativeOperation,
-        NativePredicate, OperationType, Params, Pod, PodId, PodProver, PodType, Predicate,
-        RecursivePod, StatementArg, VDSet, KEY_TYPE, SELF,
+        self, deserialize_pod, deserialize_signed_pod, hash_str, AnchoredKey, Hash, MainPodInputs,
+        NativeOperation, NativePredicate, OperationType, Params, Pod, PodId, PodProver, PodType,
+        Predicate, RecursivePod, StatementArg, VDSet, Value, KEY_TYPE, SELF,
     },
 };
 
@@ -33,58 +35,65 @@ impl PodProver for MockProver {
         params: &Params,
         _vd_set: &VDSet,
         inputs: MainPodInputs,
-    ) -> Result<Box<dyn RecursivePod>, Box<DynError>> {
+    ) -> Result<Box<dyn RecursivePod>> {
         Ok(Box::new(MockMainPod::new(params, inputs)?))
     }
 }
 
-#[derive(Clone, Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug)]
 pub struct MockMainPod {
     params: Params,
     id: PodId,
-    vds_root: Hash,
-    //   input_signed_pods: Vec<Box<dyn Pod>>,
-    //   input_main_pods: Vec<Box<dyn Pod>>,
-    // New statements introduced by this pod
-    //   input_statements: Vec<Statement>,
-    public_statements: Vec<Statement>,
-    operations: Vec<Operation>,
-    // All statements (inherited + new)
+    vd_set: VDSet,
+    input_signed_pods: Vec<Box<dyn Pod>>,
+    input_recursive_pods: Vec<Box<dyn RecursivePod>>,
+    // All statements (inherited + newly introduced by this pod)
     statements: Vec<Statement>,
+    operations: Vec<Operation>,
+    // public subset of the `statements` vector
+    public_statements: Vec<Statement>,
     // All Merkle proofs
-    // TODO: Use a backend-specific representation
-    merkle_proofs: Vec<MerkleClaimAndProof>,
-    // TODO: Add input pods
+    merkle_proofs_containers: Vec<MerkleClaimAndProof>,
 }
 
 impl fmt::Display for MockMainPod {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         writeln!(f, "MockMainPod ({}):", self.id)?;
-        // TODO print input signed pods id and type
-        // TODO print input main pods id and type
         let offset_input_signed_pods = Self::offset_input_signed_pods();
-        let offset_input_main_pods = self.offset_input_main_pods();
+        let offset_input_recursive_pods = self.offset_input_recursive_pods();
         let offset_input_statements = self.offset_input_statements();
         let offset_public_statements = self.offset_public_statements();
         for (i, st) in self.statements.iter().enumerate() {
-            if (i >= offset_input_signed_pods && i < offset_input_main_pods)
+            if self.params.max_input_signed_pods > 0
+                && (i >= offset_input_signed_pods && i < offset_input_recursive_pods)
                 && ((i - offset_input_signed_pods) % self.params.max_signed_pod_values == 0)
             {
+                let index = (i - offset_input_signed_pods) / self.params.max_signed_pod_values;
+                let pod = &self.input_signed_pods[index];
+                let id = pod.id();
+                let pod_type = pod.pod_type();
                 writeln!(
                     f,
-                    "  from input SignedPod {}:",
-                    i / self.params.max_signed_pod_values
+                    "  from input SignedPod {} (id={}, type={:?}):",
+                    index, id, pod_type
                 )?;
             }
-            if (i >= offset_input_main_pods)
+            if self.params.max_input_recursive_pods > 0
+                && (i >= offset_input_recursive_pods)
                 && (i < offset_input_statements)
-                && ((i - offset_input_main_pods) % self.params.max_input_pods_public_statements
+                && ((i - offset_input_recursive_pods)
+                    % self.params.max_input_pods_public_statements
                     == 0)
             {
+                let index = (i - offset_input_recursive_pods)
+                    / self.params.max_input_pods_public_statements;
+                let pod = &self.input_recursive_pods[index];
+                let id = pod.id();
+                let pod_type = pod.pod_type();
                 writeln!(
                     f,
-                    "  from input MainPod {}:",
-                    (i - offset_input_main_pods) / self.params.max_signed_pod_values
+                    "  from input recursive Pod {} (id={}, type={:?}):",
+                    index, id, pod_type
                 )?;
             }
             if i == offset_input_statements {
@@ -134,6 +143,8 @@ struct Data {
     operations: Vec<Operation>,
     statements: Vec<Statement>,
     merkle_proofs: Vec<MerkleClaimAndProof>,
+    input_signed_pods: Vec<(usize, PodId, serde_json::Value)>,
+    input_recursive_pods: Vec<(usize, Params, PodId, VDSet, serde_json::Value)>,
 }
 
 /// Inputs are sorted as:
@@ -145,12 +156,12 @@ impl MockMainPod {
     fn offset_input_signed_pods() -> usize {
         1
     }
-    fn offset_input_main_pods(&self) -> usize {
+    fn offset_input_recursive_pods(&self) -> usize {
         Self::offset_input_signed_pods()
             + self.params.max_input_signed_pods * self.params.max_signed_pod_values
     }
     fn offset_input_statements(&self) -> usize {
-        self.offset_input_main_pods()
+        self.offset_input_recursive_pods()
             + self.params.max_input_recursive_pods * self.params.max_input_pods_public_statements
     }
     fn offset_public_statements(&self) -> usize {
@@ -158,8 +169,6 @@ impl MockMainPod {
     }
 
     pub fn new(params: &Params, inputs: MainPodInputs) -> Result<Self> {
-        // TODO: Insert a new public statement of ValueOf with `key=KEY_TYPE,
-        // value=PodType::MockMainPod`
         let (statements, public_statements) = layout_statements(params, true, &inputs)?;
         // Extract Merkle proofs and pad.
         let merkle_proofs = extract_merkle_proofs(params, inputs.operations, inputs.statements)?;
@@ -176,38 +185,100 @@ impl MockMainPod {
         // get the id out of the public statements
         let id: PodId = PodId(calculate_id(&public_statements, params));
 
+        let pad_signed_pod: Box<dyn Pod> = Box::new(MockSignedPod::dummy());
+        let input_signed_pods: Vec<Box<dyn Pod>> = inputs
+            .signed_pods
+            .iter()
+            .map(|p| dyn_clone::clone_box(*p))
+            .chain(iter::repeat_with(|| pad_signed_pod.clone()))
+            .take(params.max_input_signed_pods)
+            .collect();
+        let pad_pod = MockEmptyPod::new_boxed(params, inputs.vd_set.clone());
+        let input_recursive_pods: Vec<Box<dyn RecursivePod>> = inputs
+            .recursive_pods
+            .iter()
+            .map(|p| dyn_clone::clone_box(*p))
+            .chain(iter::repeat_with(|| pad_pod.clone()))
+            .take(params.max_input_recursive_pods)
+            .collect();
         Ok(Self {
             params: params.clone(),
             id,
-            vds_root: inputs.vds_set.root(),
-            //  input_signed_pods,
-            //  input_main_pods,
-            //  input_statements,
+            vd_set: inputs.vd_set,
+            input_signed_pods,
+            input_recursive_pods,
             public_statements,
             statements,
             operations,
-            merkle_proofs,
+            merkle_proofs_containers: merkle_proofs,
         })
     }
 
-    fn _verify(&self) -> Result<()> {
-        // 1. TODO: Verify input pods
+    pub fn params(&self) -> &Params {
+        &self.params
+    }
+}
+
+impl Pod for MockMainPod {
+    fn params(&self) -> &Params {
+        &self.params
+    }
+
+    fn verify(&self) -> Result<()> {
+        // 1. Verify input pods
+        for pod in &self.input_signed_pods {
+            pod.verify()?;
+        }
+        for pod in &self.input_recursive_pods {
+            pod.verify()?;
+            if pod.vd_set().root() != self.vd_set.root() {
+                return Err(Error::custom(format!(
+                    "vds_root in input recursive pod doesn't match MockMainPod vds_root: {} != {}",
+                    pod.vd_set().root(),
+                    self.vd_set.root(),
+                )));
+            }
+            let (pod_type, _) = pod.pod_type();
+            // If the pod is not mock, check that its verifier data is in the set
+            if pod_type != PodType::MockMain as usize && pod_type != PodType::MockEmpty as usize {
+                let verifier_data = pod.verifier_data();
+                let verifier_data_hash = hash_verifier_data(&verifier_data);
+                if !self.vd_set.contains(verifier_data_hash) {
+                    return Err(Error::custom(format!(
+                        "vds_root in input recursive pod not in the set: {} not in {}",
+                        Hash(verifier_data_hash.elements),
+                        self.vd_set.root(),
+                    )));
+                }
+            }
+        }
 
         let input_statement_offset = self.offset_input_statements();
         // get the input_statements from the self.statements
         let input_statements = &self.statements[input_statement_offset..];
         // 2. get the id out of the public statements, and ensure it is equal to self.id
-        let ids_match = self.id == PodId(calculate_id(&self.public_statements, &self.params));
+        if self.id != PodId(calculate_id(&self.public_statements, &self.params)) {
+            return Err(Error::pod_id_invalid());
+        }
+        // 4. Verify type
         // find a ValueOf statement from the public statements with key=KEY_TYPE and check that the
         // value is PodType::MockMainPod
-        let has_type_statement = self.public_statements.iter().any(|s| {
-            s.0 == Predicate::Native(NativePredicate::Equal) && {
-                if let [StatementArg::Key(AnchoredKey { pod_id, ref key }), StatementArg::Literal(_)] = &s.1[..2] {
-                    pod_id == &SELF && key.hash() == hash_str(KEY_TYPE)
+        let type_statement = &self.public_statements[0];
+        let type_statement_ok = type_statement.0 == Predicate::Native(NativePredicate::Equal)
+            && {
+                if let [StatementArg::Key(AnchoredKey { pod_id, ref key }), StatementArg::Literal(pod_type)] =
+                    &type_statement.1[..2]
+                {
+                    pod_id == &SELF
+                        && key.hash() == hash_str(KEY_TYPE)
+                        && *pod_type == Value::from(PodType::MockMain)
                 } else {
                     false
                 }
-        }});
+            };
+        if !type_statement_ok {
+            return Err(Error::not_type_statement());
+        }
         // 3. check that all `NewEntry` operations have unique keys
         // (no duplicates)
         let value_ofs_unique = input_statements
@@ -225,7 +296,9 @@ impl MockMainPod {
                 }
             })
             .all_unique();
-        // 4. TODO: Verify type
+        if !value_ofs_unique {
+            return Err(Error::repeated_value_of());
+        }
 
         // 5. verify that all `input_statements` are correctly generated
         // by `self.operations` (where each operation can only access previous statements)
@@ -236,40 +309,19 @@ impl MockMainPod {
                 self.operations[i]
                     .deref(
                         &self.statements[..input_statement_offset + i],
-                        &self.merkle_proofs,
+                        &self.merkle_proofs_containers,
                     )
                     .unwrap()
                     .check_and_log(&self.params, &s.clone().try_into().unwrap())
             })
             .collect::<Result<Vec<_>, middleware::Error>>()
             .unwrap();
-        if !ids_match {
-            return Err(Error::pod_id_invalid());
-        }
-        if !has_type_statement {
-            return Err(Error::not_type_statement());
-        }
-        if !value_ofs_unique {
-            return Err(Error::repeated_value_of());
-        }
         if !statement_check.iter().all(|b| *b) {
             return Err(Error::statement_not_check());
         }
         Ok(())
     }
 
-    pub fn params(&self) -> &Params {
-        &self.params
-    }
-}
-
-impl Pod for MockMainPod {
-    fn params(&self) -> &Params {
-        &self.params
-    }
-    fn verify(&self) -> Result<(), Box<DynError>> {
-        Ok(self._verify()?)
-    }
     fn id(&self) -> PodId {
         self.id
     }
@@ -285,11 +337,31 @@ impl Pod for MockMainPod {
     }
 
     fn serialize_data(&self) -> serde_json::Value {
+        let input_signed_pods = self
+            .input_signed_pods
+            .iter()
+            .map(|p| (p.pod_type().0, p.id(), p.serialize_data()))
+            .collect();
+        let input_recursive_pods = self
+            .input_recursive_pods
+            .iter()
+            .map(|p| {
+                (
+                    p.pod_type().0,
+                    p.params().clone(),
+                    p.id(),
+                    p.vd_set().clone(),
+                    p.serialize_data(),
+                )
+            })
+            .collect();
         serde_json::to_value(Data {
             public_statements: self.public_statements.clone(),
             operations: self.operations.clone(),
             statements: self.statements.clone(),
-            merkle_proofs: self.merkle_proofs.clone(),
+            merkle_proofs: self.merkle_proofs_containers.clone(),
+            input_signed_pods,
+            input_recursive_pods,
         })
         .expect("serialization to json")
     }
@@ -302,8 +374,8 @@ impl RecursivePod for MockMainPod {
     fn proof(&self) -> Proof {
         panic!("MockMainPod can't be verified in a recursive MainPod circuit");
     }
-    fn vds_root(&self) -> Hash {
-        self.vds_root
+    fn vd_set(&self) -> &VDSet {
+        &self.vd_set
     }
     // MockMainPods include some internal private state which is necessary
     // for verification. In non-mock Pods, this state will not be necessary,
@@ -311,23 +383,37 @@ impl RecursivePod for MockMainPod {
     fn deserialize_data(
         params: Params,
         data: serde_json::Value,
-        vds_root: Hash,
+        vd_set: VDSet,
         id: PodId,
-    ) -> Result<Box<dyn RecursivePod>, Box<DynError>> {
+    ) -> Result<Box<dyn RecursivePod>> {
         let Data {
             public_statements,
             operations,
             statements,
             merkle_proofs,
+            input_signed_pods,
+            input_recursive_pods,
         } = serde_json::from_value(data)?;
+        let input_signed_pods = input_signed_pods
+            .into_iter()
+            .map(|(pod_type, id, data)| deserialize_signed_pod(pod_type, id, data))
+            .collect::<Result<Vec<_>>>()?;
+        let input_recursive_pods = input_recursive_pods
+            .into_iter()
+            .map(|(pod_type, params, id, vd_set, data)| {
+                deserialize_pod(pod_type, params, id, vd_set, data)
+            })
+            .collect::<Result<Vec<_>>>()?;
         Ok(Box::new(Self {
             params,
             id,
-            vds_root,
+            vd_set,
+            input_signed_pods,
+            input_recursive_pods,
             public_statements,
             operations,
             statements,
-            merkle_proofs,
+            merkle_proofs_containers: merkle_proofs,
         }))
     }
 }
@@ -341,16 +427,16 @@ pub mod tests {
         backends::plonky2::mock::signedpod::MockSigner,
         examples::{
             great_boy_pod_full_flow, tickets_pod_full_flow, zu_kyc_pod_builder,
-            zu_kyc_sign_pod_builders,
+            zu_kyc_sign_pod_builders, MOCK_VD_SET,
         },
         frontend,
-        middleware::{self, DEFAULT_VD_SET},
+        middleware::{self},
     };
 
     #[test]
     fn test_mock_main_zu_kyc() -> frontend::Result<()> {
         let params = middleware::Params::default();
-        let vd_set = &*DEFAULT_VD_SET;
+        let vd_set = &*MOCK_VD_SET;
         let (gov_id_builder, pay_stub_builder, sanction_list_builder) =
             zu_kyc_sign_pod_builders(&params);
         let mut signer = MockSigner {
@@ -381,9 +467,7 @@ pub mod tests {
 
         println!("{:#}", pod);
 
-        pod.verify()?; // TODO
-                       // println!("id: {}", pod.id());
-                       // println!("pub_statements: {:?}", pod.pub_statements());
+        pod.verify()?;
         Ok(())
     }
 

src/backends/plonky2/mock/signedpod.rs

@@ -10,9 +10,8 @@ use crate::{
     },
     constants::MAX_DEPTH,
     middleware::{
-        containers::Dictionary, hash_str, serialization::ordered_map, AnchoredKey, DynError, Key,
-        Params, Pod, PodId, PodSigner, PodType, RawValue, Statement, Value, KEY_SIGNER, KEY_TYPE,
-        SELF,
+        containers::Dictionary, hash_str, serialization::ordered_map, AnchoredKey, Key, Params,
+        Pod, PodId, PodSigner, PodType, RawValue, Statement, Value, KEY_SIGNER, KEY_TYPE, SELF,
     },
 };
 
@@ -41,11 +40,7 @@ impl MockSigner {
 }
 
 impl PodSigner for MockSigner {
-    fn sign(
-        &mut self,
-        params: &Params,
-        kvs: &HashMap<Key, Value>,
-    ) -> Result<Box<dyn Pod>, Box<DynError>> {
+    fn sign(&mut self, params: &Params, kvs: &HashMap<Key, Value>) -> Result<Box<dyn Pod>> {
         Ok(self._sign(params, kvs).map(Box::new)?)
     }
 }
@@ -69,7 +64,30 @@ impl MockSignedPod {
         self.signature.clone()
     }
 
-    fn _verify(&self) -> Result<()> {
+    pub(crate) fn deserialize(id: PodId, data: serde_json::Value) -> Result<Box<dyn Pod>> {
+        let data: Data = serde_json::from_value(data)?;
+        Ok(Box::new(Self {
+            id,
+            signature: data.signature,
+            kvs: data.kvs,
+        }))
+    }
+    /// Generate a valid MockSignedPod with a public deterministic public key and no other
+    /// key-values than the default ones.  This is used for padding.
+    pub fn dummy() -> MockSignedPod {
+        MockSigner {
+            pk: "dummy".to_string(),
+        }
+        ._sign(&Params::default(), &HashMap::new())
+        .expect("valid")
+    }
+}
+
+impl Pod for MockSignedPod {
+    fn params(&self) -> &Params {
+        panic!("MockSignedPod doesn't have params");
+    }
+    fn verify(&self) -> Result<()> {
         // 1. Verify id
         let mt = MerkleTree::new(
             MAX_DEPTH,
@@ -112,33 +130,6 @@ impl MockSignedPod {
         Ok(())
     }
 
-    pub(crate) fn deserialize(id: PodId, data: serde_json::Value) -> Result<Box<dyn Pod>> {
-        let data: Data = serde_json::from_value(data)?;
-        Ok(Box::new(Self {
-            id,
-            signature: data.signature,
-            kvs: data.kvs,
-        }))
-    }
-    /// Generate a valid MockSignedPod with a public deterministic public key and no other
-    /// key-values than the default ones.  This is used for padding.
-    pub fn dummy() -> MockSignedPod {
-        MockSigner {
-            pk: "dummy".to_string(),
-        }
-        ._sign(&Params::default(), &HashMap::new())
-        .expect("valid")
-    }
-}
-
-impl Pod for MockSignedPod {
-    fn params(&self) -> &Params {
-        panic!("MockSignedPod doesn't have params");
-    }
-    fn verify(&self) -> Result<(), Box<DynError>> {
-        Ok(self._verify()?)
-    }
-
     fn id(&self) -> PodId {
         self.id
     }
@@ -195,7 +186,7 @@ pub mod tests {
             .downcast::<MockSignedPod>()
             .unwrap();
 
-        pod._verify()?;
+        pod.verify()?;
         println!("id: {}", pod.id());
         println!("kvs: {:?}", pod.kvs());