Refactor frontend/middleware types (#194)

* unify fe/be NativeOp and NativePred

* remove Origin in favour of PodId

* Combine string and hash in Key

* use middleware::AnchoredKey in frontend

* merge frontend/middleware types

* refactor custom predicates

* clean up a bit

* fix middleware custom tests

* clean up

* clean up 2

* add acronyms in typos list

src/middleware/basetypes.rs

@@ -1,3 +1,4 @@
+// TODO: Update this doc
 //! This file exposes the backend dependent basetypes as middleware types,
 //! taking them from the feature-enabled backend.
 //!
@@ -29,12 +30,212 @@
 //! u64/i64 to F conversion. Eventually we will do those conversions through the
 //! approach described in this file, removing the imports of plonky2 in the
 //! middleware.
+//! TODO: Update this doc
 
 /// Value, Hash and F are imported based on 'features'. For example by default
 /// we use the 'plonky2' feature, but it could be used a 'plonky3' feature, so
 /// then the Value, Hash and F types would come from the plonky3 backend.
-#[cfg(feature = "backend_plonky2")]
-pub use crate::backends::plonky2::basetypes::{
-    hash_fields, hash_str, hash_value, Hash, Value, EMPTY_HASH, EMPTY_VALUE, F, HASH_SIZE,
-    SELF_ID_HASH, VALUE_SIZE,
+// #[cfg(feature = "backend_plonky2")]
+// pub use crate::backends::plonky2::basetypes::{
+//     hash_fields, hash_str, hash_value, Hash, RawValue, EMPTY_HASH, EMPTY_VALUE, F, HASH_SIZE,
+//     SELF_ID_HASH, VALUE_SIZE,
+// };
+use std::{
+    cmp::{Ord, Ordering},
+    fmt,
 };
+
+use anyhow::Result;
+use hex::{FromHex, FromHexError};
+use plonky2::{
+    field::{
+        goldilocks_field::GoldilocksField,
+        types::{Field, PrimeField64},
+    },
+    hash::poseidon::PoseidonHash,
+    plonk::config::Hasher,
+};
+
+use crate::middleware::{
+    // serialization::{
+    //     deserialize_hash_tuple, deserialize_value_tuple, serialize_hash_tuple,
+    //     serialize_value_tuple,
+    // },
+    Params,
+    ToFields,
+};
+
+/// F is the native field we use everywhere.  Currently it's Goldilocks from plonky2
+pub type F = GoldilocksField;
+
+pub const HASH_SIZE: usize = 4;
+pub const VALUE_SIZE: usize = 4;
+
+pub const EMPTY_VALUE: RawValue = RawValue([F::ZERO, F::ZERO, F::ZERO, F::ZERO]);
+pub const SELF_ID_HASH: Hash = Hash([F::ONE, F::ZERO, F::ZERO, F::ZERO]);
+pub const EMPTY_HASH: Hash = Hash([F::ZERO, F::ZERO, F::ZERO, F::ZERO]);
+
+#[derive(Clone, Copy, Debug, Default, Hash, PartialEq, Eq)]
+// #[schemars(rename = "RawValue")]
+pub struct RawValue(
+    // #[serde(
+    //     serialize_with = "serialize_value_tuple",
+    //     deserialize_with = "deserialize_value_tuple"
+    // )]
+    // We know that Serde will serialize and deserialize this as a string, so we can
+    // use the JsonSchema to validate the format.
+    // #[schemars(with = "String", regex(pattern = r"^[0-9a-fA-F]{64}$"))]
+    pub [F; VALUE_SIZE],
+);
+
+impl ToFields for RawValue {
+    fn to_fields(&self, _params: &Params) -> Vec<F> {
+        self.0.to_vec()
+    }
+}
+
+impl RawValue {
+    pub fn to_bytes(self) -> Vec<u8> {
+        self.0
+            .iter()
+            .flat_map(|e| e.to_canonical_u64().to_le_bytes())
+            .collect()
+    }
+}
+
+impl Ord for RawValue {
+    fn cmp(&self, other: &Self) -> Ordering {
+        for (lhs, rhs) in self.0.iter().zip(other.0.iter()).rev() {
+            let (lhs, rhs) = (lhs.to_canonical_u64(), rhs.to_canonical_u64());
+            match lhs.cmp(&rhs) {
+                Ordering::Less => return Ordering::Less,
+                Ordering::Greater => return Ordering::Greater,
+                _ => {}
+            }
+        }
+        Ordering::Equal
+    }
+}
+
+impl PartialOrd for RawValue {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl From<i64> for RawValue {
+    fn from(v: i64) -> Self {
+        let lo = F::from_canonical_u64((v as u64) & 0xffffffff);
+        let hi = F::from_canonical_u64((v as u64) >> 32);
+        RawValue([lo, hi, F::ZERO, F::ZERO])
+    }
+}
+
+impl From<Hash> for RawValue {
+    fn from(h: Hash) -> Self {
+        RawValue(h.0)
+    }
+}
+
+impl fmt::Display for RawValue {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        if self.0[2].is_zero() && self.0[3].is_zero() {
+            // Assume this is an integer
+            let (l0, l1) = (self.0[0].to_canonical_u64(), self.0[1].to_canonical_u64());
+            assert!(l0 < (1 << 32));
+            assert!(l1 < (1 << 32));
+            write!(f, "{}", l0 + l1 * (1 << 32))
+        } else {
+            // Assume this is a hash
+            Hash(self.0).fmt(f)
+        }
+    }
+}
+
+#[derive(Clone, Copy, Debug, Default, Hash, Eq, PartialEq)]
+pub struct Hash(
+    // #[serde(
+    //     serialize_with = "serialize_hash_tuple",
+    //     deserialize_with = "deserialize_hash_tuple"
+    // )]
+    // #[schemars(with = "String", regex(pattern = r"^[0-9a-fA-F]{64}$"))]
+    pub [F; HASH_SIZE],
+);
+
+pub fn hash_value(input: &RawValue) -> Hash {
+    hash_fields(&input.0)
+}
+
+pub fn hash_fields(input: &[F]) -> Hash {
+    Hash(PoseidonHash::hash_no_pad(input).elements)
+}
+
+impl From<RawValue> for Hash {
+    fn from(v: RawValue) -> Self {
+        Hash(v.0)
+    }
+}
+
+impl ToFields for Hash {
+    fn to_fields(&self, _params: &Params) -> Vec<F> {
+        self.0.to_vec()
+    }
+}
+
+impl PartialOrd for Hash {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl Ord for Hash {
+    fn cmp(&self, other: &Self) -> Ordering {
+        RawValue(self.0).cmp(&RawValue(other.0))
+    }
+}
+
+impl fmt::Display for Hash {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let v0 = self.0[0].to_canonical_u64();
+        for i in 0..HASH_SIZE {
+            write!(f, "{:02x}", (v0 >> (i * 8)) & 0xff)?;
+        }
+        write!(f, "…")
+    }
+}
+
+impl FromHex for Hash {
+    type Error = FromHexError;
+
+    // TODO make it dependant on backend::Value len
+    fn from_hex<T: AsRef<[u8]>>(hex: T) -> Result<Self, Self::Error> {
+        // In little endian
+        let bytes = <[u8; 32]>::from_hex(hex)?;
+        let mut buf: [u8; 8] = [0; 8];
+        let mut inner = [F::ZERO; HASH_SIZE];
+        for i in 0..HASH_SIZE {
+            buf.copy_from_slice(&bytes[8 * i..8 * (i + 1)]);
+            inner[i] = F::from_canonical_u64(u64::from_le_bytes(buf));
+        }
+        Ok(Self(inner))
+    }
+}
+
+pub fn hash_str(s: &str) -> Hash {
+    let mut input = s.as_bytes().to_vec();
+    input.push(1); // padding
+
+    // Merge 7 bytes into 1 field, because the field is slightly below 64 bits
+    let input: Vec<F> = input
+        .chunks(7)
+        .map(|bytes| {
+            let mut v: u64 = 0;
+            for b in bytes.iter().rev() {
+                v <<= 8;
+                v += *b as u64;
+            }
+            F::from_canonical_u64(v)
+        })
+        .collect();
+    hash_fields(&input)
+}

src/middleware/containers.rs

@@ -1,14 +1,14 @@
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 
 /// This file implements the types defined at
 /// https://0xparc.github.io/pod2/values.html#dictionary-array-set .
-use anyhow::Result;
+use anyhow::{anyhow, Result};
 
 #[cfg(feature = "backend_plonky2")]
-use crate::backends::plonky2::primitives::merkletree::{Iter as TreeIter, MerkleProof, MerkleTree};
+use crate::backends::plonky2::primitives::merkletree::{MerkleProof, MerkleTree};
 use crate::{
     constants::MAX_DEPTH,
-    middleware::basetypes::{hash_value, Hash, Value, EMPTY_VALUE},
+    middleware::{hash_value, Hash, Key, RawValue, Value, EMPTY_VALUE},
 };
 
 /// Dictionary: the user original keys and values are hashed to be used in the leaf.
@@ -16,47 +16,58 @@ use crate::{
 ///    leaf.value=hash(original_value)
 #[derive(Clone, Debug)]
 pub struct Dictionary {
-    // exposed with pub(crate) so that it can be modified at tests
-    pub(crate) mt: MerkleTree,
+    mt: MerkleTree,
+    kvs: HashMap<Key, Value>,
 }
 
 impl Dictionary {
-    pub fn new(kvs: &HashMap<Hash, Value>) -> Result<Self> {
-        let kvs: HashMap<Value, Value> = kvs.iter().map(|(&k, &v)| (Value(k.0), v)).collect();
+    pub fn new(kvs: HashMap<Key, Value>) -> Result<Self> {
+        let kvs_raw: HashMap<RawValue, RawValue> = kvs
+            .iter()
+            .map(|(k, v)| (RawValue(k.hash().0), v.raw()))
+            .collect();
         Ok(Self {
-            mt: MerkleTree::new(MAX_DEPTH, &kvs)?,
+            mt: MerkleTree::new(MAX_DEPTH, &kvs_raw)?,
+            kvs,
         })
     }
     pub fn commitment(&self) -> Hash {
         self.mt.root()
     }
-    pub fn get(&self, key: &Value) -> Result<Value> {
-        self.mt.get(key)
+    pub fn get(&self, key: &Key) -> Result<&Value> {
+        self.kvs
+            .get(key)
+            .ok_or_else(|| anyhow!("key \"{}\" not found", key.name()))
     }
-    pub fn prove(&self, key: &Value) -> Result<(Value, MerkleProof)> {
-        self.mt.prove(key)
+    pub fn prove(&self, key: &Key) -> Result<(&Value, MerkleProof)> {
+        let (_, mtp) = self.mt.prove(&RawValue(key.hash().0))?;
+        let value = self.kvs.get(key).expect("key exists");
+        Ok((value, mtp))
     }
-    pub fn prove_nonexistence(&self, key: &Value) -> Result<MerkleProof> {
-        self.mt.prove_nonexistence(key)
+    pub fn prove_nonexistence(&self, key: &Key) -> Result<MerkleProof> {
+        self.mt.prove_nonexistence(&RawValue(key.hash().0))
     }
-    pub fn verify(root: Hash, proof: &MerkleProof, key: &Value, value: &Value) -> Result<()> {
-        MerkleTree::verify(MAX_DEPTH, root, proof, key, value)
+    pub fn verify(root: Hash, proof: &MerkleProof, key: &Key, value: &Value) -> Result<()> {
+        let key = RawValue(key.hash().0);
+        MerkleTree::verify(MAX_DEPTH, root, proof, &key, &value.raw())
     }
-    pub fn verify_nonexistence(root: Hash, proof: &MerkleProof, key: &Value) -> Result<()> {
-        MerkleTree::verify_nonexistence(MAX_DEPTH, root, proof, key)
+    pub fn verify_nonexistence(root: Hash, proof: &MerkleProof, key: &Key) -> Result<()> {
+        let key = RawValue(key.hash().0);
+        MerkleTree::verify_nonexistence(MAX_DEPTH, root, proof, &key)
     }
-    pub fn iter(&self) -> TreeIter {
-        self.mt.iter()
-    }
-}
-impl<'a> IntoIterator for &'a Dictionary {
-    type Item = (&'a Value, &'a Value);
-    type IntoIter = TreeIter<'a>;
-
-    fn into_iter(self) -> Self::IntoIter {
-        self.mt.iter()
+    // TODO: Rename to dict to be consistent maybe?
+    pub fn kvs(&self) -> &HashMap<Key, Value> {
+        &self.kvs
     }
 }
+// impl<'a> IntoIterator for &'a Dictionary {
+//     type Item = (&'a RawValue, &'a RawValue);
+//     type IntoIter = TreeIter<'a>;
+//
+//     fn into_iter(self) -> Self::IntoIter {
+//         self.mt.iter()
+//     }
+// }
 
 impl PartialEq for Dictionary {
     fn eq(&self, other: &Self) -> bool {
@@ -71,42 +82,48 @@ impl Eq for Dictionary {}
 #[derive(Clone, Debug)]
 pub struct Set {
     mt: MerkleTree,
+    set: HashSet<Value>,
 }
 
 impl Set {
-    pub fn new(set: &[Value]) -> Result<Self> {
-        let kvs: HashMap<Value, Value> = set
+    pub fn new(set: HashSet<Value>) -> Result<Self> {
+        let kvs_raw: HashMap<RawValue, RawValue> = set
             .iter()
             .map(|e| {
-                let h = hash_value(e);
-                (Value::from(h), EMPTY_VALUE)
+                let h = hash_value(&e.raw());
+                (RawValue::from(h), EMPTY_VALUE)
             })
             .collect();
         Ok(Self {
-            mt: MerkleTree::new(MAX_DEPTH, &kvs)?,
+            mt: MerkleTree::new(MAX_DEPTH, &kvs_raw)?,
+            set,
         })
     }
     pub fn commitment(&self) -> Hash {
         self.mt.root()
     }
-    pub fn contains(&self, value: &Value) -> Result<bool> {
-        self.mt.contains(value)
+    pub fn contains(&self, value: &Value) -> bool {
+        self.set.contains(value)
     }
     pub fn prove(&self, value: &Value) -> Result<MerkleProof> {
-        let (_, proof) = self.mt.prove(value)?;
+        let h = hash_value(&value.raw());
+        let (_, proof) = self.mt.prove(&RawValue::from(h))?;
         Ok(proof)
     }
     pub fn prove_nonexistence(&self, value: &Value) -> Result<MerkleProof> {
-        self.mt.prove_nonexistence(value)
+        let h = hash_value(&value.raw());
+        self.mt.prove_nonexistence(&RawValue::from(h))
     }
     pub fn verify(root: Hash, proof: &MerkleProof, value: &Value) -> Result<()> {
-        MerkleTree::verify(MAX_DEPTH, root, proof, value, &EMPTY_VALUE)
+        let h = hash_value(&value.raw());
+        MerkleTree::verify(MAX_DEPTH, root, proof, &RawValue::from(h), &EMPTY_VALUE)
     }
     pub fn verify_nonexistence(root: Hash, proof: &MerkleProof, value: &Value) -> Result<()> {
-        MerkleTree::verify_nonexistence(MAX_DEPTH, root, proof, value)
+        let h = hash_value(&value.raw());
+        MerkleTree::verify_nonexistence(MAX_DEPTH, root, proof, &RawValue::from(h))
     }
-    pub fn iter(&self) -> TreeIter {
-        self.mt.iter()
+    pub fn set(&self) -> &HashSet<Value> {
+        &self.set
     }
 }
 
@@ -124,34 +141,46 @@ impl Eq for Set {}
 #[derive(Clone, Debug)]
 pub struct Array {
     mt: MerkleTree,
+    array: Vec<Value>,
 }
 
 impl Array {
-    pub fn new(array: &[Value]) -> Result<Self> {
-        let kvs: HashMap<Value, Value> = array
+    pub fn new(array: Vec<Value>) -> Result<Self> {
+        let kvs_raw: HashMap<RawValue, RawValue> = array
             .iter()
             .enumerate()
-            .map(|(i, &e)| (Value::from(i as i64), e))
+            .map(|(i, e)| (RawValue::from(i as i64), e.raw()))
             .collect();
 
         Ok(Self {
-            mt: MerkleTree::new(MAX_DEPTH, &kvs)?,
+            mt: MerkleTree::new(MAX_DEPTH, &kvs_raw)?,
+            array,
         })
     }
     pub fn commitment(&self) -> Hash {
         self.mt.root()
     }
-    pub fn get(&self, i: usize) -> Result<Value> {
-        self.mt.get(&Value::from(i as i64))
+    pub fn get(&self, i: usize) -> Result<&Value> {
+        self.array
+            .get(i)
+            .ok_or_else(|| anyhow!("index {} out of bounds 0..{}", i, self.array.len()))
     }
-    pub fn prove(&self, i: usize) -> Result<(Value, MerkleProof)> {
-        self.mt.prove(&Value::from(i as i64))
+    pub fn prove(&self, i: usize) -> Result<(&Value, MerkleProof)> {
+        let (_, mtp) = self.mt.prove(&RawValue::from(i as i64))?;
+        let value = self.array.get(i).expect("valid index");
+        Ok((value, mtp))
     }
     pub fn verify(root: Hash, proof: &MerkleProof, i: usize, value: &Value) -> Result<()> {
-        MerkleTree::verify(MAX_DEPTH, root, proof, &Value::from(i as i64), value)
+        MerkleTree::verify(
+            MAX_DEPTH,
+            root,
+            proof,
+            &RawValue::from(i as i64),
+            &value.raw(),
+        )
     }
-    pub fn iter(&self) -> TreeIter {
-        self.mt.iter()
+    pub fn array(&self) -> &[Value] {
+        &self.array
     }
 }
 

src/middleware/custom.rs

@@ -1,101 +1,85 @@
-use std::{collections::HashMap, fmt, hash as h, iter, iter::zip, sync::Arc};
+use std::{fmt, iter, sync::Arc};
 
 use anyhow::{anyhow, Result};
 use plonky2::field::types::Field;
-use schemars::JsonSchema;
-use serde::{Deserialize, Serialize};
 
+// use schemars::JsonSchema;
+
+// use serde::{Deserialize, Serialize};
 use crate::{
-    backends::plonky2::basetypes::HASH_SIZE,
-    middleware::{
-        hash_fields, AnchoredKey, Hash, NativePredicate, Params, PodId, Statement, StatementArg,
-        ToFields, Value, F,
-    },
-    util::hashmap_insert_no_dupe,
+    middleware::HASH_SIZE,
+    middleware::{hash_fields, Hash, Key, NativePredicate, Params, ToFields, Value, F},
 };
 
-// BEGIN Custom 1b
-
-#[derive(Clone, Debug, PartialEq, Eq, h::Hash, Serialize, Deserialize, JsonSchema)]
-pub enum HashOrWildcard {
-    Hash(Hash),
-    Wildcard(usize),
+#[derive(Clone, Debug, PartialEq)]
+pub struct Wildcard {
+    pub name: String,
+    pub index: usize,
 }
 
-impl HashOrWildcard {
-    /// Matches a hash or wildcard against a value, returning a pair
-    /// representing a wildcard binding (if any) or an error if no
-    /// match is possible.
-    pub fn match_against(&self, v: &Value) -> Result<Option<(usize, Value)>> {
-        match self {
-            HashOrWildcard::Hash(h) if &Value::from(*h) == v => Ok(None),
-            HashOrWildcard::Wildcard(i) => Ok(Some((*i, *v))),
-            _ => Err(anyhow!(
-                "Failed to match hash or wildcard {} against value {}.",
-                self,
-                v
-            )),
-        }
+impl Wildcard {
+    pub fn new(name: String, index: usize) -> Self {
+        Self { name, index }
     }
 }
 
-impl fmt::Display for HashOrWildcard {
+impl fmt::Display for Wildcard {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "*{}[{}]", self.index, self.name)
+    }
+}
+
+impl ToFields for Wildcard {
+    fn to_fields(&self, _params: &Params) -> Vec<F> {
+        vec![F::from_canonical_u64(self.index as u64)]
+    }
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub enum KeyOrWildcard {
+    Key(Key),
+    Wildcard(Wildcard),
+}
+
+impl fmt::Display for KeyOrWildcard {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
-            Self::Hash(h) => write!(f, "{}", h),
-            Self::Wildcard(n) => write!(f, "*{}", n),
+            Self::Key(k) => write!(f, "{}", k),
+            Self::Wildcard(wc) => write!(f, "{}", wc),
         }
     }
 }
 
-impl ToFields for HashOrWildcard {
+impl ToFields for KeyOrWildcard {
     fn to_fields(&self, params: &Params) -> Vec<F> {
         match self {
-            HashOrWildcard::Hash(h) => h.to_fields(params),
-            HashOrWildcard::Wildcard(w) => (0..HASH_SIZE - 1)
-                .chain(iter::once(*w))
-                .map(|x| F::from_canonical_u64(x as u64))
+            KeyOrWildcard::Key(k) => k.hash().to_fields(params),
+            KeyOrWildcard::Wildcard(wc) => iter::once(F::ZERO)
+                .take(HASH_SIZE - 1)
+                .chain(iter::once(F::from_canonical_u64(wc.index as u64)))
                 .collect(),
         }
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, h::Hash, Serialize, Deserialize, JsonSchema)]
+#[derive(Clone, Debug, PartialEq)]
 pub enum StatementTmplArg {
     None,
     Literal(Value),
-    Key(HashOrWildcard, HashOrWildcard),
-}
-
-impl StatementTmplArg {
-    /// Matches a statement template argument against a statement
-    /// argument, returning a wildcard correspondence in the case of
-    /// one or more wildcard matches, nothing in the case of a
-    /// literal/hash match, and an error otherwise.
-    pub fn match_against(&self, s_arg: &StatementArg) -> Result<Vec<(usize, Value)>> {
-        match (self, s_arg) {
-            (Self::None, StatementArg::None) => Ok(vec![]),
-            (Self::Literal(v), StatementArg::Literal(w)) if v == w => Ok(vec![]),
-            (Self::Key(tmpl_o, tmpl_k), StatementArg::Key(AnchoredKey(PodId(o), k))) => {
-                let o_corr = tmpl_o.match_against(&(*o).into())?;
-                let k_corr = tmpl_k.match_against(&(*k).into())?;
-                Ok([o_corr, k_corr].into_iter().flatten().collect())
-            }
-            _ => Err(anyhow!(
-                "Failed to match statement template argument {:?} against statement argument {:?}.",
-                self,
-                s_arg
-            )),
-        }
-    }
+    // AnchoredKey
+    Key(Wildcard, KeyOrWildcard),
+    // TODO: This naming is a bit confusing: a WildcardLiteral that contains a Wildcard...
+    // Could we merge WildcardValue and Value and allow wildcard value apart from pod_id and key?
+    WildcardLiteral(Wildcard),
 }
 
 impl ToFields for StatementTmplArg {
     fn to_fields(&self, params: &Params) -> Vec<F> {
         // None => (0, ...)
         // Literal(value) => (1, [value], 0, 0, 0, 0)
-        // Key(hash_or_wildcard1, hash_or_wildcard2)
-        //    => (2, [hash_or_wildcard1], [hash_or_wildcard2])
+        // Key(wildcard1, key_or_wildcard2)
+        //    => (2, [wildcard1], [key_or_wildcard2])
+        // WildcardLiteral(wildcard) => (3, [wildcard], 0, 0, 0, 0)
         // In all three cases, we pad to 2 * hash_size + 1 = 9 field elements
         let statement_tmpl_arg_size = 2 * HASH_SIZE + 1;
         match self {
@@ -107,15 +91,22 @@ impl ToFields for StatementTmplArg {
             }
             StatementTmplArg::Literal(v) => {
                 let fields: Vec<F> = iter::once(F::from_canonical_u64(1))
-                    .chain(v.to_fields(params))
+                    .chain(v.raw().to_fields(params))
                     .chain(iter::repeat_with(|| F::from_canonical_u64(0)).take(HASH_SIZE))
                     .collect();
                 fields
             }
-            StatementTmplArg::Key(hw1, hw2) => {
+            StatementTmplArg::Key(wc1, kw2) => {
                 let fields: Vec<F> = iter::once(F::from_canonical_u64(2))
-                    .chain(hw1.to_fields(params))
-                    .chain(hw2.to_fields(params))
+                    .chain(wc1.to_fields(params))
+                    .chain(kw2.to_fields(params))
+                    .collect();
+                fields
+            }
+            StatementTmplArg::WildcardLiteral(wc) => {
+                let fields: Vec<F> = iter::once(F::from_canonical_u64(3))
+                    .chain(wc.to_fields(params))
+                    .chain(iter::repeat_with(|| F::from_canonical_u64(0)).take(HASH_SIZE))
                     .collect();
                 fields
             }
@@ -129,50 +120,37 @@ impl fmt::Display for StatementTmplArg {
             Self::None => write!(f, "none"),
             Self::Literal(v) => write!(f, "{}", v),
             Self::Key(pod_id, key) => write!(f, "({}, {})", pod_id, key),
+            Self::WildcardLiteral(v) => write!(f, "{}", v),
         }
     }
 }
 
-// END
-
-// BEGIN Custom 2
-
-// pub enum StatementTmplArg {
-//     None,
-//     Literal(Value),
-//     Wildcard(usize),
-// }
-
-// END
-
 /// Statement Template for a Custom Predicate
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
-pub struct StatementTmpl(pub Predicate, pub Vec<StatementTmplArg>);
+#[derive(Clone, Debug, PartialEq)]
+pub struct StatementTmpl {
+    pub pred: Predicate,
+    pub args: Vec<StatementTmplArg>,
+}
 
 impl StatementTmpl {
     pub fn pred(&self) -> &Predicate {
-        &self.0
+        &self.pred
     }
     pub fn args(&self) -> &[StatementTmplArg] {
-        &self.1
+        &self.args
     }
-    /// Matches a statement template against a statement, returning
-    /// the variable bindings as an association list. Returns an error
-    /// if there is type or argument mismatch.
-    pub fn match_against(&self, s: &Statement) -> Result<Vec<(usize, Value)>> {
-        type P = Predicate;
-        if matches!(self, Self(P::BatchSelf(_), _)) {
-            Err(anyhow!(
-                "Cannot check self-referencing statement templates."
-            ))
-        } else if self.pred() != &s.predicate() {
-            Err(anyhow!("Type mismatch between {:?} and {}.", self, s))
-        } else {
-            zip(self.args(), s.args())
-                .map(|(t_arg, s_arg)| t_arg.match_against(&s_arg))
-                .collect::<Result<Vec<_>>>()
-                .map(|v| v.concat())
+}
+
+impl fmt::Display for StatementTmpl {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}(", self.pred)?;
+        for (i, arg) in self.args.iter().enumerate() {
+            if i != 0 {
+                write!(f, ", ")?;
+            }
+            write!(f, "{}", arg)?;
         }
+        writeln!(f)
     }
 }
 
@@ -185,25 +163,26 @@ impl ToFields for StatementTmpl {
         // TODO think if this check should go into the StatementTmpl creation,
         // instead of at the `to_fields` method, where we should assume that the
         // values are already valid
-        if self.1.len() > params.max_statement_args {
+        if self.args.len() > params.max_statement_args {
             panic!("Statement template has too many arguments");
         }
 
         let mut fields: Vec<F> = self
-            .0
+            .pred
             .to_fields(params)
             .into_iter()
-            .chain(self.1.iter().flat_map(|sta| sta.to_fields(params)))
+            .chain(self.args.iter().flat_map(|sta| sta.to_fields(params)))
             .collect();
         fields.resize_with(params.statement_tmpl_size(), || F::from_canonical_u64(0));
         fields
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
+#[derive(Clone, Debug, PartialEq)]
 /// NOTE: fields are not public (outside of crate) to enforce the struct instantiation through
 /// the `::and/or` methods, which performs checks on the values.
 pub struct CustomPredicate {
+    pub name: String, // Non-cryptographic metadata
     /// true for "and", false for "or"
     pub(crate) conjunction: bool,
     pub(crate) statements: Vec<StatementTmpl>,
@@ -213,13 +192,24 @@ pub struct CustomPredicate {
 }
 
 impl CustomPredicate {
-    pub fn and(params: &Params, statements: Vec<StatementTmpl>, args_len: usize) -> Result<Self> {
-        Self::new(params, true, statements, args_len)
+    pub fn and(
+        name: String,
+        params: &Params,
+        statements: Vec<StatementTmpl>,
+        args_len: usize,
+    ) -> Result<Self> {
+        Self::new(name, params, true, statements, args_len)
     }
-    pub fn or(params: &Params, statements: Vec<StatementTmpl>, args_len: usize) -> Result<Self> {
-        Self::new(params, false, statements, args_len)
+    pub fn or(
+        name: String,
+        params: &Params,
+        statements: Vec<StatementTmpl>,
+        args_len: usize,
+    ) -> Result<Self> {
+        Self::new(name, params, false, statements, args_len)
     }
     pub fn new(
+        name: String,
         params: &Params,
         conjunction: bool,
         statements: Vec<StatementTmpl>,
@@ -230,6 +220,7 @@ impl CustomPredicate {
         }
 
         Ok(Self {
+            name,
             conjunction,
             statements,
             args_len,
@@ -266,8 +257,8 @@ impl fmt::Display for CustomPredicate {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         writeln!(f, "{}<", if self.conjunction { "and" } else { "or" })?;
         for st in &self.statements {
-            write!(f, "  {}", st.0)?;
-            for (i, arg) in st.1.iter().enumerate() {
+            write!(f, "  {}(", st.pred)?;
+            for (i, arg) in st.args.iter().enumerate() {
                 if i != 0 {
                     write!(f, ", ")?;
                 }
@@ -287,7 +278,7 @@ impl fmt::Display for CustomPredicate {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
+#[derive(Clone, Debug, PartialEq)]
 pub struct CustomPredicateBatch {
     pub name: String,
     pub predicates: Vec<CustomPredicate>,
@@ -324,67 +315,23 @@ impl CustomPredicateBatch {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
-pub struct CustomPredicateRef(pub Arc<CustomPredicateBatch>, pub usize);
+#[derive(Clone, Debug, PartialEq)]
+pub struct CustomPredicateRef {
+    pub batch: Arc<CustomPredicateBatch>,
+    pub index: usize,
+}
 
 impl CustomPredicateRef {
-    pub fn arg_len(&self) -> usize {
-        self.0.predicates[self.1].args_len
+    pub fn new(batch: Arc<CustomPredicateBatch>, index: usize) -> Self {
+        Self { batch, index }
     }
-    pub fn match_against(&self, statements: &[Statement]) -> Result<HashMap<usize, Value>> {
-        let mut bindings = HashMap::new();
-        // Single out custom predicate, replacing batch-self
-        // references with custom predicate references.
-        let custom_predicate = {
-            let cp = &Arc::unwrap_or_clone(self.0.clone()).predicates[self.1];
-            CustomPredicate {
-                conjunction: cp.conjunction,
-                statements: cp
-                    .statements
-                    .iter()
-                    .map(|StatementTmpl(p, args)| {
-                        StatementTmpl(
-                            match p {
-                                Predicate::BatchSelf(i) => {
-                                    Predicate::Custom(CustomPredicateRef(self.0.clone(), *i))
-                                }
-                                _ => p.clone(),
-                            },
-                            args.to_vec(),
-                        )
-                    })
-                    .collect(),
-                args_len: cp.args_len,
-            }
-        };
-        match custom_predicate.conjunction {
-                    true if custom_predicate.statements.len() == statements.len() => {
-                        // Match op args against statement templates
-                    let match_bindings = iter::zip(custom_predicate.statements, statements).map(
-                        |(s_tmpl, s)| s_tmpl.match_against(s)
-                    ).collect::<Result<Vec<_>>>()
-                        .map(|v| v.concat())?;
-                    // Add bindings to binding table, throwing if there is an inconsistency.
-                    match_bindings.into_iter().try_for_each(|kv| hashmap_insert_no_dupe(&mut bindings, kv))?;
-                    Ok(bindings)
-                    },
-                    false if statements.len() == 1 => {
-                        // Match op arg against each statement template
-                        custom_predicate.statements.iter().map(
-                            |s_tmpl| {
-                                let mut bindings = bindings.clone();
-                                s_tmpl.match_against(&statements[0])?.into_iter().try_for_each(|kv| hashmap_insert_no_dupe(&mut bindings, kv))?;
-                                Ok::<_, anyhow::Error>(bindings)
-                            }
-                        ).find(|m| m.is_ok()).unwrap_or(Err(anyhow!("Statement {} does not match disjunctive custom predicate {}.", &statements[0], custom_predicate)))
-                    },
-                    _ =>                     Err(anyhow!("Custom predicate statement template list {:?} does not match op argument list {:?}.", custom_predicate.statements, statements))
-                }
+    pub fn arg_len(&self) -> usize {
+        self.batch.predicates[self.index].args_len
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
-#[serde(tag = "type", content = "value")]
+#[derive(Clone, Debug, PartialEq)]
+// #[serde(tag = "type", content = "value")]
 pub enum Predicate {
     Native(NativePredicate),
     BatchSelf(usize),
@@ -414,10 +361,12 @@ impl ToFields for Predicate {
             Self::BatchSelf(i) => iter::once(F::from_canonical_u64(2))
                 .chain(iter::once(F::from_canonical_usize(*i)))
                 .collect(),
-            Self::Custom(CustomPredicateRef(pb, i)) => iter::once(F::from_canonical_u64(3))
-                .chain(pb.hash(params).0)
-                .chain(iter::once(F::from_canonical_usize(*i)))
-                .collect(),
+            Self::Custom(CustomPredicateRef { batch, index }) => {
+                iter::once(F::from_canonical_u64(3))
+                    .chain(batch.hash(params).0)
+                    .chain(iter::once(F::from_canonical_usize(*index)))
+                    .collect()
+            }
         };
         fields.resize_with(Params::predicate_size(), || F::from_canonical_u64(0));
         fields
@@ -429,7 +378,13 @@ impl fmt::Display for Predicate {
         match self {
             Self::Native(p) => write!(f, "{:?}", p),
             Self::BatchSelf(i) => write!(f, "self.{}", i),
-            Self::Custom(CustomPredicateRef(pb, i)) => write!(f, "{}.{}", pb.name, i),
+            Self::Custom(CustomPredicateRef { batch, index }) => {
+                write!(
+                    f,
+                    "{}.{}[{}]",
+                    batch.name, index, batch.predicates[*index].name
+                )
+            }
         }
     }
 }
@@ -441,18 +396,29 @@ mod tests {
     use anyhow::Result;
     use plonky2::field::goldilocks_field::GoldilocksField;
 
+    use super::*;
     use crate::middleware::{
         AnchoredKey, CustomPredicate, CustomPredicateBatch, CustomPredicateRef, Hash,
-        HashOrWildcard, NativePredicate, Operation, Params, PodId, PodType, Predicate, Statement,
-        StatementTmpl, StatementTmplArg, SELF,
+        KeyOrWildcard, NativePredicate, Operation, Params, PodId, PodType, Predicate, Statement,
+        StatementTmpl, StatementTmplArg, WildcardValue, SELF,
     };
 
     fn st(p: Predicate, args: Vec<StatementTmplArg>) -> StatementTmpl {
-        StatementTmpl(p, args)
+        StatementTmpl { pred: p, args }
+    }
+
+    fn kow_wc(i: usize) -> KOW {
+        KOW::Wildcard(wc(i))
+    }
+    fn wc(i: usize) -> Wildcard {
+        Wildcard {
+            name: format!("{}", i),
+            index: i,
+        }
     }
 
     type STA = StatementTmplArg;
-    type HOW = HashOrWildcard;
+    type KOW = KeyOrWildcard;
     type P = Predicate;
     type NP = NativePredicate;
 
@@ -468,44 +434,42 @@ mod tests {
         let cust_pred_batch = Arc::new(CustomPredicateBatch {
             name: "is_double".to_string(),
             predicates: vec![CustomPredicate::and(
+                "_".into(),
                 &params,
                 vec![
                     st(
                         P::Native(NP::ValueOf),
-                        vec![
-                            STA::Key(HOW::Wildcard(4), HOW::Wildcard(5)),
-                            STA::Literal(2.into()),
-                        ],
+                        vec![STA::Key(wc(4), kow_wc(5)), STA::Literal(2.into())],
                     ),
                     st(
                         P::Native(NP::ProductOf),
                         vec![
-                            STA::Key(HOW::Wildcard(0), HOW::Wildcard(1)),
-                            STA::Key(HOW::Wildcard(4), HOW::Wildcard(5)),
-                            STA::Key(HOW::Wildcard(2), HOW::Wildcard(3)),
+                            STA::Key(wc(0), kow_wc(1)),
+                            STA::Key(wc(4), kow_wc(5)),
+                            STA::Key(wc(2), kow_wc(3)),
                         ],
                     ),
                 ],
-                4,
+                2,
             )?],
         });
 
         let custom_statement = Statement::Custom(
-            CustomPredicateRef(cust_pred_batch.clone(), 0),
+            CustomPredicateRef::new(cust_pred_batch.clone(), 0),
             vec![
-                AnchoredKey(SELF, "Some value".into()),
-                AnchoredKey(SELF, "Some other value".into()),
+                WildcardValue::PodId(SELF),
+                WildcardValue::Key(Key::from("Some value")),
             ],
         );
 
         let custom_deduction = Operation::Custom(
-            CustomPredicateRef(cust_pred_batch, 0),
+            CustomPredicateRef::new(cust_pred_batch, 0),
             vec![
-                Statement::ValueOf(AnchoredKey(SELF, "Some constant".into()), 2.into()),
+                Statement::ValueOf(AnchoredKey::from((SELF, "Some constant")), 2.into()),
                 Statement::ProductOf(
-                    AnchoredKey(SELF, "Some value".into()),
-                    AnchoredKey(SELF, "Some constant".into()),
-                    AnchoredKey(SELF, "Some other value".into()),
+                    AnchoredKey::from((SELF, "Some value")),
+                    AnchoredKey::from((SELF, "Some constant")),
+                    AnchoredKey::from((SELF, "Some other value")),
                 ),
             ],
         );
@@ -517,30 +481,34 @@ mod tests {
 
     #[test]
     fn ethdos_test() -> Result<()> {
-        let params = Params::default();
+        let params = Params {
+            max_custom_predicate_wildcards: 12,
+            ..Default::default()
+        };
 
         let eth_friend_cp = CustomPredicate::and(
+            "eth_friend_cp".into(),
             &params,
             vec![
                 st(
                     P::Native(NP::ValueOf),
                     vec![
-                        STA::Key(HOW::Wildcard(4), HashOrWildcard::Hash("type".into())),
+                        STA::Key(wc(4), KeyOrWildcard::Key("type".into())),
                         STA::Literal(PodType::Signed.into()),
                     ],
                 ),
                 st(
                     P::Native(NP::Equal),
                     vec![
-                        STA::Key(HOW::Wildcard(4), HashOrWildcard::Hash("signer".into())),
-                        STA::Key(HOW::Wildcard(0), HOW::Wildcard(1)),
+                        STA::Key(wc(4), KeyOrWildcard::Key("signer".into())),
+                        STA::Key(wc(0), kow_wc(1)),
                     ],
                 ),
                 st(
                     P::Native(NP::Equal),
                     vec![
-                        STA::Key(HOW::Wildcard(4), HashOrWildcard::Hash("attestation".into())),
-                        STA::Key(HOW::Wildcard(2), HOW::Wildcard(3)),
+                        STA::Key(wc(4), KeyOrWildcard::Key("attestation".into())),
+                        STA::Key(wc(2), kow_wc(3)),
                     ],
                 ),
             ],
@@ -552,81 +520,89 @@ mod tests {
             predicates: vec![eth_friend_cp],
         });
 
+        // 0
         let eth_dos_base = CustomPredicate::and(
+            "eth_dos_base".into(),
             &params,
             vec![
                 st(
                     P::Native(NP::Equal),
-                    vec![
-                        STA::Key(HOW::Wildcard(0), HOW::Wildcard(1)),
-                        STA::Key(HOW::Wildcard(2), HOW::Wildcard(3)),
-                    ],
+                    vec![STA::Key(wc(0), kow_wc(1)), STA::Key(wc(2), kow_wc(3))],
                 ),
                 st(
                     P::Native(NP::ValueOf),
-                    vec![
-                        STA::Key(HOW::Wildcard(4), HOW::Wildcard(5)),
-                        STA::Literal(0.into()),
-                    ],
+                    vec![STA::Key(wc(4), kow_wc(5)), STA::Literal(0.into())],
                 ),
             ],
             6,
         )?;
 
+        // 1
         let eth_dos_ind = CustomPredicate::and(
+            "eth_dos_ind".into(),
             &params,
             vec![
                 st(
                     P::BatchSelf(2),
                     vec![
-                        STA::Key(HOW::Wildcard(0), HOW::Wildcard(1)),
-                        STA::Key(HOW::Wildcard(10), HOW::Wildcard(11)),
-                        STA::Key(HOW::Wildcard(8), HOW::Wildcard(9)),
+                        STA::WildcardLiteral(wc(0)),
+                        STA::WildcardLiteral(wc(1)),
+                        STA::WildcardLiteral(wc(10)),
+                        STA::WildcardLiteral(wc(11)),
+                        STA::WildcardLiteral(wc(8)),
+                        STA::WildcardLiteral(wc(9)),
                     ],
                 ),
                 st(
                     P::Native(NP::ValueOf),
-                    vec![
-                        STA::Key(HOW::Wildcard(6), HOW::Wildcard(7)),
-                        STA::Literal(1.into()),
-                    ],
+                    vec![STA::Key(wc(6), kow_wc(7)), STA::Literal(1.into())],
                 ),
                 st(
                     P::Native(NP::SumOf),
                     vec![
-                        STA::Key(HOW::Wildcard(4), HOW::Wildcard(5)),
-                        STA::Key(HOW::Wildcard(8), HOW::Wildcard(9)),
-                        STA::Key(HOW::Wildcard(6), HOW::Wildcard(7)),
+                        STA::Key(wc(4), kow_wc(5)),
+                        STA::Key(wc(8), kow_wc(9)),
+                        STA::Key(wc(6), kow_wc(7)),
                     ],
                 ),
                 st(
-                    P::Custom(CustomPredicateRef(eth_friend_batch.clone(), 0)),
+                    P::Custom(CustomPredicateRef::new(eth_friend_batch.clone(), 0)),
                     vec![
-                        STA::Key(HOW::Wildcard(10), HOW::Wildcard(11)),
-                        STA::Key(HOW::Wildcard(2), HOW::Wildcard(3)),
+                        STA::WildcardLiteral(wc(10)),
+                        STA::WildcardLiteral(wc(11)),
+                        STA::WildcardLiteral(wc(2)),
+                        STA::WildcardLiteral(wc(3)),
                     ],
                 ),
             ],
             6,
         )?;
 
+        // 2
         let eth_dos_distance_either = CustomPredicate::or(
+            "eth_dos_distance_either".into(),
             &params,
             vec![
                 st(
                     P::BatchSelf(0),
                     vec![
-                        STA::Key(HOW::Wildcard(0), HOW::Wildcard(1)),
-                        STA::Key(HOW::Wildcard(2), HOW::Wildcard(3)),
-                        STA::Key(HOW::Wildcard(4), HOW::Wildcard(5)),
+                        STA::WildcardLiteral(wc(0)),
+                        STA::WildcardLiteral(wc(1)),
+                        STA::WildcardLiteral(wc(2)),
+                        STA::WildcardLiteral(wc(3)),
+                        STA::WildcardLiteral(wc(4)),
+                        STA::WildcardLiteral(wc(5)),
                     ],
                 ),
                 st(
                     P::BatchSelf(1),
                     vec![
-                        STA::Key(HOW::Wildcard(0), HOW::Wildcard(1)),
-                        STA::Key(HOW::Wildcard(2), HOW::Wildcard(3)),
-                        STA::Key(HOW::Wildcard(4), HOW::Wildcard(5)),
+                        STA::WildcardLiteral(wc(0)),
+                        STA::WildcardLiteral(wc(1)),
+                        STA::WildcardLiteral(wc(2)),
+                        STA::WildcardLiteral(wc(3)),
+                        STA::WildcardLiteral(wc(4)),
+                        STA::WildcardLiteral(wc(5)),
                     ],
                 ),
             ],
@@ -646,11 +622,14 @@ mod tests {
 
         // Example statement
         let ethdos_example = Statement::Custom(
-            CustomPredicateRef(eth_dos_distance_batch.clone(), 2),
+            CustomPredicateRef::new(eth_dos_distance_batch.clone(), 2),
             vec![
-                AnchoredKey(pod_id1, "Alice".into()),
-                AnchoredKey(pod_id2, "Bob".into()),
-                AnchoredKey(SELF, "Seven".into()),
+                WildcardValue::PodId(pod_id1),
+                WildcardValue::Key(Key::from("Alice")),
+                WildcardValue::PodId(pod_id2),
+                WildcardValue::Key(Key::from("Bob")),
+                WildcardValue::PodId(SELF),
+                WildcardValue::Key(Key::from("Seven")),
             ],
         );
 
@@ -659,17 +638,20 @@ mod tests {
 
         // This could arise as the inductive step.
         let ethdos_ind_example = Statement::Custom(
-            CustomPredicateRef(eth_dos_distance_batch.clone(), 1),
+            CustomPredicateRef::new(eth_dos_distance_batch.clone(), 1),
             vec![
-                AnchoredKey(pod_id1, "Alice".into()),
-                AnchoredKey(pod_id2, "Bob".into()),
-                AnchoredKey(SELF, "Seven".into()),
+                WildcardValue::PodId(pod_id1),
+                WildcardValue::Key(Key::from("Alice")),
+                WildcardValue::PodId(pod_id2),
+                WildcardValue::Key(Key::from("Bob")),
+                WildcardValue::PodId(SELF),
+                WildcardValue::Key(Key::from("Seven")),
             ],
         );
 
         assert!(Operation::Custom(
-            CustomPredicateRef(eth_dos_distance_batch.clone(), 2),
-            vec![ethdos_ind_example.clone()]
+            CustomPredicateRef::new(eth_dos_distance_batch.clone(), 2),
+            vec![Statement::None, ethdos_ind_example.clone()]
         )
         .check(&params, &ethdos_example)?);
 
@@ -678,30 +660,35 @@ mod tests {
         // less than 7, and Charlie is ETH-friends with Bob.
         let ethdos_facts = vec![
             Statement::Custom(
-                CustomPredicateRef(eth_dos_distance_batch.clone(), 2),
+                CustomPredicateRef::new(eth_dos_distance_batch.clone(), 2),
                 vec![
-                    AnchoredKey(pod_id1, "Alice".into()),
-                    AnchoredKey(pod_id3, "Charlie".into()),
-                    AnchoredKey(pod_id4, "Six".into()),
+                    WildcardValue::PodId(pod_id1),
+                    WildcardValue::Key(Key::from("Alice")),
+                    WildcardValue::PodId(pod_id3),
+                    WildcardValue::Key(Key::from("Charlie")),
+                    WildcardValue::PodId(pod_id4),
+                    WildcardValue::Key(Key::from("Six")),
                 ],
             ),
-            Statement::ValueOf(AnchoredKey(SELF, "One".into()), 1.into()),
+            Statement::ValueOf(AnchoredKey::from((SELF, "One")), 1.into()),
             Statement::SumOf(
-                AnchoredKey(SELF, "Seven".into()),
-                AnchoredKey(pod_id4, "Six".into()),
-                AnchoredKey(SELF, "One".into()),
+                AnchoredKey::from((SELF, "Seven")),
+                AnchoredKey::from((pod_id4, "Six")),
+                AnchoredKey::from((SELF, "One")),
             ),
             Statement::Custom(
-                CustomPredicateRef(eth_friend_batch.clone(), 0),
+                CustomPredicateRef::new(eth_friend_batch.clone(), 0),
                 vec![
-                    AnchoredKey(pod_id3, "Charlie".into()),
-                    AnchoredKey(pod_id2, "Bob".into()),
+                    WildcardValue::PodId(pod_id3),
+                    WildcardValue::Key(Key::from("Charlie")),
+                    WildcardValue::PodId(pod_id2),
+                    WildcardValue::Key(Key::from("Bob")),
                 ],
             ),
         ];
 
         assert!(Operation::Custom(
-            CustomPredicateRef(eth_dos_distance_batch.clone(), 1),
+            CustomPredicateRef::new(eth_dos_distance_batch.clone(), 1),
             ethdos_facts
         )
         .check(&params, &ethdos_ind_example)?);

src/middleware/mod.rs

@@ -1,7 +1,15 @@
 //! The middleware includes the type definitions and the traits used to connect the frontend and
 //! the backend.
 
+use std::sync::Arc;
 mod basetypes;
+use std::{
+    cmp::{Ordering, PartialEq, PartialOrd},
+    hash,
+};
+
+use anyhow::anyhow;
+use containers::{Array, Dictionary, Set};
 pub mod containers;
 mod custom;
 mod operation;
@@ -14,12 +22,214 @@ pub use basetypes::*;
 pub use custom::*;
 use dyn_clone::DynClone;
 pub use operation::*;
-use schemars::JsonSchema;
-use serde::{Deserialize, Serialize};
+// use schemars::JsonSchema;
+// use serde::{Deserialize, Serialize};
 pub use statement::*;
 
 pub const SELF: PodId = PodId(SELF_ID_HASH);
 
+// TODO: Move all value-related types to to `value.rs`
+#[derive(Clone, Debug)]
+// TODO #[schemars(transform = serialization::transform_value_schema)]
+pub enum TypedValue {
+    // Serde cares about the order of the enum variants, with untagged variants
+    // appearing at the end.
+    // Variants without "untagged" will be serialized as "tagged" values by
+    // default, meaning that a Set appears in JSON as {"Set":[...]}
+    // and not as [...]
+    // Arrays, Strings and Booleans are untagged, as there is a natural JSON
+    // representation for them that is unambiguous to deserialize and is fully
+    // compatible with the semantics of the POD types.
+    // As JSON integers do not specify precision, and JavaScript is limited to
+    // 53-bit precision for integers, integers are represented as tagged
+    // strings, with a custom serializer and deserializer.
+    // TAGGED TYPES:
+    Set(Set),
+    Dictionary(Dictionary),
+    Int(
+        // TODO #[serde(serialize_with = "serialize_i64", deserialize_with = "deserialize_i64")]
+        // #[schemars(with = "String", regex(pattern = r"^\d+$"))]
+        i64,
+    ),
+    // Uses the serialization for middleware::Value:
+    Raw(RawValue),
+    // UNTAGGED TYPES:
+    // #[serde(untagged)]
+    // #[schemars(skip)]
+    Array(Array),
+    // #[serde(untagged)]
+    // #[schemars(skip)]
+    String(String),
+    // #[serde(untagged)]
+    // #[schemars(skip)]
+    Bool(bool),
+}
+
+impl From<&str> for TypedValue {
+    fn from(s: &str) -> Self {
+        TypedValue::String(s.to_string())
+    }
+}
+
+impl From<String> for TypedValue {
+    fn from(s: String) -> Self {
+        TypedValue::String(s)
+    }
+}
+
+impl From<i64> for TypedValue {
+    fn from(v: i64) -> Self {
+        TypedValue::Int(v)
+    }
+}
+
+impl From<bool> for TypedValue {
+    fn from(b: bool) -> Self {
+        TypedValue::Bool(b)
+    }
+}
+
+impl From<Hash> for TypedValue {
+    fn from(h: Hash) -> Self {
+        TypedValue::Raw(RawValue(h.0))
+    }
+}
+
+impl From<Set> for TypedValue {
+    fn from(s: Set) -> Self {
+        TypedValue::Set(s)
+    }
+}
+
+impl From<Dictionary> for TypedValue {
+    fn from(d: Dictionary) -> Self {
+        TypedValue::Dictionary(d)
+    }
+}
+
+impl From<Array> for TypedValue {
+    fn from(a: Array) -> Self {
+        TypedValue::Array(a)
+    }
+}
+
+impl From<RawValue> for TypedValue {
+    fn from(v: RawValue) -> Self {
+        TypedValue::Raw(v)
+    }
+}
+
+impl From<PodType> for TypedValue {
+    fn from(t: PodType) -> Self {
+        TypedValue::from(t as i64)
+    }
+}
+
+impl TryFrom<&TypedValue> for i64 {
+    type Error = anyhow::Error;
+    fn try_from(v: &TypedValue) -> std::result::Result<Self, Self::Error> {
+        if let TypedValue::Int(n) = v {
+            Ok(*n)
+        } else {
+            Err(anyhow!("Value not an int"))
+        }
+    }
+}
+
+impl fmt::Display for TypedValue {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            TypedValue::String(s) => write!(f, "\"{}\"", s),
+            TypedValue::Int(v) => write!(f, "{}", v),
+            TypedValue::Bool(b) => write!(f, "{}", b),
+            TypedValue::Dictionary(d) => write!(f, "dict:{}", d.commitment()),
+            TypedValue::Set(s) => write!(f, "set:{}", s.commitment()),
+            TypedValue::Array(a) => write!(f, "arr:{}", a.commitment()),
+            TypedValue::Raw(v) => write!(f, "{}", v),
+        }
+    }
+}
+
+impl From<&TypedValue> for RawValue {
+    fn from(v: &TypedValue) -> Self {
+        match v {
+            TypedValue::String(s) => RawValue::from(hash_str(s)),
+            TypedValue::Int(v) => RawValue::from(*v),
+            TypedValue::Bool(b) => RawValue::from(*b as i64),
+            TypedValue::Dictionary(d) => RawValue::from(d.commitment()),
+            TypedValue::Set(s) => RawValue::from(s.commitment()),
+            TypedValue::Array(a) => RawValue::from(a.commitment()),
+            TypedValue::Raw(v) => *v,
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct Value {
+    // The `TypedValue` is under `Arc` so that cloning a `Value` is cheap.
+    typed: Arc<TypedValue>,
+    raw: RawValue,
+}
+
+impl PartialEq for Value {
+    fn eq(&self, other: &Self) -> bool {
+        self.raw == other.raw
+    }
+}
+
+impl Eq for Value {}
+
+impl PartialOrd for Value {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.raw.cmp(&other.raw))
+    }
+}
+
+impl Ord for Value {
+    fn cmp(&self, other: &Self) -> Ordering {
+        self.raw.cmp(&other.raw)
+    }
+}
+
+impl hash::Hash for Value {
+    fn hash<H: hash::Hasher>(&self, state: &mut H) {
+        self.raw.hash(state)
+    }
+}
+
+impl fmt::Display for Value {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.typed)
+    }
+}
+
+impl Value {
+    pub fn new(value: TypedValue) -> Self {
+        let raw_value = RawValue::from(&value);
+        Self {
+            typed: Arc::new(value),
+            raw: raw_value,
+        }
+    }
+
+    pub fn typed(&self) -> &TypedValue {
+        &self.typed
+    }
+    pub fn raw(&self) -> RawValue {
+        self.raw
+    }
+}
+
+// A Value can be created from any type Into<TypedValue> type: bool, string-like, i64, ...
+impl<T> From<T> for Value
+where
+    T: Into<TypedValue>,
+{
+    fn from(t: T) -> Self {
+        Self::new(t.into())
+    }
+}
+
 impl fmt::Display for PodId {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         if *self == SELF {
@@ -32,30 +242,93 @@ impl fmt::Display for PodId {
     }
 }
 
-/// AnchoredKey is a tuple containing (OriginId: PodId, key: Hash)
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
-pub struct AnchoredKey(pub PodId, pub Hash);
+impl From<&Value> for Hash {
+    fn from(v: &Value) -> Self {
+        Self(v.raw.0)
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub struct Key {
+    name: String,
+    hash: Hash,
+}
+
+impl Key {
+    pub fn new(name: String) -> Self {
+        let hash = hash_str(&name);
+        Self { name, hash }
+    }
+
+    pub fn name(&self) -> &str {
+        &self.name
+    }
+    pub fn hash(&self) -> Hash {
+        self.hash
+    }
+    pub fn raw(&self) -> RawValue {
+        RawValue(self.hash.0)
+    }
+}
+
+// A Key can easily be created from a string-like type
+impl<T> From<T> for Key
+where
+    T: Into<String>,
+{
+    fn from(t: T) -> Self {
+        Self::new(t.into())
+    }
+}
+
+impl ToFields for Key {
+    fn to_fields(&self, params: &Params) -> Vec<F> {
+        self.hash.to_fields(params)
+    }
+}
+
+impl fmt::Display for Key {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.name)?;
+        Ok(())
+    }
+}
+
+impl From<Key> for RawValue {
+    fn from(key: Key) -> RawValue {
+        RawValue(key.hash.0)
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub struct AnchoredKey {
+    pub pod_id: PodId,
+    pub key: Key,
+}
 
 impl AnchoredKey {
-    pub fn origin(&self) -> PodId {
-        self.0
-    }
-    pub fn key(&self) -> Hash {
-        self.1
+    pub fn new(pod_id: PodId, key: Key) -> Self {
+        Self { pod_id, key }
     }
 }
 
 impl fmt::Display for AnchoredKey {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{}.{}", self.0, self.1)?;
+        write!(f, "{}.{}", self.pod_id, self.key)?;
         Ok(())
     }
 }
 
-/// An entry consists of a key-value pair.
-pub type Entry = (String, Value);
+impl<T> From<(PodId, T)> for AnchoredKey
+where
+    T: Into<Key>,
+{
+    fn from((pod_id, t): (PodId, T)) -> Self {
+        Self::new(pod_id, t.into())
+    }
+}
 
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Default, Serialize, Deserialize, JsonSchema)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Default)]
 pub struct PodId(pub Hash);
 
 impl ToFields for PodId {
@@ -72,6 +345,7 @@ pub enum PodType {
     Signed = 3,
     Main = 4,
 }
+
 impl fmt::Display for PodType {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
@@ -84,13 +358,7 @@ impl fmt::Display for PodType {
     }
 }
 
-impl From<PodType> for Value {
-    fn from(v: PodType) -> Self {
-        Value::from(v as i64)
-    }
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct Params {
     pub max_input_signed_pods: usize,
     pub max_input_main_pods: usize,
@@ -102,6 +370,7 @@ pub struct Params {
     // max number of statements that can be ANDed or ORed together
     // in a custom predicate
     pub max_custom_predicate_arity: usize,
+    pub max_custom_predicate_wildcards: usize,
     pub max_custom_batch_size: usize,
     // maximum number of merkle proofs
     pub max_merkle_proofs: usize,
@@ -120,6 +389,7 @@ impl Default for Params {
             max_statement_args: 5,
             max_operation_args: 5,
             max_custom_predicate_arity: 5,
+            max_custom_predicate_wildcards: 10,
             max_custom_batch_size: 5,
             max_merkle_proofs: 5,
             max_depth_mt_gadget: 32,
@@ -213,7 +483,7 @@ pub trait Pod: fmt::Debug + DynClone {
 dyn_clone::clone_trait_object!(Pod);
 
 pub trait PodSigner {
-    fn sign(&mut self, params: &Params, kvs: &HashMap<Hash, Value>) -> Result<Box<dyn Pod>>;
+    fn sign(&mut self, params: &Params, kvs: &HashMap<Key, Value>) -> Result<Box<dyn Pod>>;
 }
 
 /// This is a filler type that fulfills the Pod trait and always verifies.  It's empty.  This

src/middleware/operation.rs

@@ -1,25 +1,26 @@
-use std::{fmt, iter};
+use std::{fmt, iter, sync::Arc};
 
 use anyhow::{anyhow, Result};
 use log::error;
 use plonky2::field::types::Field;
-use serde::{Deserialize, Serialize};
 
+// use serde::{Deserialize, Serialize};
 use crate::{
     backends::plonky2::primitives::merkletree::{MerkleProof, MerkleTree},
     middleware::{
-        AnchoredKey, CustomPredicateRef, NativePredicate, Params, Predicate, Statement,
-        StatementArg, ToFields, Value, F, SELF,
+        custom::KeyOrWildcard, AnchoredKey, CustomPredicateBatch, CustomPredicateRef,
+        NativePredicate, Params, Predicate, Statement, StatementArg, StatementTmplArg, ToFields,
+        Wildcard, WildcardValue, F, SELF,
     },
 };
 
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Clone, Debug, PartialEq)]
 pub enum OperationType {
     Native(NativeOperation),
     Custom(CustomPredicateRef),
 }
 
-#[derive(Clone, Debug, PartialEq, Eq)]
+#[derive(Clone, Debug, PartialEq)]
 pub enum OperationAux {
     None,
     MerkleProof(MerkleProof),
@@ -41,17 +42,19 @@ impl ToFields for OperationType {
             Self::Native(p) => iter::once(F::from_canonical_u64(1))
                 .chain(p.to_fields(params))
                 .collect(),
-            Self::Custom(CustomPredicateRef(pb, i)) => iter::once(F::from_canonical_u64(3))
-                .chain(pb.hash(params).0)
-                .chain(iter::once(F::from_canonical_usize(*i)))
-                .collect(),
+            Self::Custom(CustomPredicateRef { batch, index }) => {
+                iter::once(F::from_canonical_u64(3))
+                    .chain(batch.hash(params).0)
+                    .chain(iter::once(F::from_canonical_usize(*index)))
+                    .collect()
+            }
         };
         fields.resize_with(Params::operation_type_size(), || F::from_canonical_u64(0));
         fields
     }
 }
 
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum NativeOperation {
     None = 0,
     NewEntry = 1,
@@ -68,6 +71,14 @@ pub enum NativeOperation {
     SumOf = 13,
     ProductOf = 14,
     MaxOf = 15,
+
+    // Syntactic sugar operations.  These operations are not supported by the backend.  The
+    // frontend compiler is responsible of translating these operations into the operations above.
+    DictContainsFromEntries = 1001,
+    DictNotContainsFromEntries = 1002,
+    SetContainsFromEntries = 1003,
+    SetNotContainsFromEntries = 1004,
+    ArrayContainsFromEntries = 1005,
 }
 
 impl ToFields for NativeOperation {
@@ -108,6 +119,7 @@ impl OperationType {
                 NativeOperation::SumOf => Some(Predicate::Native(NativePredicate::SumOf)),
                 NativeOperation::ProductOf => Some(Predicate::Native(NativePredicate::ProductOf)),
                 NativeOperation::MaxOf => Some(Predicate::Native(NativePredicate::MaxOf)),
+                no => unreachable!("Unexpected syntactic sugar op {:?}", no),
             },
             OperationType::Custom(cpr) => Some(Predicate::Custom(cpr.clone())),
         }
@@ -115,7 +127,7 @@ impl OperationType {
 }
 
 // TODO: Refine this enum.
-#[derive(Clone, Debug, PartialEq, Eq)]
+#[derive(Clone, Debug, PartialEq)]
 pub enum Operation {
     None,
     NewEntry,
@@ -263,7 +275,10 @@ impl Operation {
             Self::CopyStatement(s1) => Some(s1.args()),
             Self::EqualFromEntries(ValueOf(ak1, v1), ValueOf(ak2, v2)) => {
                 if v1 == v2 {
-                    Some(vec![StatementArg::Key(*ak1), StatementArg::Key(*ak2)])
+                    Some(vec![
+                        StatementArg::Key(ak1.clone()),
+                        StatementArg::Key(ak2.clone()),
+                    ])
                 } else {
                     return Err(anyhow!("Invalid operation"));
                 }
@@ -273,7 +288,10 @@ impl Operation {
             }
             Self::NotEqualFromEntries(ValueOf(ak1, v1), ValueOf(ak2, v2)) => {
                 if v1 != v2 {
-                    Some(vec![StatementArg::Key(*ak1), StatementArg::Key(*ak2)])
+                    Some(vec![
+                        StatementArg::Key(ak1.clone()),
+                        StatementArg::Key(ak2.clone()),
+                    ])
                 } else {
                     return Err(anyhow!("Invalid operation"));
                 }
@@ -283,7 +301,10 @@ impl Operation {
             }
             Self::GtFromEntries(ValueOf(ak1, v1), ValueOf(ak2, v2)) => {
                 if v1 > v2 {
-                    Some(vec![StatementArg::Key(*ak1), StatementArg::Key(*ak2)])
+                    Some(vec![
+                        StatementArg::Key(ak1.clone()),
+                        StatementArg::Key(ak2.clone()),
+                    ])
                 } else {
                     return Err(anyhow!("Invalid operation"));
                 }
@@ -293,7 +314,10 @@ impl Operation {
             }
             Self::LtFromEntries(ValueOf(ak1, v1), ValueOf(ak2, v2)) => {
                 if v1 < v2 {
-                    Some(vec![StatementArg::Key(*ak1), StatementArg::Key(*ak2)])
+                    Some(vec![
+                        StatementArg::Key(ak1.clone()),
+                        StatementArg::Key(ak2.clone()),
+                    ])
                 } else {
                     return Err(anyhow!("Invalid operation"));
                 }
@@ -303,7 +327,10 @@ impl Operation {
             }
             Self::TransitiveEqualFromStatements(Equal(ak1, ak2), Equal(ak3, ak4)) => {
                 if ak2 == ak3 {
-                    Some(vec![StatementArg::Key(*ak1), StatementArg::Key(*ak4)])
+                    Some(vec![
+                        StatementArg::Key(ak1.clone()),
+                        StatementArg::Key(ak4.clone()),
+                    ])
                 } else {
                     return Err(anyhow!("Invalid operation"));
                 }
@@ -311,48 +338,54 @@ impl Operation {
             Self::TransitiveEqualFromStatements(_, _) => {
                 return Err(anyhow!("Invalid operation"));
             }
-            Self::GtToNotEqual(Gt(ak1, ak2)) => {
-                Some(vec![StatementArg::Key(*ak1), StatementArg::Key(*ak2)])
-            }
+            Self::GtToNotEqual(Gt(ak1, ak2)) => Some(vec![
+                StatementArg::Key(ak1.clone()),
+                StatementArg::Key(ak2.clone()),
+            ]),
             Self::GtToNotEqual(_) => {
                 return Err(anyhow!("Invalid operation"));
             }
-            Self::LtToNotEqual(Gt(ak1, ak2)) => {
-                Some(vec![StatementArg::Key(*ak1), StatementArg::Key(*ak2)])
-            }
+            Self::LtToNotEqual(Gt(ak1, ak2)) => Some(vec![
+                StatementArg::Key(ak1.clone()),
+                StatementArg::Key(ak2.clone()),
+            ]),
             Self::LtToNotEqual(_) => {
                 return Err(anyhow!("Invalid operation"));
             }
             Self::ContainsFromEntries(ValueOf(ak1, v1), ValueOf(ak2, v2), ValueOf(ak3, v3), pf)
-                if MerkleTree::verify(pf.siblings.len(), (*v1).into(), pf, v2, v3).is_ok() =>
+                if MerkleTree::verify(pf.siblings.len(), v1.into(), pf, &v2.raw(), &v3.raw())
+                    .is_ok() =>
             {
                 Some(vec![
-                    StatementArg::Key(*ak1),
-                    StatementArg::Key(*ak2),
-                    StatementArg::Key(*ak3),
+                    StatementArg::Key(ak1.clone()),
+                    StatementArg::Key(ak2.clone()),
+                    StatementArg::Key(ak3.clone()),
                 ])
             }
             Self::ContainsFromEntries(_, _, _, _) => {
                 return Err(anyhow!("Invalid operation"));
             }
             Self::NotContainsFromEntries(ValueOf(ak1, v1), ValueOf(ak2, v2), pf)
-                if MerkleTree::verify_nonexistence(pf.siblings.len(), (*v1).into(), pf, v2)
+                if MerkleTree::verify_nonexistence(pf.siblings.len(), v1.into(), pf, &v2.raw())
                     .is_ok() =>
             {
-                Some(vec![StatementArg::Key(*ak1), StatementArg::Key(*ak2)])
+                Some(vec![
+                    StatementArg::Key(ak1.clone()),
+                    StatementArg::Key(ak2.clone()),
+                ])
             }
             Self::NotContainsFromEntries(_, _, _) => {
                 return Err(anyhow!("Invalid operation"));
             }
             Self::SumOf(ValueOf(ak1, v1), ValueOf(ak2, v2), ValueOf(ak3, v3)) => {
-                let v1: i64 = (*v1).try_into()?;
-                let v2: i64 = (*v2).try_into()?;
-                let v3: i64 = (*v3).try_into()?;
+                let v1: i64 = v1.typed().try_into()?;
+                let v2: i64 = v2.typed().try_into()?;
+                let v3: i64 = v3.typed().try_into()?;
                 if v1 == v2 + v3 {
                     Some(vec![
-                        StatementArg::Key(*ak1),
-                        StatementArg::Key(*ak2),
-                        StatementArg::Key(*ak3),
+                        StatementArg::Key(ak1.clone()),
+                        StatementArg::Key(ak2.clone()),
+                        StatementArg::Key(ak3.clone()),
                     ])
                 } else {
                     return Err(anyhow!("Invalid operation"));
@@ -362,14 +395,14 @@ impl Operation {
                 return Err(anyhow!("Invalid operation"));
             }
             Self::ProductOf(ValueOf(ak1, v1), ValueOf(ak2, v2), ValueOf(ak3, v3)) => {
-                let v1: i64 = (*v1).try_into()?;
-                let v2: i64 = (*v2).try_into()?;
-                let v3: i64 = (*v3).try_into()?;
+                let v1: i64 = v1.typed().try_into()?;
+                let v2: i64 = v2.typed().try_into()?;
+                let v3: i64 = v3.typed().try_into()?;
                 if v1 == v2 * v3 {
                     Some(vec![
-                        StatementArg::Key(*ak1),
-                        StatementArg::Key(*ak2),
-                        StatementArg::Key(*ak3),
+                        StatementArg::Key(ak1.clone()),
+                        StatementArg::Key(ak2.clone()),
+                        StatementArg::Key(ak3.clone()),
                     ])
                 } else {
                     return Err(anyhow!("Invalid operation"));
@@ -379,14 +412,14 @@ impl Operation {
                 return Err(anyhow!("Invalid operation"));
             }
             Self::MaxOf(ValueOf(ak1, v1), ValueOf(ak2, v2), ValueOf(ak3, v3)) => {
-                let v1: i64 = (*v1).try_into()?;
-                let v2: i64 = (*v2).try_into()?;
-                let v3: i64 = (*v3).try_into()?;
+                let v1: i64 = v1.typed().try_into()?;
+                let v2: i64 = v2.typed().try_into()?;
+                let v3: i64 = v3.typed().try_into()?;
                 if v1 == std::cmp::max(v2, v3) {
                     Some(vec![
-                        StatementArg::Key(*ak1),
-                        StatementArg::Key(*ak2),
-                        StatementArg::Key(*ak3),
+                        StatementArg::Key(ak1.clone()),
+                        StatementArg::Key(ak2.clone()),
+                        StatementArg::Key(ak3.clone()),
                     ])
                 } else {
                     return Err(anyhow!("Invalid operation"));
@@ -413,11 +446,11 @@ impl Operation {
         Ok(valid)
     }
     /// Checks the given operation against a statement.
-    pub fn check(&self, _params: &Params, output_statement: &Statement) -> Result<bool> {
+    pub fn check(&self, params: &Params, output_statement: &Statement) -> Result<bool> {
         use Statement::*;
         match (self, output_statement) {
             (Self::None, None) => Ok(true),
-            (Self::NewEntry, ValueOf(AnchoredKey(pod_id, _), _)) => Ok(pod_id == &SELF),
+            (Self::NewEntry, ValueOf(AnchoredKey { pod_id, .. }, _)) => Ok(pod_id == &SELF),
             (Self::CopyStatement(s1), s2) => Ok(s1 == s2),
             (Self::EqualFromEntries(ValueOf(ak1, v1), ValueOf(ak2, v2)), Equal(ak3, ak4)) => {
                 Ok(v1 == v2 && ak3 == ak1 && ak4 == ak2)
@@ -451,40 +484,15 @@ impl Operation {
                 Self::SumOf(ValueOf(ak1, v1), ValueOf(ak2, v2), ValueOf(ak3, v3)),
                 SumOf(ak4, ak5, ak6),
             ) => {
-                let v1: i64 = (*v1).try_into()?;
-                let v2: i64 = (*v2).try_into()?;
-                let v3: i64 = (*v3).try_into()?;
+                let v1: i64 = v1.typed().try_into()?;
+                let v2: i64 = v2.typed().try_into()?;
+                let v3: i64 = v3.typed().try_into()?;
                 Ok((v1 == v2 + v3) && ak4 == ak1 && ak5 == ak2 && ak6 == ak3)
             }
-            (Self::Custom(CustomPredicateRef(cpb, i), args), Custom(cpr, s_args))
-                if cpb == &cpr.0 && i == &cpr.1 =>
+            (Self::Custom(CustomPredicateRef { batch, index }, args), Custom(cpr, s_args))
+                if batch == &cpr.batch && index == &cpr.index =>
             {
-                // Bind according to custom predicate pattern match against arg list.
-                let bindings = cpr.match_against(args)?;
-                // Check arg length
-                let arg_len = cpr.arg_len();
-                if arg_len != 2 * s_args.len() {
-                    Err(anyhow!("Custom predicate arg list {:?} must have {} arguments after destructuring.", s_args, arg_len))
-                } else {
-                    let bound_args = (0..arg_len)
-                        .map(|i| {
-                            bindings.get(&i).cloned().ok_or(anyhow!(
-                                "Wildcard {} of custom predicate {:?} is unbound.",
-                                i,
-                                cpr
-                            ))
-                        })
-                        .collect::<Result<Vec<_>>>()?;
-                    let s_args = s_args
-                        .iter()
-                        .flat_map(|AnchoredKey(o, k)| [Value::from(o.0), (*k).into()])
-                        .collect::<Vec<_>>();
-                    if bound_args != s_args {
-                        Err(anyhow!("Arguments to output statement {} do not match those implied by operation {:?}", output_statement,self))
-                    } else {
-                        Ok(true)
-                    }
-                }
+                check_custom_pred(params, batch, *index, args, s_args)
             }
             _ => Err(anyhow!(
                 "Invalid deduction: {:?} ⇏ {:#}",
@@ -495,6 +503,120 @@ impl Operation {
     }
 }
 
+/// Check that a StatementArg follows a StatementTmplArg based on the currently mapped wildcards.
+/// Update the wildcard map with newly found wildcards.
+pub fn check_st_tmpl(
+    st_tmpl_arg: &StatementTmplArg,
+    st_arg: &StatementArg,
+    // Map from wildcards to values that we have seen so far.
+    wildcard_map: &mut [Option<WildcardValue>],
+) -> bool {
+    // Check that the value `v` at wildcard `wc` exists in the map or set it.
+    fn check_or_set(
+        v: WildcardValue,
+        wc: &Wildcard,
+        wildcard_map: &mut [Option<WildcardValue>],
+    ) -> bool {
+        if let Some(prev) = &wildcard_map[wc.index] {
+            if *prev != v {
+                // TODO: Return nice error
+                return false;
+            }
+        } else {
+            wildcard_map[wc.index] = Some(v);
+        }
+        true
+    }
+
+    match (st_tmpl_arg, st_arg) {
+        (StatementTmplArg::None, StatementArg::None) => true,
+        (StatementTmplArg::Literal(lhs), StatementArg::Literal(rhs)) if lhs == rhs => true,
+        (
+            StatementTmplArg::Key(pod_id_wc, key_or_wc),
+            StatementArg::Key(AnchoredKey { pod_id, key }),
+        ) => {
+            let pod_id_ok = check_or_set(WildcardValue::PodId(*pod_id), pod_id_wc, wildcard_map);
+            let key_ok = match key_or_wc {
+                KeyOrWildcard::Key(tmpl_key) => tmpl_key == key,
+                KeyOrWildcard::Wildcard(key_wc) => {
+                    check_or_set(WildcardValue::Key(key.clone()), key_wc, wildcard_map)
+                }
+            };
+            pod_id_ok && key_ok
+        }
+        (StatementTmplArg::WildcardLiteral(wc), StatementArg::WildcardLiteral(v)) => {
+            check_or_set(v.clone(), wc, wildcard_map)
+        }
+        _ => false,
+    }
+}
+
+fn check_custom_pred(
+    params: &Params,
+    batch: &Arc<CustomPredicateBatch>,
+    index: usize,
+    args: &[Statement],
+    s_args: &[WildcardValue],
+) -> Result<bool> {
+    let pred = &batch.predicates[index];
+    if pred.statements.len() != args.len() {
+        return Err(anyhow!(
+            "Custom predicate operation needs {} statements but has {}.",
+            pred.statements.len(),
+            args.len()
+        ));
+    }
+    if pred.args_len != s_args.len() {
+        return Err(anyhow!(
+            "Custom predicate statement needs {} args but has {}.",
+            pred.args_len,
+            s_args.len()
+        ));
+    }
+
+    // Check that all wildcard have consistent values as assigned in the statements while storing a
+    // map of their values.  Count the number of statements that match the templates by predicate.
+    // NOTE: We assume the statements have the same order as defined in the custom predicate.  For
+    // disjunctions we expect Statement::None for the unused statements.
+    let mut num_matches = 0;
+    let mut wildcard_map = vec![None; params.max_custom_predicate_wildcards];
+    for (st_tmpl, st) in pred.statements.iter().zip(args) {
+        let st_args = st.args();
+        for (st_tmpl_arg, st_arg) in st_tmpl.args.iter().zip(&st_args) {
+            if !check_st_tmpl(st_tmpl_arg, st_arg, &mut wildcard_map) {
+                // TODO: Better errors.  Example:
+                // println!("{} doesn't match {}", st_arg, st_tmpl_arg);
+                // println!("{} doesn't match {}", st, st_tmpl);
+                return Ok(false);
+            }
+        }
+
+        let st_tmpl_pred = match &st_tmpl.pred {
+            Predicate::BatchSelf(i) => Predicate::Custom(CustomPredicateRef {
+                batch: batch.clone(),
+                index: *i,
+            }),
+            p => p.clone(),
+        };
+        if st_tmpl_pred == st.predicate() {
+            num_matches += 1;
+        }
+    }
+
+    // Check that the resolved wildcard match the statement arguments.
+    for (s_arg, wc_value) in s_args.iter().zip(wildcard_map.iter()) {
+        if !wc_value.as_ref().is_none_or(|wc_value| *wc_value == *s_arg) {
+            return Ok(false);
+        }
+    }
+
+    if pred.conjunction {
+        Ok(num_matches == pred.statements.len())
+    } else {
+        Ok(num_matches > 0)
+    }
+}
+
 impl ToFields for Operation {
     fn to_fields(&self, _params: &Params) -> Vec<F> {
         todo!()

src/middleware/serialization.rs

@@ -1,3 +1,5 @@
+// TODO: Reenable
+/*
 use plonky2::field::types::Field;
 use serde::Deserialize;
 
@@ -67,3 +69,4 @@ where
 {
     deserialize_field_tuple::<D, VALUE_SIZE>(deserializer)
 }
+*/

src/middleware/statement.rs

@@ -2,14 +2,16 @@ use std::{fmt, iter};
 
 use anyhow::{anyhow, Result};
 use plonky2::field::types::Field;
-use schemars::JsonSchema;
-use serde::{Deserialize, Serialize};
+// use schemars::JsonSchema;
+// use serde::{Deserialize, Serialize};
 use strum_macros::FromRepr;
 
 use crate::middleware::{
-    AnchoredKey, CustomPredicateRef, Params, Predicate, ToFields, Value, F, VALUE_SIZE,
+    AnchoredKey, CustomPredicateRef, Key, Params, PodId, Predicate, RawValue, ToFields, Value, F,
+    VALUE_SIZE,
 };
 
+// TODO: Maybe store KEY_SIGNER and KEY_TYPE as Key with lazy_static
 // hash(KEY_SIGNER) = [2145458785152392366, 15113074911296146791, 15323228995597834291, 11804480340100333725]
 pub const KEY_SIGNER: &str = "_signer";
 // hash(KEY_TYPE) = [17948789436443445142, 12513915140657440811, 15878361618879468769, 938231894693848619]
@@ -18,7 +20,7 @@ pub const STATEMENT_ARG_F_LEN: usize = 8;
 pub const OPERATION_ARG_F_LEN: usize = 1;
 pub const OPERATION_AUX_F_LEN: usize = 1;
 
-#[derive(Clone, Copy, Debug, FromRepr, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)]
+#[derive(Clone, Copy, Debug, FromRepr, PartialEq, Eq, Hash)]
 pub enum NativePredicate {
     None = 0,
     ValueOf = 1,
@@ -31,6 +33,14 @@ pub enum NativePredicate {
     SumOf = 8,
     ProductOf = 9,
     MaxOf = 10,
+
+    // Syntactic sugar predicates.  These predicates are not supported by the backend.  The
+    // frontend compiler is responsible of translating these predicates into the predicates above.
+    DictContains = 1000,
+    DictNotContains = 1001,
+    SetContains = 1002,
+    SetNotContains = 1003,
+    ArrayContains = 1004, // there is no ArrayNotContains
 }
 
 impl ToFields for NativePredicate {
@@ -39,8 +49,41 @@ impl ToFields for NativePredicate {
     }
 }
 
+#[derive(Clone, Debug, PartialEq)]
+pub enum WildcardValue {
+    PodId(PodId),
+    Key(Key),
+}
+
+impl WildcardValue {
+    pub fn raw(&self) -> RawValue {
+        match self {
+            WildcardValue::PodId(pod_id) => RawValue::from(pod_id.0),
+            WildcardValue::Key(key) => key.raw(),
+        }
+    }
+}
+
+impl fmt::Display for WildcardValue {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            WildcardValue::PodId(pod_id) => write!(f, "{}", pod_id),
+            WildcardValue::Key(key) => write!(f, "{}", key),
+        }
+    }
+}
+
+impl ToFields for WildcardValue {
+    fn to_fields(&self, params: &Params) -> Vec<F> {
+        match self {
+            WildcardValue::PodId(pod_id) => pod_id.to_fields(params),
+            WildcardValue::Key(key) => key.to_fields(params),
+        }
+    }
+}
+
 /// Type encapsulating statements with their associated arguments.
-#[derive(Clone, Debug, PartialEq, Eq)]
+#[derive(Clone, Debug, PartialEq)]
 pub enum Statement {
     None,
     ValueOf(AnchoredKey, Value),
@@ -57,7 +100,7 @@ pub enum Statement {
     SumOf(AnchoredKey, AnchoredKey, AnchoredKey),
     ProductOf(AnchoredKey, AnchoredKey, AnchoredKey),
     MaxOf(AnchoredKey, AnchoredKey, AnchoredKey),
-    Custom(CustomPredicateRef, Vec<AnchoredKey>),
+    Custom(CustomPredicateRef, Vec<WildcardValue>),
 }
 
 impl Statement {
@@ -95,7 +138,7 @@ impl Statement {
             Self::SumOf(ak1, ak2, ak3) => vec![Key(ak1), Key(ak2), Key(ak3)],
             Self::ProductOf(ak1, ak2, ak3) => vec![Key(ak1), Key(ak2), Key(ak3)],
             Self::MaxOf(ak1, ak2, ak3) => vec![Key(ak1), Key(ak2), Key(ak3)],
-            Self::Custom(_, args) => Vec::from_iter(args.into_iter().map(Key)),
+            Self::Custom(_, args) => Vec::from_iter(args.into_iter().map(WildcardLiteral)),
         }
     }
     pub fn from_args(pred: Predicate, args: Vec<StatementArg>) -> Result<Self> {
@@ -103,35 +146,45 @@ impl Statement {
         let st: Result<Self> = match pred {
             Native(NativePredicate::None) => Ok(Self::None),
             Native(NativePredicate::ValueOf) => {
-                if let (StatementArg::Key(a0), StatementArg::Literal(v1)) = (args[0], args[1]) {
+                if let (StatementArg::Key(a0), StatementArg::Literal(v1)) =
+                    (args[0].clone(), args[1].clone())
+                {
                     Ok(Self::ValueOf(a0, v1))
                 } else {
                     Err(anyhow!("Incorrect statement args"))
                 }
             }
             Native(NativePredicate::Equal) => {
-                if let (StatementArg::Key(a0), StatementArg::Key(a1)) = (args[0], args[1]) {
+                if let (StatementArg::Key(a0), StatementArg::Key(a1)) =
+                    (args[0].clone(), args[1].clone())
+                {
                     Ok(Self::Equal(a0, a1))
                 } else {
                     Err(anyhow!("Incorrect statement args"))
                 }
             }
             Native(NativePredicate::NotEqual) => {
-                if let (StatementArg::Key(a0), StatementArg::Key(a1)) = (args[0], args[1]) {
+                if let (StatementArg::Key(a0), StatementArg::Key(a1)) =
+                    (args[0].clone(), args[1].clone())
+                {
                     Ok(Self::NotEqual(a0, a1))
                 } else {
                     Err(anyhow!("Incorrect statement args"))
                 }
             }
             Native(NativePredicate::Gt) => {
-                if let (StatementArg::Key(a0), StatementArg::Key(a1)) = (args[0], args[1]) {
+                if let (StatementArg::Key(a0), StatementArg::Key(a1)) =
+                    (args[0].clone(), args[1].clone())
+                {
                     Ok(Self::Gt(a0, a1))
                 } else {
                     Err(anyhow!("Incorrect statement args"))
                 }
             }
             Native(NativePredicate::Lt) => {
-                if let (StatementArg::Key(a0), StatementArg::Key(a1)) = (args[0], args[1]) {
+                if let (StatementArg::Key(a0), StatementArg::Key(a1)) =
+                    (args[0].clone(), args[1].clone())
+                {
                     Ok(Self::Lt(a0, a1))
                 } else {
                     Err(anyhow!("Incorrect statement args"))
@@ -139,7 +192,7 @@ impl Statement {
             }
             Native(NativePredicate::Contains) => {
                 if let (StatementArg::Key(a0), StatementArg::Key(a1), StatementArg::Key(a2)) =
-                    (args[0], args[1], args[2])
+                    (args[0].clone(), args[1].clone(), args[2].clone())
                 {
                     Ok(Self::Contains(a0, a1, a2))
                 } else {
@@ -147,7 +200,9 @@ impl Statement {
                 }
             }
             Native(NativePredicate::NotContains) => {
-                if let (StatementArg::Key(a0), StatementArg::Key(a1)) = (args[0], args[1]) {
+                if let (StatementArg::Key(a0), StatementArg::Key(a1)) =
+                    (args[0].clone(), args[1].clone())
+                {
                     Ok(Self::NotContains(a0, a1))
                 } else {
                     Err(anyhow!("Incorrect statement args"))
@@ -155,7 +210,7 @@ impl Statement {
             }
             Native(NativePredicate::SumOf) => {
                 if let (StatementArg::Key(a0), StatementArg::Key(a1), StatementArg::Key(a2)) =
-                    (args[0], args[1], args[2])
+                    (args[0].clone(), args[1].clone(), args[2].clone())
                 {
                     Ok(Self::SumOf(a0, a1, a2))
                 } else {
@@ -164,7 +219,7 @@ impl Statement {
             }
             Native(NativePredicate::ProductOf) => {
                 if let (StatementArg::Key(a0), StatementArg::Key(a1), StatementArg::Key(a2)) =
-                    (args[0], args[1], args[2])
+                    (args[0].clone(), args[1].clone(), args[2].clone())
                 {
                     Ok(Self::ProductOf(a0, a1, a2))
                 } else {
@@ -173,23 +228,24 @@ impl Statement {
             }
             Native(NativePredicate::MaxOf) => {
                 if let (StatementArg::Key(a0), StatementArg::Key(a1), StatementArg::Key(a2)) =
-                    (args[0], args[1], args[2])
+                    (args[0].clone(), args[1].clone(), args[2].clone())
                 {
                     Ok(Self::MaxOf(a0, a1, a2))
                 } else {
                     Err(anyhow!("Incorrect statement args"))
                 }
             }
+            Native(np) => Err(anyhow!("Predicate {:?} is syntax sugar", np)),
             BatchSelf(_) => unreachable!(),
             Custom(cpr) => {
-                let ak_args: Result<Vec<AnchoredKey>> = args
+                let v_args: Result<Vec<WildcardValue>> = args
                     .iter()
                     .map(|x| match x {
-                        StatementArg::Key(ak) => Ok(*ak),
+                        StatementArg::WildcardLiteral(v) => Ok(v.clone()),
                         _ => Err(anyhow!("Incorrect statement args")),
                     })
                     .collect();
-                Ok(Self::Custom(cpr, ak_args?))
+                Ok(Self::Custom(cpr, v_args?))
             }
         };
         st
@@ -207,23 +263,24 @@ impl ToFields for Statement {
 
 impl fmt::Display for Statement {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{:?} ", self.predicate())?;
+        write!(f, "{}(", self.predicate())?;
         for (i, arg) in self.args().iter().enumerate() {
             if i != 0 {
-                write!(f, " ")?;
+                write!(f, ", ")?;
             }
             write!(f, "{}", arg)?;
         }
-        Ok(())
+        write!(f, ")")
     }
 }
 
 /// Statement argument type. Useful for statement decompositions.
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Clone, Debug, PartialEq)]
 pub enum StatementArg {
     None,
     Literal(Value),
     Key(AnchoredKey),
+    WildcardLiteral(WildcardValue),
 }
 
 impl fmt::Display for StatementArg {
@@ -231,7 +288,8 @@ impl fmt::Display for StatementArg {
         match self {
             StatementArg::None => write!(f, "none"),
             StatementArg::Literal(v) => write!(f, "{}", v),
-            StatementArg::Key(r) => write!(f, "{}.{}", r.0, r.1),
+            StatementArg::Key(r) => write!(f, "{}.{}", r.pod_id, r.key),
+            StatementArg::WildcardLiteral(v) => write!(f, "{}", v),
         }
     }
 }
@@ -242,13 +300,13 @@ impl StatementArg {
     }
     pub fn literal(&self) -> Result<Value> {
         match self {
-            Self::Literal(value) => Ok(*value),
+            Self::Literal(value) => Ok(value.clone()),
             _ => Err(anyhow!("Statement argument {:?} is not a literal.", self)),
         }
     }
     pub fn key(&self) -> Result<AnchoredKey> {
         match self {
-            Self::Key(ak) => Ok(*ak),
+            Self::Key(ak) => Ok(ak.clone()),
             _ => Err(anyhow!("Statement argument {:?} is not a key.", self)),
         }
     }
@@ -265,16 +323,23 @@ impl ToFields for StatementArg {
         // dealing with `Literal` it would be of length 4.
         let f = match self {
             StatementArg::None => vec![F::ZERO; STATEMENT_ARG_F_LEN],
-            StatementArg::Literal(v) => {
-                v.0.into_iter()
-                    .chain(iter::repeat(F::ZERO).take(STATEMENT_ARG_F_LEN - VALUE_SIZE))
-                    .collect()
-            }
+            StatementArg::Literal(v) => v
+                .raw()
+                .0
+                .into_iter()
+                .chain(iter::repeat(F::ZERO).take(STATEMENT_ARG_F_LEN - VALUE_SIZE))
+                .collect(),
             StatementArg::Key(ak) => {
-                let mut fields = ak.0.to_fields(_params);
-                fields.extend(ak.1.to_fields(_params));
+                let mut fields = ak.pod_id.to_fields(_params);
+                fields.extend(ak.key.to_fields(_params));
                 fields
             }
+            StatementArg::WildcardLiteral(v) => v
+                .raw()
+                .0
+                .into_iter()
+                .chain(iter::repeat(F::ZERO).take(STATEMENT_ARG_F_LEN - VALUE_SIZE))
+                .collect(),
         };
         assert_eq!(f.len(), STATEMENT_ARG_F_LEN); // sanity check
         f