From 86560857a7b91c205aa105a434607a31589236ad Mon Sep 17 00:00:00 2001 From: Erick Tryzelaar Date: Wed, 28 May 2014 10:46:29 -0700 Subject: [PATCH] add a JsonDeserializer, clean up tests --- bench_enum.rs | 4 +- bench_struct.rs | 8 +- de.rs | 24 ++-- json.rs | 314 +++++++++++++++++++++++++++++++++++++++++++----- 4 files changed, 304 insertions(+), 46 deletions(-) diff --git a/bench_enum.rs b/bench_enum.rs index 231f20a1..b9492266 100644 --- a/bench_enum.rs +++ b/bench_enum.rs @@ -223,13 +223,13 @@ mod deserializer { match self.stack.pop() { Some(AnimalState(Dog)) => { self.stack.push(EndState); - Some(Ok(EnumStart("Animal", "Dog"))) + Some(Ok(EnumStart("Animal", "Dog", 0))) } Some(AnimalState(Frog(x0, x1))) => { self.stack.push(EndState); self.stack.push(IntState(x1)); self.stack.push(StringState(x0)); - Some(Ok(EnumStart("Animal", "Frog"))) + Some(Ok(EnumStart("Animal", "Frog", 2))) } Some(IntState(x)) => { Some(Ok(Int(x))) diff --git a/bench_struct.rs b/bench_struct.rs index cb711b68..eb76fbf3 100644 --- a/bench_struct.rs +++ b/bench_struct.rs @@ -19,7 +19,7 @@ impl> Deserializable for Inner { #[inline] fn deserialize_token(d: &mut D, token: Token) -> Result { match token { - de::StructStart("Inner") | + de::StructStart("Inner", _) | de::MapStart(_) => { let mut a = None; let mut b = None; @@ -83,7 +83,7 @@ impl> Deserializable for Outer { #[inline] fn deserialize_token(d: &mut D, token: Token) -> Result { match token { - de::StructStart("Outer") | + de::StructStart("Outer", _) | de::MapStart(_) => { let mut inner = None; @@ -390,7 +390,7 @@ mod deserializer { self.stack.push(EndState); self.stack.push(VecState(inner)); self.stack.push(FieldState("inner")); - Some(Ok(StructStart("Outer"))) + Some(Ok(StructStart("Outer", 1))) } Some(InnerState(Inner { a: (), b, c })) => { self.stack.push(EndState); @@ -402,7 +402,7 @@ mod deserializer { self.stack.push(NullState); self.stack.push(FieldState("a")); - Some(Ok(StructStart("Inner"))) + Some(Ok(StructStart("Inner", 3))) } Some(FieldState(name)) => Some(Ok(Str(name))), Some(VecState(value)) => { diff --git a/de.rs b/de.rs index ea2b1d6b..6fbe8a72 100644 --- a/de.rs +++ b/de.rs @@ -26,8 +26,8 @@ pub enum Token { Option(bool), TupleStart(uint), - StructStart(&'static str), - EnumStart(&'static str, &'static str), + StructStart(&'static str, uint), + EnumStart(&'static str, &'static str, uint), SeqStart(uint), MapStart(uint), @@ -154,7 +154,7 @@ pub trait Deserializer: Iterator> { #[inline] fn expect_struct_start(&mut self, token: Token, name: &str) -> Result<(), E> { match token { - StructStart(n) => { + StructStart(n, _) => { if name == n { Ok(()) } else { @@ -187,9 +187,9 @@ pub trait Deserializer: Iterator> { } #[inline] - fn expect_enum_start<'a>(&mut self, token: Token, name: &str, variants: &[&str]) -> Result { + fn expect_enum_start(&mut self, token: Token, name: &str, variants: &[&str]) -> Result { match token { - EnumStart(n, v) => { + EnumStart(n, v, _) => { if name == n { match variants.iter().position(|variant| *variant == v) { Some(position) => Ok(position), @@ -511,7 +511,7 @@ pub fn ignore_token>(d: &mut D, token: Token) -> Result<() Option(true) => { ignore(d) } Option(false) => { Ok(()) } - EnumStart(_, _) => { + EnumStart(_, _, _) => { loop { match try!(d.expect_token()) { End => { return Ok(()); } @@ -520,7 +520,7 @@ pub fn ignore_token>(d: &mut D, token: Token) -> Result<() } } - StructStart(_) => { + StructStart(_, _) => { loop { match try!(d.expect_token()) { End => { return Ok(()); } @@ -817,7 +817,7 @@ mod tests { #[test] fn test_tokens_struct_empty() { let tokens = vec!( - StructStart("Outer"), + StructStart("Outer", 1), Str("inner"), SeqStart(0), End, @@ -833,10 +833,10 @@ mod tests { #[test] fn test_tokens_struct() { let tokens = vec!( - StructStart("Outer"), + StructStart("Outer", 1), Str("inner"), SeqStart(1), - StructStart("Inner"), + StructStart("Inner", 3), Str("a"), Null, @@ -877,7 +877,7 @@ mod tests { #[test] fn test_tokens_enum() { let tokens = vec!( - EnumStart("Animal", "Dog"), + EnumStart("Animal", "Dog", 0), End, ); @@ -887,7 +887,7 @@ mod tests { assert_eq!(value, Dog); let tokens = vec!( - EnumStart("Animal", "Frog"), + EnumStart("Animal", "Frog", 2), String("Henry".to_strbuf()), Int(349), End, diff --git a/json.rs b/json.rs index fb840a52..c13004f8 100644 --- a/json.rs +++ b/json.rs @@ -244,24 +244,158 @@ use std::str::ScalarValue; use std::str; use std::string::String; use std::vec::Vec; +use std::vec; use de; use collections::{HashMap, TreeMap}; +use collections::treemap; /// Represents a json value -#[deriving(Clone, Eq)] +#[deriving(Clone, Eq, Show)] pub enum Json { + Null, + Boolean(bool), Number(f64), String(String), - Boolean(bool), List(List), - Object(Box), - Null, + Object(Object), } pub type List = Vec; pub type Object = TreeMap; +impl> de::Deserializable for Json { + #[inline] + fn deserialize_token(d: &mut D, token: de::Token) -> Result { + match token { + de::Null => Ok(Null), + de::Bool(x) => Ok(Boolean(x)), + de::Int(x) => Ok(Number(x as f64)), + de::I8(x) => Ok(Number(x as f64)), + de::I16(x) => Ok(Number(x as f64)), + de::I32(x) => Ok(Number(x as f64)), + de::I64(x) => Ok(Number(x as f64)), + de::Uint(x) => Ok(Number(x as f64)), + de::U8(x) => Ok(Number(x as f64)), + de::U16(x) => Ok(Number(x as f64)), + de::U32(x) => Ok(Number(x as f64)), + de::U64(x) => Ok(Number(x as f64)), + de::F32(x) => Ok(Number(x as f64)), + de::F64(x) => Ok(Number(x)), + de::Char(x) => Ok(String(x.to_str())), + de::Str(x) => Ok(String(x.to_str())), + de::String(x) => Ok(String(x)), + de::Option(false) => Ok(Null), + de::Option(true) => de::Deserializable::deserialize(d), + de::TupleStart(_) | de::SeqStart(_) => { + let list = try!(de::Deserializable::deserialize_token(d, token)); + Ok(List(list)) + } + de::StructStart(_, _) | de::MapStart(_) => { + let object = try!(de::Deserializable::deserialize_token(d, token)); + Ok(Object(object)) + } + de::EnumStart(_, name, len) => { + let token = de::SeqStart(len); + let fields: Vec = try!(de::Deserializable::deserialize_token(d, token)); + if fields.is_empty() { + Ok(String(name.to_strbuf())) + } else { + let mut object = TreeMap::new(); + object.insert("variant".to_strbuf(), String(name.to_strbuf())); + object.insert("fields".to_strbuf(), List(fields)); + Ok(Object(object)) + } + } + de::End => Err(d.syntax_error()), + } + } +} + +enum JsonDeserializerState { + JsonDeserializerValueState(Json), + JsonDeserializerListState(vec::MoveItems), + JsonDeserializerObjectState(treemap::MoveEntries), +} + +struct JsonDeserializer { + stack: Vec, +} + +impl JsonDeserializer { + /// Creates a new decoder instance for decoding the specified JSON value. + pub fn new(json: Json) -> JsonDeserializer { + JsonDeserializer { + stack: vec!(JsonDeserializerValueState(json)), + } + } +} + +impl Iterator> for JsonDeserializer { + #[inline] + fn next(&mut self) -> Option> { + loop { + match self.stack.pop() { + Some(JsonDeserializerValueState(value)) => { + let token = match value { + Null => de::Null, + Boolean(x) => de::Bool(x), + Number(x) => de::F64(x), + String(x) => de::String(x), + List(x) => { + let len = x.len(); + self.stack.push(JsonDeserializerListState(x.move_iter())); + de::SeqStart(len) + } + Object(x) => { + let len = x.len(); + self.stack.push(JsonDeserializerObjectState(x.move_iter())); + de::MapStart(len) + } + }; + + return Some(Ok(token)); + } + Some(JsonDeserializerListState(mut iter)) => { + match iter.next() { + Some(value) => { + self.stack.push(JsonDeserializerListState(iter)); + self.stack.push(JsonDeserializerValueState(value)); + // loop around. + } + None => { + return Some(Ok(de::End)); + } + } + } + Some(JsonDeserializerObjectState(mut iter)) => { + match iter.next() { + Some((key, value)) => { + self.stack.push(JsonDeserializerObjectState(iter)); + self.stack.push(JsonDeserializerValueState(value)); + return Some(Ok(de::String(key))); + } + None => { + return Some(Ok(de::End)); + } + } + } + None => { return None; } + } + } + } +} + +impl de::Deserializer for JsonDeserializer { + fn end_of_stream_error(&self) -> ParserError { + SyntaxError(EOFWhileParsingValue, 0, 0) + } + + fn syntax_error(&self) -> ParserError { + SyntaxError(InvalidSyntax, 0, 0) + } +} + /// The errors that can arise while parsing a JSON stream. #[deriving(Clone, Eq)] pub enum ErrorCode { @@ -1615,6 +1749,60 @@ impl> de::Deserializer for Parser { } } } + + /* + #[inline] + fn expect_enum_start(&mut self, token: de::Token, _name: &str, variants: &[&str]) -> Result { + match token { + Str(name) => + + } + + let name = match self.pop() { + String(s) => s, + Object(mut o) => { + let n = match o.pop(&"variant".to_strbuf()) { + Some(String(s)) => s, + Some(val) => { + return Err(ExpectedError("String".to_strbuf(), + format_strbuf!("{}", val))) + } + None => { + return Err(MissingFieldError("variant".to_strbuf())) + } + }; + match o.pop(&"fields".to_strbuf()) { + Some(List(l)) => { + for field in l.move_iter().rev() { + self.stack.push(field.clone()); + } + }, + Some(val) => { + return Err(ExpectedError("List".to_strbuf(), + format_strbuf!("{}", val))) + } + None => { + return Err(MissingFieldError("fields".to_strbuf())) + } + } + n + } + json => { + return Err(ExpectedError("String or Object".to_strbuf(), + format_strbuf!("{}", json))) + } + }; + let idx = match names.iter() + .position(|n| { + str::eq_slice(*n, name.as_slice()) + }) { + Some(idx) => idx, + None => return Err(UnknownVariantError(name)) + }; + f(self, idx) + + } + */ } /* @@ -1733,6 +1921,15 @@ pub fn from_iter< } +/// Decodes a json value from a `Json`. +pub fn from_json< + T: de::Deserializable +>(json: Json) -> Result { + let mut d = JsonDeserializer::new(json); + de::Deserializable::deserialize(&mut d) +} + + /* @@ -1757,7 +1954,6 @@ pub fn from_str(s: &str) -> Result { } */ - /* /// A structure to decode JSON to values in rust. pub struct Decoder { stack: Vec, @@ -1797,7 +1993,9 @@ macro_rules! expect( }) ) + /* impl ::Decoder for Decoder { + fn read_nil(&mut self) -> DecodeResult<()> { debug!("read_nil"); try!(expect!(self.pop(), Null)); @@ -2206,7 +2404,7 @@ impl ToJson for TreeMap { for (key, value) in self.iter() { d.insert((*key).clone(), value.to_json()); } - Object(box d) + Object(d) } } @@ -2216,7 +2414,7 @@ impl ToJson for HashMap { for (key, value) in self.iter() { d.insert((*key).clone(), value.to_json()); } - Object(box d) + Object(d) } } @@ -2255,7 +2453,17 @@ mod tests { TrailingCharacters}; */ + use super::{ + Json, + Null, + Boolean, + Number, + String, + List, + Object, + }; use super::{Parser, ParserError, from_iter}; + use super::{JsonDeserializer, from_json}; use super::{ EOFWhileParsingList, EOFWhileParsingObject, @@ -2270,7 +2478,9 @@ mod tests { }; use de; + use std::fmt::Show; use std::io; + use std::str; use collections::TreeMap; #[deriving(Eq, Show)] @@ -2311,7 +2521,7 @@ mod tests { #[inline] fn deserialize_token(d: &mut D, token: de::Token) -> Result { match token { - de::StructStart("Inner") | + de::StructStart("Inner", _) | de::MapStart(_) => { let mut a = None; let mut b = None; @@ -2373,7 +2583,7 @@ mod tests { #[inline] fn deserialize_token(d: &mut D, token: de::Token) -> Result { match token { - de::StructStart("Outer") | + de::StructStart("Outer", _) | de::MapStart(_) => { let mut inner = None; @@ -2664,44 +2874,82 @@ mod tests { } */ + fn test_parser_err< + 'a, + T: Eq + Show + de::Deserializable>> + >(errors: &[(&'a str, ParserError)]) { + for &(s, err) in errors.iter() { + let v: Result = from_iter(s.chars()); + assert_eq!(v, Err(err)); + } + } + + fn test_parser_ok< + 'a, + T: Eq + Show + de::Deserializable>> + >(errors: &[(&'a str, T)]) { + for &(s, ref value) in errors.iter() { + let v: T = from_iter(s.chars()).unwrap(); + assert_eq!(v, *value); + } + } + + fn test_json_deserializer_ok< + T: Eq + Show + de::Deserializable + >(errors: &[(Json, T)]) { + for &(ref json, ref value) in errors.iter() { + let v: T = from_json(json.clone()).unwrap(); + assert_eq!(v, *value); + } + } + #[test] fn test_decode_null() { - let errors = [ + test_parser_err::<()>([ ("n", SyntaxError(InvalidSyntax, 1, 2)), ("nul", SyntaxError(InvalidSyntax, 1, 4)), ("nulla", SyntaxError(TrailingCharacters, 1, 5)), - ]; + ]); - for &(s, err) in errors.iter() { - let v: Result<(), ParserError> = from_iter(s.chars()); - assert_eq!(v, Err(err)); - } + test_parser_ok([ + ("null", ()), + ]); - let v: () = from_iter("null".chars()).unwrap(); - assert_eq!(v, ()); + test_parser_ok([ + ("null", Null), + ]); + + test_json_deserializer_ok([ + (Null, ()), + ]); } #[test] fn test_decode_bool() { - let errors = [ + test_parser_err::([ ("t", SyntaxError(InvalidSyntax, 1, 2)), ("truz", SyntaxError(InvalidSyntax, 1, 4)), ("f", SyntaxError(InvalidSyntax, 1, 2)), ("faz", SyntaxError(InvalidSyntax, 1, 3)), ("truea", SyntaxError(TrailingCharacters, 1, 5)), ("falsea", SyntaxError(TrailingCharacters, 1, 6)), - ]; + ]); - for &(s, err) in errors.iter() { - let v: Result = from_iter(s.chars()); - assert_eq!(v, Err(err)); - } + test_parser_ok([ + ("true", true), + ("false", false), + ]); - let v: bool = from_iter("true".chars()).unwrap(); - assert_eq!(v, true); + test_parser_ok([ + ("true", Boolean(true)), + ("false", Boolean(false)), + ]); - let v: bool = from_iter("false".chars()).unwrap(); - assert_eq!(v, false); + + test_json_deserializer_ok([ + (Boolean(true), true), + (Boolean(false), false), + ]); } /* @@ -3023,9 +3271,16 @@ mod tests { assert_eq!(value, Ok(Some("jodhpurs".to_strbuf()))); } - /* + /* #[test] fn test_decode_enum() { + let value: Result = from_iter("\"Dog\"".chars()); + assert_eq!(value, Ok(Dog)); + + let s = "{\"variant\":\"Frog\",\"fields\":[\"Henry\",349]}"; + let value: Result = from_iter(s.chars()); + assert_eq!(value, Ok(Frog("Henry".to_strbuf(), 349))); + let mut decoder = Decoder::new(from_str("\"Dog\"").unwrap()); let value: Animal = Decodable::decode(&mut decoder).unwrap(); assert_eq!(value, Dog); @@ -3034,8 +3289,11 @@ mod tests { let mut decoder = Decoder::new(from_str(s).unwrap()); let value: Animal = Decodable::decode(&mut decoder).unwrap(); assert_eq!(value, Frog("Henry".to_strbuf(), 349)); + assert_eq!(value, Dog); } + */ + /* #[test] fn test_decode_map() { let s = "{\"a\": \"Dog\", \"b\": {\"variant\":\"Frog\",\