From d82c7f8ef21e5c3cee922d3a9c8f8ca43c3a61f3 Mon Sep 17 00:00:00 2001 From: Nick Alexander Date: Sat, 30 Jun 2018 16:21:50 -0700 Subject: [PATCH] Cull unused `mentat_parser_utils` crate. With the transition toward parsing with `rust-peg` and away from `combine`, we're not using some of the many helpers we built to support our unusual `combine` usage. They can just go! --- Cargo.toml | 3 - README.md | 4 - parser-utils/Cargo.toml | 12 - parser-utils/src/lib.rs | 72 --- parser-utils/src/log.rs | 71 --- parser-utils/src/macros.rs | 137 ----- parser-utils/src/value_and_span.rs | 795 ----------------------------- src/conn.rs | 1 - src/store.rs | 1 - tools/cli/Cargo.toml | 3 - 10 files changed, 1099 deletions(-) delete mode 100644 parser-utils/Cargo.toml delete mode 100644 parser-utils/src/lib.rs delete mode 100644 parser-utils/src/log.rs delete mode 100644 parser-utils/src/macros.rs delete mode 100644 parser-utils/src/value_and_span.rs diff --git a/Cargo.toml b/Cargo.toml index 93ee9ca6..b3a5061d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,9 +42,6 @@ features = ["limits"] [dependencies.edn] path = "edn" -[dependencies.mentat_parser_utils] -path = "parser-utils" - [dependencies.mentat_core] path = "core" diff --git a/README.md b/README.md index e8602d5c..6f246ff6 100644 --- a/README.md +++ b/README.md @@ -169,10 +169,6 @@ This is the lowest-level Mentat crate. It collects together the following things - Common utilities (some in the `util` module, and others that should be moved there or broken out) like `Either`, `InternSet`, and `RcCounter`. - Reusable lazy namespaced keywords (_e.g._, `DB_TYPE_DOUBLE`) that are used by `mentat_db` and EDN serialization of core structs. -#### `mentat_parser_utils` - -This is a utility library for writing `combine` parsers over streams of `edn::Value`/`edn::ValueAndSpan`. - ### Types #### `mentat_query` diff --git a/parser-utils/Cargo.toml b/parser-utils/Cargo.toml deleted file mode 100644 index 6484bced..00000000 --- a/parser-utils/Cargo.toml +++ /dev/null @@ -1,12 +0,0 @@ -[package] -name = "mentat_parser_utils" -version = "0.1.0" -authors = ["Victor Porof ", "Richard Newman "] -workspace = ".." - -[dependencies] -combine = "2.3.2" -itertools = "0.7" - -[dependencies.edn] - path = "../edn" diff --git a/parser-utils/src/lib.rs b/parser-utils/src/lib.rs deleted file mode 100644 index c0408f72..00000000 --- a/parser-utils/src/lib.rs +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2016 Mozilla -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -extern crate combine; -extern crate edn; -extern crate itertools; - -/// A `ValueParseError` is a `combine::primitives::ParseError`-alike that implements the `Debug`, -/// `Display`, and `std::error::Error` traits. In addition, it doesn't capture references. -/// -/// This is achieved by wrapping slices of type `&'a [edn::Value]` in an owning type that implements -/// `Display`; rather than introducing a newtype like `DisplayVec`, we re-use `edn::Value::Vector`. -#[derive(PartialEq)] -pub struct ValueParseError { - pub position: edn::Span, - // Think of this as `Vec>>`; see above. - pub errors: Vec>, -} - -#[macro_use] -pub mod macros; - -pub use macros::{ - KeywordMapParser, - ResultParser, -}; - -pub mod log; -pub mod value_and_span; -pub use value_and_span::{ - Stream, -}; - -impl std::fmt::Debug for ValueParseError { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, - "ParseError {{ position: {:?}, errors: {:?} }}", - self.position, - self.errors) - } -} - -impl std::fmt::Display for ValueParseError { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - try!(writeln!(f, "Parse error at {:?}", self.position)); - combine::primitives::Error::fmt_errors(&self.errors, f) - } -} - -impl std::error::Error for ValueParseError { - fn description(&self) -> &str { - "parse error parsing EDN values" - } -} - -impl<'a> From>> for ValueParseError { - fn from(e: combine::primitives::ParseError>) -> ValueParseError { - ValueParseError { - position: e.position.0, - errors: e.errors.into_iter() - .map(|e| e.map_token(|t| t.clone()).map_range(|r| r.clone())) - .collect(), - } - } -} diff --git a/parser-utils/src/log.rs b/parser-utils/src/log.rs deleted file mode 100644 index e1cac87f..00000000 --- a/parser-utils/src/log.rs +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2016 Mozilla -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -use combine::{ - ParseError, - Parser, - ParseResult, - Stream, -}; - -#[derive(Clone)] -pub(crate) struct Log(P, T) - where P: Parser, - T: ::std::fmt::Debug; - -impl Parser for Log - where I: Stream, - I::Item: ::std::fmt::Debug, - P: Parser, - P::Output: ::std::fmt::Debug, - T: ::std::fmt::Debug, -{ - type Input = I; - type Output = P::Output; - - fn parse_stream(&mut self, input: I) -> ParseResult { - let head = input.clone().uncons(); - let result = self.0.parse_stream(input.clone()); - match result { - Ok((ref value, _)) => eprintln!("{:?}: [{:?} ...] => Ok({:?})", self.1, head.ok(), value), - Err(_) => eprintln!("{:?}: [{:?} ...] => Err(_)", self.1, head.ok()), - } - result - } - - fn add_error(&mut self, errors: &mut ParseError) { - self.0.add_error(errors); - } -} - -#[inline(always)] -pub(crate) fn log(p: P, msg: T) -> Log - where P: Parser, - T: ::std::fmt::Debug, -{ - Log(p, msg) -} - -/// We need a trait to define `Parser.log` and have it live outside of the `combine` crate. -pub(crate) trait LogParsing: Parser + Sized { - fn log(self, msg: T) -> Log - where Self: Sized, - T: ::std::fmt::Debug; -} - -impl

LogParsing for P - where P: Parser, -{ - fn log(self, msg: T) -> Log - where T: ::std::fmt::Debug, - { - log(self, msg) - } -} diff --git a/parser-utils/src/macros.rs b/parser-utils/src/macros.rs deleted file mode 100644 index d6a3d0a2..00000000 --- a/parser-utils/src/macros.rs +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright 2016 Mozilla -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -// use combine::{ -// ParseResult, -// }; -// use combine::combinator::{ -// Expected, -// FnParser, -// }; - -use combine::{ - ParseResult, -}; -// use combine::primitives; // To not shadow Error. -// use combine::primitives::{ -// Consumed, -// FastResult, -// }; -use combine::combinator::{ - Expected, - FnParser, -}; - -/// A type definition for a function parser that either parses an `O` from an input stream of type -/// `I`, or fails with an "expected" failure. -/// See for more -/// illumination. -/// Nothing about this is specific to the result type of the parser. -pub type ResultParser = Expected ParseResult>>; - -pub struct KeywordMapParser(pub T); - -/// `satisfy_unwrap!` makes it a little easier to implement a `satisfy_map` -/// body that matches a particular `Value` enum case, otherwise returning `None`. -#[macro_export] -macro_rules! satisfy_unwrap { - ( $cas: path, $var: ident, $body: block ) => { - satisfy_map(|x: edn::Value| if let $cas($var) = x $body else { None }) - } -} - -/// Generate a `satisfy_map` expression that matches a `PlainSymbol` -/// value with the given name. -/// -/// We do this rather than using `combine::token` so that we don't -/// need to allocate a new `String` inside a `PlainSymbol` inside a `Value` -/// just to match input. -#[macro_export] -macro_rules! matches_plain_symbol { - ($name: expr, $input: ident) => { - satisfy_map(|x: edn::Value| { - if let edn::Value::PlainSymbol(ref s) = x { - if s.name() == $name { - return Some(()); - } - } - return None; - }).parse_stream($input) - } -} - -#[macro_export] -macro_rules! def_parser { - ( $parser: ident, $name: ident, $result_type: ty, $body: block ) => { - impl<'p> $parser<'p> { - fn $name<'a>() -> ResultParser<$result_type, $crate::value_and_span::Stream<'a>> { - fn inner<'a>(input: $crate::value_and_span::Stream<'a>) -> ParseResult<$result_type, $crate::value_and_span::Stream<'a>> { - $body.parse_lazy(input).into() - } - parser(inner as fn($crate::value_and_span::Stream<'a>) -> ParseResult<$result_type, $crate::value_and_span::Stream<'a>>).expected(stringify!($name)) - } - } - } -} - -/// `assert_parses_to!` simplifies some of the boilerplate around running a -/// parser function against input and expecting a certain result. -#[macro_export] -macro_rules! assert_parses_to { - ( $parser: expr, $input: expr, $expected: expr ) => {{ - let input = $input.with_spans(); - let par = $parser(); - let stream = input.atom_stream(); - let result = par.skip(eof()).parse(stream).map(|x| x.0); - assert_eq!(result, Ok($expected)); - }} -} - -/// `assert_edn_parses_to!` simplifies some of the boilerplate around running a parser function -/// against string input and expecting a certain result. -#[macro_export] -macro_rules! assert_edn_parses_to { - ( $parser: expr, $input: expr, $expected: expr ) => {{ - let input = edn::parse::value($input).expect("to be able to parse input as EDN"); - let par = $parser(); - let stream = input.atom_stream(); - let result = par.skip(eof()).parse(stream).map(|x| x.0); - assert_eq!(result, Ok($expected)); - }} -} - -/// `assert_parse_failure_contains!` simplifies running a parser function against string input and -/// expecting a certain failure. This is working around the complexity of pattern matching parse -/// errors that contain spans. -#[macro_export] -macro_rules! assert_parse_failure_contains { - ( $parser: expr, $input: expr, $expected: expr ) => {{ - let input = edn::parse::value($input).expect("to be able to parse input as EDN"); - let par = $parser(); - let stream = input.atom_stream(); - let result = par.skip(eof()).parse(stream).map(|x| x.0).map_err(|e| -> $crate::ValueParseError { e.into() }); - assert!(format!("{:?}", result).contains($expected), "Expected {:?} to contain {:?}", result, $expected); - }} -} - -#[macro_export] -macro_rules! keyword_map_of { - ($(($keyword:expr, $value:expr)),+) => {{ - let mut seen = std::collections::BTreeSet::default(); - - $( - if !seen.insert($keyword) { - panic!("keyword map has repeated key: {}", stringify!($keyword)); - } - )+ - - KeywordMapParser(($(($keyword, $value)),+)) - }} -} diff --git a/parser-utils/src/value_and_span.rs b/parser-utils/src/value_and_span.rs deleted file mode 100644 index 5715442c..00000000 --- a/parser-utils/src/value_and_span.rs +++ /dev/null @@ -1,795 +0,0 @@ -// Copyright 2016 Mozilla -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#![allow(dead_code)] - -use std; -use std::cmp::Ordering; -use std::fmt::{ - Debug, - Display, - Formatter, -}; - -use combine::{ - ConsumedResult, - ParseError, - Parser, - ParseResult, - StreamOnce, - parser, - satisfy_map, -}; -use combine::primitives; // To not shadow Error. -use combine::primitives::{ - FastResult, -}; -use combine::combinator::{ - Expected, - FnParser, -}; - -use edn; - -use macros::{ - KeywordMapParser, -}; - -/// A wrapper to let us order `edn::Span` in whatever way is appropriate for parsing with `combine`. -#[derive(Clone, Copy, Debug)] -pub struct SpanPosition(pub edn::Span); - -impl Display for SpanPosition { - fn fmt(&self, f: &mut Formatter) -> ::std::fmt::Result { - self.0.fmt(f) - } -} - -impl PartialEq for SpanPosition { - fn eq(&self, other: &Self) -> bool { - self.cmp(other) == Ordering::Equal - } -} - -impl Eq for SpanPosition { } - -impl PartialOrd for SpanPosition { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for SpanPosition { - fn cmp(&self, other: &Self) -> Ordering { - (self.0).0.cmp(&(other.0).0) - } -} - -/// An iterator specifically for iterating `edn::ValueAndSpan` instances in various ways. -/// -/// Enumerating each iteration type allows us to have a single `combine::Stream` implementation -/// yielding `ValueAndSpan` items, which allows us to yield uniform `combine::ParseError` types from -/// disparate parsers. -#[derive(Clone)] -pub enum Iter<'a> { - Empty, - Atom(std::iter::Once<&'a edn::ValueAndSpan>), - Vector(std::slice::Iter<'a, edn::ValueAndSpan>), - List(std::collections::linked_list::Iter<'a, edn::ValueAndSpan>), - /// Iterates a map {:k1 v1, :k2 v2, ...} as a single `flat_map` slice [k1, v1, k2, v2, ...]. - Map(std::iter::FlatMap, - std::iter::Chain, std::iter::Once<&'a edn::ValueAndSpan>>, - fn((&'a edn::ValueAndSpan, &'a edn::ValueAndSpan)) -> std::iter::Chain, std::iter::Once<&'a edn::ValueAndSpan>>>), - /// Iterates a map with vector values {:k1 [v11 v12 ...], :k2 [v21 v22 ...], ...} as a single - /// flattened map [k1, v11, v12, ..., k2, v21, v22, ...]. - KeywordMap(std::iter::FlatMap, - std::iter::Chain, Box>>, - fn((&'a edn::ValueAndSpan, &'a edn::ValueAndSpan)) -> std::iter::Chain, Box>>>), - // TODO: Support Set and Map more naturally. This is significantly more work because the - // existing BTreeSet and BTreeMap iterators do not implement Clone, and implementing Clone for - // them is involved. Since we don't really need to parse sets and maps at this time, this will - // do for now. -} - -impl<'a> Iterator for Iter<'a> { - type Item = &'a edn::ValueAndSpan; - - fn next(&mut self) -> Option { - match *self { - Iter::Empty => None, - Iter::Atom(ref mut i) => i.next(), - Iter::Vector(ref mut i) => i.next(), - Iter::List(ref mut i) => i.next(), - Iter::Map(ref mut i) => i.next(), - Iter::KeywordMap(ref mut i) => i.next(), - } - } -} - -/// A single `combine::Stream` implementation iterating `edn::ValueAndSpan` instances. Equivalent -/// to `combine::IteratorStream` as produced by `combine::from_iter`, but specialized to -/// `edn::ValueAndSpan`. -#[derive(Clone)] -pub struct Stream<'a>(Iter<'a>, SpanPosition); - -/// Things specific to parsing with `combine` and our `Stream` that need a trait to live outside of -/// the `edn` crate. -pub trait Item<'a>: Clone + PartialEq + Sized { - /// Position could be specialized to `SpanPosition`. - type Position: Clone + Ord + std::fmt::Display; - - /// A slight generalization of `combine::Positioner` that allows to set the position based on - /// the `edn::ValueAndSpan` being iterated. - fn start(&self) -> Self::Position; - fn update_position(&self, &mut Self::Position); - - fn child_iter(&'a self) -> Iter<'a>; - fn child_stream(&'a self) -> Stream<'a>; - fn atom_iter(&'a self) -> Iter<'a>; - fn atom_stream(&'a self) -> Stream<'a>; - - fn keyword_map_iter(&'a self) -> Iter<'a>; - fn keyword_map_stream(&'a self) -> Stream<'a>; -} - -impl<'a> Item<'a> for edn::ValueAndSpan { - type Position = SpanPosition; - - fn start(&self) -> Self::Position { - SpanPosition(self.span.clone()) - } - - fn update_position(&self, position: &mut Self::Position) { - *position = SpanPosition(self.span.clone()) - } - - fn keyword_map_iter(&'a self) -> Iter<'a> { - fn flatten_k_vector<'a>((k, v): (&'a edn::ValueAndSpan, &'a edn::ValueAndSpan)) -> std::iter::Chain, Box>> { - std::iter::once(k).chain(Box::new(v.child_iter())) - } - - match self.inner.as_map() { - Some(ref map) => Iter::KeywordMap(map.iter().flat_map(flatten_k_vector)), - None => Iter::Empty - } - } - - fn keyword_map_stream(&'a self) -> Stream<'a> { - let span = self.span.clone(); - Stream(self.keyword_map_iter(), SpanPosition(span)) - } - - fn child_iter(&'a self) -> Iter<'a> { - fn flatten_k_v<'a>((k, v): (&'a edn::ValueAndSpan, &'a edn::ValueAndSpan)) -> std::iter::Chain, std::iter::Once<&'a edn::ValueAndSpan>> { - std::iter::once(k).chain(std::iter::once(v)) - } - - match self.inner { - edn::SpannedValue::Vector(ref values) => Iter::Vector(values.iter()), - edn::SpannedValue::List(ref values) => Iter::List(values.iter()), - // Parsing pairs with `combine` is tricky; parsing sequences is easy. - edn::SpannedValue::Map(ref map) => Iter::Map(map.iter().flat_map(flatten_k_v)), - _ => Iter::Empty, - } - } - - fn child_stream(&'a self) -> Stream<'a> { - let span = self.span.clone(); - Stream(self.child_iter(), SpanPosition(span)) - } - - fn atom_iter(&'a self) -> Iter<'a> { - Iter::Atom(std::iter::once(self)) - } - - fn atom_stream(&'a self) -> Stream<'a> { - let span = self.span.clone(); - Stream(self.atom_iter(), SpanPosition(span)) - } -} - -/// `OfExactly` and `of_exactly` allow us to express nested parsers naturally. -/// -/// For example, `vector().of_exactly(many(list()))` parses a vector-of-lists, like [(1 2) (:a :b) ("test") ()]. -/// -/// The "outer" parser `P` and the "nested" parser `N` must be compatible: `P` must produce an -/// output `edn::ValueAndSpan` which can itself be turned into a stream of child elements; and `N` -/// must accept the resulting input `Stream`. This compatibility allows us to lift errors from the -/// nested parser to the outer parser, which is part of what has made parsing `&'a [edn::Value]` -/// difficult. -#[derive(Clone)] -pub struct OfExactly(P, N); - -pub trait Streaming<'a> { - fn as_stream(self) -> Stream<'a>; -} - -impl<'a> Streaming<'a> for &'a edn::ValueAndSpan { - fn as_stream(self) -> Stream<'a> { - self.child_stream() - } -} - -impl<'a> Streaming<'a> for Stream<'a> { - fn as_stream(self) -> Stream<'a> { - self - } -} - -impl<'a, P, N, M, O> Parser for OfExactly - where P: Parser, Output=M>, - N: Parser, Output=O>, - M: 'a + Streaming<'a>, -{ - type Input = P::Input; - type Output = O; - #[inline] - fn parse_lazy(&mut self, input: Self::Input) -> ConsumedResult { - use self::FastResult::*; - - match self.0.parse_lazy(input) { - ConsumedOk((outer_value, outer_input)) => { - match self.1.parse_lazy(outer_value.as_stream()) { - ConsumedOk((inner_value, mut inner_input)) | EmptyOk((inner_value, mut inner_input)) => { - match inner_input.uncons() { - Err(ref err) if *err == primitives::Error::end_of_input() => ConsumedOk((inner_value, outer_input)), - _ => EmptyErr(ParseError::empty(inner_input.position())), - } - }, - // TODO: Improve the error output to reference the nested value (or span) in - // some way. This seems surprisingly difficult to do, so we just surface the - // inner error message right now. See also the comment below. - EmptyErr(e) | ConsumedErr(e) => ConsumedErr(e), - } - }, - EmptyOk((outer_value, outer_input)) => { - match self.1.parse_lazy(outer_value.as_stream()) { - ConsumedOk((inner_value, mut inner_input)) | EmptyOk((inner_value, mut inner_input)) => { - match inner_input.uncons() { - Err(ref err) if *err == primitives::Error::end_of_input() => EmptyOk((inner_value, outer_input)), - _ => EmptyErr(ParseError::empty(inner_input.position())), - } - }, - // TODO: Improve the error output. See the comment above. - EmptyErr(e) | ConsumedErr(e) => EmptyErr(e), - } - }, - ConsumedErr(e) => ConsumedErr(e), - EmptyErr(e) => EmptyErr(e), - } - } - - fn add_error(&mut self, errors: &mut ParseError) { - self.0.add_error(errors); - } -} - -#[inline(always)] -pub fn of_exactly<'a, P, N, M, O>(p: P, n: N) -> OfExactly - where P: Parser, Output=M>, - N: Parser, Output=O>, - M: 'a + Streaming<'a>, -{ - OfExactly(p, n) -} - -/// We need a trait to define `Parser.of` and have it live outside of the `combine` crate. -pub trait OfExactlyParsing: Parser + Sized { - fn of_exactly(self, n: N) -> OfExactly - where Self: Sized, - N: Parser; -} - -impl<'a, P, M> OfExactlyParsing for P - where P: Parser, Output=M>, - M: 'a + Streaming<'a>, -{ - fn of_exactly(self, n: N) -> OfExactly - where N: Parser - { - of_exactly(self, n) - } -} - -/// Equivalent to `combine::IteratorStream`. -impl<'a> StreamOnce for Stream<'a> -{ - type Item = &'a edn::ValueAndSpan; - type Range = &'a edn::ValueAndSpan; - type Position = SpanPosition; - - #[inline] - fn uncons(&mut self) -> std::result::Result> { - match self.0.next() { - Some(x) => { - x.update_position(&mut self.1); - Ok(x) - }, - None => Err(primitives::Error::end_of_input()), - } - } - - #[inline(always)] - fn position(&self) -> Self::Position { - self.1.clone() - } -} - -/// Shorthands, just enough to convert the `mentat_db` crate for now. Written using `Box` for now: -/// it's simple and we can address allocation issues if and when they surface. -pub fn vector_<'a>(input: Stream<'a>) -> ParseResult, Stream<'a>> { - satisfy_map(|v: &'a edn::ValueAndSpan| { - if v.inner.is_vector() { - Some(v.child_stream()) - } else { - None - } - }) - .parse_lazy(input) - .into() -} - -pub fn vector<'a>() -> Expected, fn(Stream<'a>) -> ParseResult, Stream<'a>>>> { - parser(vector_ as fn(Stream<'a>) -> ParseResult, Stream<'a>>).expected("vector") -} - -pub fn list_<'a>(input: Stream<'a>) -> ParseResult, Stream<'a>> { - satisfy_map(|v: &'a edn::ValueAndSpan| { - if v.inner.is_list() { - Some(v.child_stream()) - } else { - None - } - }) - .parse_lazy(input) - .into() -} - -pub fn list<'a>() -> Expected, fn(Stream<'a>) -> ParseResult, Stream<'a>>>> { - parser(list_ as fn(Stream<'a>) -> ParseResult, Stream<'a>>).expected("list") -} - -pub fn seq_<'a>(input: Stream<'a>) -> ParseResult, Stream<'a>> { - satisfy_map(|v: &'a edn::ValueAndSpan| { - if v.inner.is_list() || v.inner.is_vector() { - Some(v.child_stream()) - } else { - None - } - }) - .parse_lazy(input) - .into() -} - -pub fn seq<'a>() -> Expected, fn(Stream<'a>) -> ParseResult, Stream<'a>>>> { - parser(seq_ as fn(Stream<'a>) -> ParseResult, Stream<'a>>).expected("vector|list") -} - -pub fn map_<'a>(input: Stream<'a>) -> ParseResult, Stream<'a>> { - satisfy_map(|v: &'a edn::ValueAndSpan| { - if v.inner.is_map() { - Some(v.child_stream()) - } else { - None - } - }) - .parse_lazy(input) - .into() -} - -pub fn map<'a>() -> Expected, fn(Stream<'a>) -> ParseResult, Stream<'a>>>> { - parser(map_ as fn(Stream<'a>) -> ParseResult, Stream<'a>>).expected("map") -} - -/// A `[k v]` pair in the map form of a keyword map must have the shape `[:k, [v1, v2, ...]]`, with -/// none of `v1`, `v2`, ... a keyword: without loss of generality, we cannot represent the case -/// where `vn` is a keyword `:l`, since `[:k v1 v2 ... :l]`, isn't a valid keyword map in vector -/// form. This function tests that a `[k v]` pair obeys these constraints. -/// -/// If we didn't test this, then we might flatten a map `[:k [:l]] to `[:k :l]`, which isn't a valid -/// keyword map in vector form. -pub fn is_valid_keyword_map_k_v<'a>((k, v): (&'a edn::ValueAndSpan, &'a edn::ValueAndSpan)) -> bool { - if !k.inner.is_keyword() { - return false; - } - match v.inner.as_vector() { - None => { - return false; - }, - Some(ref vs) => { - if !vs.iter().all(|vv| !vv.inner.is_keyword()) { - return false; - } - }, - } - return true; -} - -pub fn keyword_map_<'a>(input: Stream<'a>) -> ParseResult, Stream<'a>> { - satisfy_map(|v: &'a edn::ValueAndSpan| { - v.inner.as_map().and_then(|map| { - if map.iter().all(is_valid_keyword_map_k_v) { - println!("yes {:?}", map); - Some(v.keyword_map_stream()) - } else { - println!("no {:?}", map); - None - } - }) - }) - .parse_lazy(input) - .into() -} - -pub fn keyword_map<'a>() -> Expected, fn(Stream<'a>) -> ParseResult, Stream<'a>>>> { - parser(keyword_map_ as fn(Stream<'a>) -> ParseResult, Stream<'a>>).expected("keyword map") -} - -pub fn integer_<'a>(input: Stream<'a>) -> ParseResult> { - satisfy_map(|v: &'a edn::ValueAndSpan| v.inner.as_integer()) - .parse_lazy(input) - .into() -} - -pub fn integer<'a>() -> Expected, fn(Stream<'a>) -> ParseResult>>> { - parser(integer_ as fn(Stream<'a>) -> ParseResult>).expected("integer") -} - -pub fn any_keyword_<'a>(input: Stream<'a>) -> ParseResult<&'a edn::Keyword, Stream<'a>> { - satisfy_map(|v: &'a edn::ValueAndSpan| v.inner.as_keyword()) - .parse_lazy(input) - .into() -} - -pub fn namespaced_keyword_<'a>(input: Stream<'a>) -> ParseResult<&'a edn::Keyword, Stream<'a>> { - satisfy_map(|v: &'a edn::ValueAndSpan| v.inner.as_namespaced_keyword()) - .parse_lazy(input) - .into() -} - -pub fn any_keyword<'a>() -> Expected, fn(Stream<'a>) -> ParseResult<&'a edn::Keyword, Stream<'a>>>> { - parser(any_keyword_ as fn(Stream<'a>) -> ParseResult<&'a edn::Keyword, Stream<'a>>).expected("any_keyword") -} - -pub fn namespaced_keyword<'a>() -> Expected, fn(Stream<'a>) -> ParseResult<&'a edn::Keyword, Stream<'a>>>> { - parser(namespaced_keyword_ as fn(Stream<'a>) -> ParseResult<&'a edn::Keyword, Stream<'a>>).expected("namespaced_keyword") -} - -pub fn forward_any_keyword_<'a>(input: Stream<'a>) -> ParseResult<&'a edn::Keyword, Stream<'a>> { - satisfy_map(|v: &'a edn::ValueAndSpan| v.inner.as_keyword().and_then(|k| if k.is_forward() { Some(k) } else { None })) - .parse_lazy(input) - .into() -} - -pub fn forward_any_keyword<'a>() -> Expected, fn(Stream<'a>) -> ParseResult<&'a edn::Keyword, Stream<'a>>>> { - parser(forward_any_keyword_ as fn(Stream<'a>) -> ParseResult<&'a edn::Keyword, Stream<'a>>).expected("forward_any_keyword") -} - -pub fn forward_namespaced_keyword_<'a>(input: Stream<'a>) -> ParseResult<&'a edn::Keyword, Stream<'a>> { - satisfy_map(|v: &'a edn::ValueAndSpan| v.inner.as_namespaced_keyword().and_then(|k| if k.is_forward() { Some(k) } else { None })) - .parse_lazy(input) - .into() -} - -pub fn forward_namespaced_keyword<'a>() -> Expected, fn(Stream<'a>) -> ParseResult<&'a edn::Keyword, Stream<'a>>>> { - parser(forward_namespaced_keyword_ as fn(Stream<'a>) -> ParseResult<&'a edn::Keyword, Stream<'a>>).expected("forward_namespaced_keyword") -} - -pub fn backward_namespaced_keyword_<'a>(input: Stream<'a>) -> ParseResult<&'a edn::Keyword, Stream<'a>> { - satisfy_map(|v: &'a edn::ValueAndSpan| v.inner.as_namespaced_keyword().and_then(|k| if k.is_backward() { Some(k) } else { None })) - .parse_lazy(input) - .into() -} - -pub fn backward_namespaced_keyword<'a>() -> Expected, fn(Stream<'a>) -> ParseResult<&'a edn::Keyword, Stream<'a>>>> { - parser(backward_namespaced_keyword_ as fn(Stream<'a>) -> ParseResult<&'a edn::Keyword, Stream<'a>>).expected("backward_namespaced_keyword") -} - -/// Generate a `satisfy` expression that matches a `PlainSymbol` value with the given name. -/// -/// We do this rather than using `combine::token` so that we don't need to allocate a new `String` -/// inside a `PlainSymbol` inside a `SpannedValue` inside a `ValueAndSpan` just to match input. -#[macro_export] -macro_rules! def_matches_plain_symbol { - ( $parser: ident, $name: ident, $input: expr ) => { - def_parser!($parser, $name, &'a edn::ValueAndSpan, { - satisfy(|v: &'a edn::ValueAndSpan| { - match v.inner { - edn::SpannedValue::PlainSymbol(ref s) => s.name() == $input, - _ => false, - } - }) - }); - } -} - -/// Generate a `satisfy` expression that matches a `Keyword` value with the given name. -/// -/// We do this rather than using `combine::token` to save allocations. -#[macro_export] -macro_rules! def_matches_keyword { - ( $parser: ident, $name: ident, $input: expr ) => { - def_parser!($parser, $name, &'a edn::ValueAndSpan, { - satisfy(|v: &'a edn::ValueAndSpan| { - match v.inner { - edn::SpannedValue::Keyword(ref s) if !s.is_namespaced() => s.name() == $input, - _ => false, - } - }) - }); - } -} - -/// Generate a `satisfy` expression that matches a `Keyword` value with the given -/// namespace and name. -/// -/// We do this rather than using `combine::token` to save allocations. -#[macro_export] -macro_rules! def_matches_namespaced_keyword { - ( $parser: ident, $name: ident, $input_namespace: expr, $input_name: expr ) => { - def_parser!($parser, $name, &'a edn::ValueAndSpan, { - satisfy(|v: &'a edn::ValueAndSpan| { - match v.inner { - edn::SpannedValue::Keyword(ref s) if s.is_namespaced() => { - let (ns, n) = s.components(); - ns == $input_namespace && n == $input_name - }, - _ => false, - } - }) - }); - } -} - -use combine::primitives::{ - Error, - Info, -}; -use combine::primitives::FastResult::*; - -/// Compare to `tuple_parser!` in `combine`. -/// -/// This uses edge cases in Rust's hygienic macro system to represent arbitrary values. That is, -/// `$value: ident` represents both a type in the tuple parameterizing `KeywordMapParser` (since -/// `(A, B, C)` is a valid type declaration) and also a variable value extracted from the underlying -/// instance value. `$tmp: ident` represents an optional value to return. -/// -/// This unrolls the cases. Each loop iteration reads a token. It then unrolls the known cases, -/// checking if any case matches the keyword string. If yes, we parse further. If no, we move on -/// to the next case. If no case matches, we fail. -macro_rules! keyword_map_parser { - ($(($keyword:ident, $value:ident, $tmp:ident)),+) => { - impl <'a, $($value:),+> Parser for KeywordMapParser<($((&'static str, $value)),+)> - where $($value: Parser>),+ - { - type Input = Stream<'a>; - type Output = ($(Option<$value::Output>),+); - - #[allow(non_snake_case)] - fn parse_lazy(&mut self, - mut input: Stream<'a>) - -> ConsumedResult<($(Option<$value::Output>),+), Stream<'a>> { - let ($((ref $keyword, ref mut $value)),+) = (*self).0; - let mut consumed = false; - - $( - let mut $tmp = None; - )+ - - loop { - match input.uncons() { - Ok(value) => { - $( - if let Some(ref keyword) = value.inner.as_plain_keyword() { - if &keyword.name() == $keyword { - if $tmp.is_some() { - // Repeated match -- bail out! Providing good error - // messages is hard; this will do for now. - return ConsumedErr(ParseError::new(input.position(), Error::Unexpected(Info::Token(value)))); - } - - consumed = true; - - $tmp = match $value.parse_lazy(input.clone()) { - ConsumedOk((x, new_input)) => { - input = new_input; - Some(x) - } - EmptyErr(mut err) => { - if let Ok(t) = input.uncons() { - err.add_error(Error::Unexpected(Info::Token(t))); - } - if consumed { - return ConsumedErr(err) - } else { - return EmptyErr(err) - } - } - ConsumedErr(err) => return ConsumedErr(err), - EmptyOk((x, new_input)) => { - input = new_input; - Some(x) - } - }; - - continue - } - } - )+ - - // No keyword matched! Bail out. - return ConsumedErr(ParseError::new(input.position(), Error::Unexpected(Info::Token(value)))); - }, - Err(err) => { - if consumed { - return ConsumedOk((($($tmp),+), input)) - } else { - if err == Error::end_of_input() { - return EmptyOk((($($tmp),+), input)); - } - return EmptyErr(ParseError::new(input.position(), err)) - } - }, - } - } - } - } - } -} - -keyword_map_parser!((Ak, Av, At), (Bk, Bv, Bt)); -keyword_map_parser!((Ak, Av, At), (Bk, Bv, Bt), (Ck, Cv, Ct)); -keyword_map_parser!((Ak, Av, At), (Bk, Bv, Bt), (Ck, Cv, Ct), (Dk, Dv, Dt)); -keyword_map_parser!((Ak, Av, At), (Bk, Bv, Bt), (Ck, Cv, Ct), (Dk, Dv, Dt), (Ek, Ev, Et)); -keyword_map_parser!((Ak, Av, At), (Bk, Bv, Bt), (Ck, Cv, Ct), (Dk, Dv, Dt), (Ek, Ev, Et), (Fk, Fv, Ft)); -keyword_map_parser!((Ak, Av, At), (Bk, Bv, Bt), (Ck, Cv, Ct), (Dk, Dv, Dt), (Ek, Ev, Et), (Fk, Fv, Ft), (Gk, Gv, Gt)); - -#[cfg(test)] -mod tests { - use combine::{ - eof, - many, - satisfy, - }; - - use super::*; - - use macros::{ - ResultParser, - }; - - /// A little test parser. - pub struct Test<'a>(std::marker::PhantomData<&'a ()>); - - def_matches_namespaced_keyword!(Test, add, "db", "add"); - - def_parser!(Test, entid, i64, { - integer() - .map(|x| x) - .or(namespaced_keyword().map(|_| -1)) - }); - - #[test] - #[should_panic(expected = r#"keyword map has repeated key: "x""#)] - fn test_keyword_map_of() { - keyword_map_of!(("x", Test::entid()), - ("x", Test::entid())); - } - - #[test] - fn test_iter() { - // A vector and a map iterated as a keyword map produce the same elements. - let input = edn::parse::value("[:y 3 4 :x 1 2]").expect("to be able to parse input as EDN"); - assert_eq!(input.child_iter().cloned().map(|x| x.without_spans()).into_iter().collect::>(), - edn::parse::value("[:y 3 4 :x 1 2]").expect("to be able to parse input as EDN").without_spans().into_vector().expect("an EDN vector")); - - let input = edn::parse::value("{:x [1 2] :y [3 4]}").expect("to be able to parse input as EDN"); - assert_eq!(input.keyword_map_iter().cloned().map(|x| x.without_spans()).into_iter().collect::>(), - edn::parse::value("[:y 3 4 :x 1 2]").expect("to be able to parse input as EDN").without_spans().into_vector().expect("an EDN vector")); - - // Parsing a keyword map in map and vector form produces the same elements. The order (:y - // before :x) is a foible of our EDN implementation and could be easily changed. - assert_edn_parses_to!(|| keyword_map().or(vector()).map(|x| x.0.map(|x| x.clone().without_spans()).into_iter().collect::>()), - "{:x [1] :y [2]}", - edn::parse::value("[:y 2 :x 1]").expect("to be able to parse input as EDN").without_spans().into_vector().expect("an EDN vector")); - - assert_edn_parses_to!(|| keyword_map().or(vector()).map(|x| x.0.map(|x| x.clone().without_spans()).into_iter().collect::>()), - "[:y 2 :x 1]", - edn::parse::value("[:y 2 :x 1]").expect("to be able to parse input as EDN").without_spans().into_vector().expect("an EDN vector")); - } - - #[test] - fn test_keyword_map() { - assert_edn_parses_to!(|| vector().of_exactly(keyword_map_of!(("x", Test::entid()), ("y", Test::entid()))), - "[:y 2 :x 1]", - (Some(1), Some(2))); - - assert_edn_parses_to!(|| vector().of_exactly(keyword_map_of!(("x", Test::entid()), ("y", Test::entid()))), - "[:x 1 :y 2]", - (Some(1), Some(2))); - - assert_edn_parses_to!(|| vector().of_exactly(keyword_map_of!(("x", Test::entid()), ("y", Test::entid()))), - "[:x 1]", - (Some(1), None)); - - assert_edn_parses_to!(|| vector().of_exactly(keyword_map_of!(("x", vector().of_exactly(many::, _>(Test::entid()))), - ("y", vector().of_exactly(many::, _>(Test::entid()))))), - "[:x [] :y [1 2]]", - (Some(vec![]), Some(vec![1, 2]))); - - assert_edn_parses_to!(|| vector().of_exactly(keyword_map_of!(("x", vector().of_exactly(many::, _>(Test::entid()))), - ("y", vector().of_exactly(many::, _>(Test::entid()))))), - "[]", - (None, None)); - } - - #[test] - fn test_keyword_map_failures() { - assert_parse_failure_contains!(|| vector().of_exactly(keyword_map_of!(("x", Test::entid()), ("y", Test::entid()))), - "[:x 1 :x 2]", - r#"errors: [Unexpected(Token(ValueAndSpan { inner: Keyword(Keyword(NamespaceableName { namespace: None, name: "x" }))"#); - } - - - // assert_edn_parses_to!(|| keyword_map().or(vector()).map(|x| x.0.map(|x| x.clone().without_spans()).into_iter().collect::>()), "{:x [1] :y [2]}", vec![]); - - // assert_edn_parses_to!(|| keyword_map().or(vector()).of_exactly((Test::entid(), Test::entid())), "{:x [1] :y [2]}", (-1, 1)); - - // assert_edn_parses_to!(|| kw_map().of_exactly((Test::entid(), Test::entid())), "[:a 0 :b 0 1]", (1, 1)); - - // assert_edn_parses_to!(|| keyword_map_of(&[(":kw1", Test::entid()), - // (":kw2", (Test::entid(), Test::entid())),]), - // "{:kw1 0 :kw2 1 :x/y}", ((Some(0), Some((0, 1))))); - - - - - // let input = edn::parse::value("[:x/y]").expect("to be able to parse input as EDN"); - // let par = vector().of_exactly(Test::entid()); - // let stream: Stream = (&input).atom_stream(); - // let result = par.skip(eof()).parse(stream).map(|x| x.0); - // assert_eq!(result, Ok(1)); - // } - - // #[test] - // fn test_keyword_map() { - // assert_keyword_map_eq!( - // "[:foo 1 2 3 :bar 4]", - // Some("{:foo [1 2 3] :bar [4]}")); - - // // Trailing keywords aren't allowed. - // assert_keyword_map_eq!( - // "[:foo]", - // None); - // assert_keyword_map_eq!( - // "[:foo 2 :bar]", - // None); - - // // Duplicate keywords aren't allowed. - // assert_keyword_map_eq!( - // "[:foo 2 :foo 1]", - // None); - - // // Starting with anything but a keyword isn't allowed. - // assert_keyword_map_eq!( - // "[2 :foo 1]", - // None); - - // // Consecutive keywords aren't allowed. - // assert_keyword_map_eq!( - // "[:foo :bar 1]", - // None); - - // // Empty lists return an empty map. - // assert_keyword_map_eq!( - // "[]", - // Some("{}")); - // } -} diff --git a/src/conn.rs b/src/conn.rs index c4510b52..9e5ca8d6 100644 --- a/src/conn.rs +++ b/src/conn.rs @@ -836,7 +836,6 @@ mod tests { use super::*; extern crate time; - extern crate mentat_parser_utils; use std::time::{ Instant, diff --git a/src/store.rs b/src/store.rs index b11deddf..21bad12c 100644 --- a/src/store.rs +++ b/src/store.rs @@ -245,7 +245,6 @@ mod tests { use super::*; extern crate time; - extern crate mentat_parser_utils; use std::collections::{ BTreeSet, diff --git a/tools/cli/Cargo.toml b/tools/cli/Cargo.toml index dc377e13..0c4ec2f7 100644 --- a/tools/cli/Cargo.toml +++ b/tools/cli/Cargo.toml @@ -39,9 +39,6 @@ features = ["limits"] path = "../.." default-features = false -[dependencies.mentat_parser_utils] -path = "../../parser-utils" - [dependencies.edn] path = "../../edn"