//! Break a string into a set of directory-object Items.

//!

//! This module defines Item, which represents a basic entry in a

//! directory document, and NetDocReader, which is used to break a

//! string into Items.

use crate::parse::keyword::Keyword;

use crate::types::misc::FromBytes;

use crate::util::PeekableIterator;

use crate::{Error, NetdocErrorKind as EK, Pos, Result};

use base64ct::{Base64, Encoding};

use itertools::Itertools;

use std::cell::{Ref, RefCell};

use std::iter::Peekable;

use std::str::FromStr;

use tor_error::internal;

/// Useful constants for netdoc object syntax

pub(crate) mod object {

    /// indicates the start of an object

    pub(crate) const BEGIN_STR: &str = "-----BEGIN ";

    /// indicates the end of an object

    pub(crate) const END_STR: &str = "-----END ";

    /// indicates the end of a begin or end tag.

    pub(crate) const TAG_END: &str = "-----";

    /// Maximum PEM base64 line length (not enforced during parsing)

    pub(crate) const BASE64_PEM_MAX_LINE: usize = 64;

}

/// Return true iff a given character is "space" according to the rules

/// of dir-spec.txt

/// Check that all the characters in `s` are valid base64.

///

/// This is not a perfect check for base64ness -- it is mainly meant

/// to help us recover after unterminated base64.

            _ => {

            }

        };

    }

/// A tagged object that is part of a directory Item.

///

/// This represents a single blob within a pair of "-----BEGIN

/// FOO-----" and "-----END FOO-----".  The data is not guaranteed to

/// be actual base64 when this object is created: doing so would

/// require either that we parse the base64 twice, or that we allocate

/// a buffer to hold the data before it's needed.

#[derive(Clone, Copy, Debug)]

pub(crate) struct Object<'a> {

    /// Reference to the "tag" string (the 'foo') in the BEGIN line.

    tag: &'a str,

    /// Reference to the allegedly base64-encoded data.  This may or

    /// may not actually be base64 at this point.

    data: &'a str,

    /// Reference to the END line for this object.  This doesn't

    /// need to be parsed, but it's used to find where this object

    /// ends.

    endline: &'a str,

}

/// A single part of a directory object.

///

/// Each Item -- called an "entry" in dir-spec.txt -- has a keyword, a

/// (possibly empty) set of arguments, and an optional object.

///

/// This is a zero-copy implementation that points to slices within a

/// containing string.

#[derive(Clone, Debug)]

pub(crate) struct Item<'a, K: Keyword> {

    /// The keyword that determines the type of this item.

    kwd: K,

    /// A reference to the actual string that defines the keyword for

    /// this item.

    kwd_str: &'a str,

    /// Reference to the arguments that appear in the same line after the

    /// keyword.  Does not include the terminating newline or the

    /// space that separates the keyword for its arguments.

    args: &'a str,

    /// The arguments, split by whitespace.  This vector is constructed

    /// as needed, using interior mutability.

    split_args: RefCell<Option<Vec<&'a str>>>,

    /// If present, a base-64-encoded object that appeared at the end

    /// of this item.

    object: Option<Object<'a>>,

}

/// A cursor into a string that returns Items one by one.

///

/// (This type isn't used directly, but is returned wrapped in a Peekable.)

#[derive(Debug)]

struct NetDocReaderBase<'a, K: Keyword> {

    /// The string we're parsing.

    s: &'a str,

    /// Our position within the string.

    off: usize,

    /// Tells Rust it's okay that we are parameterizing on K.

    _k: std::marker::PhantomData<K>,

}

impl<'a, K: Keyword> NetDocReaderBase<'a, K> {

    /// Create a new NetDocReader to split a string into tokens.

        Ok(NetDocReaderBase {

            off: 0,

        })

    /// Return the current Pos within the string.

    /// Skip forward by n bytes.

    ///

    /// (Note that standard caveats with byte-oriented processing of

    /// UTF-8 strings apply.)

    /// Return the remaining number of bytes in this reader.

    /// Return true if the next characters in this reader are `s`

    /// Try to extract a NL-terminated line from this reader.  Always

    /// remove data if the reader is nonempty.

            // TODO: we should probably detect \r and do something about it.

            // Just ignoring it isn't the right answer, though.

        } else {

        }

    /// Try to extract a line that begins with a keyword from this reader.

    ///

    /// Returns a (kwd, args) tuple on success.

        } else {

        };

            // This case seems like it can't happen: split always returns

            // something, apparently.

        };

        // TODO(nickm): dir-spec does not yet allow unicode in the arguments, but we're

        // assuming that proposal 285 is accepted.

            // take a zero-length slice, so it will be within the string.

        };

    /// Try to extract an Object beginning wrapped within BEGIN/END tags.

    ///

    /// Returns Ok(Some(Object(...))) on success if an object is

    /// found, Ok(None) if no object is found, and Err only if a

    /// corrupt object is found.

        use object::*;

            // Exit if this line isn't plausible base64.  Otherwise,

            // an unterminated base64 block could potentially

            // "consume" all the rest of the string, which would stop

            // us from recovering.

        };

    /// Read the next Item from this NetDocReaderBase.

    ///

    /// If successful, returns Ok(Some(Item)), or Ok(None) if exhausted.

    /// Returns Err on failure.

    ///

    /// Always consumes at least one line if possible; always ends on a

    /// line boundary if one exists.

}

/// Return true iff 's' is a valid keyword or annotation.

///

/// (Only allow annotations if `anno_ok` is true.`

    /// Helper: return true if this character can appear in keywords.

/// Return true iff 's' is a valid keywords string for a BEGIN/END tag.

/// When used as an Iterator, returns a sequence of `Result<Item>`.

impl<'a, K: Keyword> Iterator for NetDocReaderBase<'a, K> {

    type Item = Result<Item<'a, K>>;

}

/// Helper: as base64::decode(), but allows newlines in the middle of the

/// encoded object.

    // base64 module hates whitespace.

impl<'a, K: Keyword> Item<'a, K> {

    /// Return the parsed keyword part of this item.

    /// Return the keyword part of this item, as a string.

    /// Return true if the keyword for this item is in 'ks'.

    /// Return the arguments of this item, as a single string.

    /// Return the arguments of this item as a vector.

        // We're using an interior mutability pattern here to lazily

        // construct the vector.

    /// Return an iterator over the arguments of this item.

    /// Return the nth argument of this item, if there is one.

    /// Return the nth argument of this item, or an error if it isn't there.

    /// Try to parse the nth argument (if it exists) into some type

    /// that supports FromStr.

    ///

    /// Returns Ok(None) if the argument doesn't exist.

    {

                }

            },

        }

    /// Try to parse the nth argument (if it exists) into some type

    /// that supports FromStr.

    ///

    /// Return an error if the argument doesn't exist.

    {

        }

    /// Return the number of arguments for this Item

    /// Return true iff this Item has an associated object.

    /// Return the tag of this item's associated object, if it has one.

    /// Try to decode the base64 contents of this Item's associated object.

    ///

    /// On success, return the object's tag and decoded contents.

            }

        }

    /// Try to decode the base64 contents of this Item's associated object,

    /// and make sure that its tag matches 'want_tag'.

                } else {

                }

            }

        }

    /// Try to decode the base64 contents of this item's associated object

    /// as a given type that implements FromBytes.

        // Unwrap may be safe because above `.obj()` should return an Error if

        // wanted tag was not present

        #[allow(clippy::unwrap_used)]

    /// Return the position of this item.

    ///

    /// This position won't be useful unless it is later contextualized

    /// with the containing string.

    /// Return the position of this Item in a string.

    ///

    /// Returns None if this item doesn't actually belong to the string.

    /// Return the position of the n'th argument of this item.

    ///

    /// If this item does not have a n'th argument, return the

    /// position of the end of the final argument.

        } else {

        }

    /// Return the position at the end of the last argument.  (This will

    /// point to a newline.)

    /// Return the position of the end of this object. (This will point to a

    /// newline.)

        }

    /// If this item occurs within s, return the byte offset

    /// immediately after the end of this item.

    /// Return the text of this item, if it originated within `str`,

    /// from the start of its keyword up to and including its final newline.

    #[allow(dead_code)] // unused when hsdesc not enabled.

}

/// Represents an Item that might not be present, whose arguments we

/// want to inspect.  If the Item is there, this acts like a proxy to the

/// item; otherwise, it treats the item as having no arguments.

pub(crate) struct MaybeItem<'a, 'b, K: Keyword>(Option<&'a Item<'b, K>>);

// All methods here are as for Item.

impl<'a, 'b, K: Keyword> MaybeItem<'a, 'b, K> {

    /// Return the position of this item, if it has one.

        }

    /// Construct a MaybeItem from an Option reference to an item.

    /// If this item is present, parse its argument at position `idx`.

    /// Treat the absence or malformedness of the argument as an error,

    /// but treat the absence of this item as acceptable.

    #[cfg(any(test, feature = "routerdesc"))]

    {

            },

        }

    /// If this item is present, return its arguments as a single string.

    /// If this item is present, parse all of its arguments as a

    /// single string.

    {

                }

            },

        }

}

/// Extension trait for `Result<Item>` -- makes it convenient to implement

/// PauseAt predicates

pub(crate) trait ItemResult<K: Keyword> {

    /// Return true if this is an ok result with an annotation.

    fn is_ok_with_annotation(&self) -> bool;

    /// Return true if this is an ok result with a non-annotation.

    fn is_ok_with_non_annotation(&self) -> bool;

    /// Return true if this is an ok result with the keyword 'k'

    /// Return true if this is an ok result with a keyword in the slice 'ks'

    fn is_ok_with_kwd_in(&self, ks: &[K]) -> bool;

    /// Return true if this is an ok result with a keyword not in the slice 'ks'

    fn is_ok_with_kwd_not_in(&self, ks: &[K]) -> bool;

    /// Return true if this is an empty-line error.

    fn is_empty_line(&self) -> bool;

}

impl<'a, K: Keyword> ItemResult<K> for Result<Item<'a, K>> {

        }

        }

        }

        }

        )

}

/// A peekable cursor into a string that returns Items one by one.

///

/// This is an [`Iterator`], yielding [`Item`]s.

#[derive(Debug)]

pub(crate) struct NetDocReader<'a, K: Keyword> {

    // TODO: I wish there were some way around having this string

    // reference, since we already need one inside NetDocReaderBase.

    /// The underlying string being parsed.

    s: &'a str,

    /// A stream of tokens being parsed by this NetDocReader.

    tokens: Peekable<NetDocReaderBase<'a, K>>,

}

impl<'a, K: Keyword> NetDocReader<'a, K> {

    /// Construct a new NetDocReader to read tokens from `s`.

        Ok(NetDocReader {

        })

    /// Return a reference to the string used for this NetDocReader.

    /// Return a wrapper around the peekable iterator in this

    /// NetDocReader that reads tokens until it reaches an element where

    /// 'f' is true.

    {

    /// Return true if there are no more items in this NetDocReader.

    // The implementation sadly needs to mutate the inner state, even if it's not *semantically*

    // mutated..  We don't want inner mutability just to placate clippy for an internal API.

    #[allow(clippy::wrong_self_convention)]

    #[allow(dead_code)] // TODO perhaps we should remove this ?

    /// Give an error if there are remaining tokens in this NetDocReader.

        }

    /// Give an error if there are remaining tokens in this NetDocReader.

    ///

    /// Like [`should_be_exhausted`](Self::should_be_exhausted),

    /// but permit empty lines at the end of the document.

    #[cfg(feature = "routerdesc")]

        use crate::err::NetdocErrorKind as K;

            }

        }

    /// Return the position from which the underlying reader is about to take

    /// the next token.  Use to make sure that the reader is progressing.

        }

}

impl<'a, K: Keyword> Iterator for NetDocReader<'a, K> {

    type Item = Result<Item<'a, K>>;

}

impl<'a, K: Keyword> PeekableIterator for NetDocReader<'a, K> {

}

impl<'a, K: Keyword> itertools::PeekingNext for NetDocReader<'a, K> {

    {

}

/// Check additional UTF-8 rules that the netdoc metaformat imposes on

/// our documents.

//

// NOTE: We might decide in the future to loosen our rules here

// for parsers that handle concatenated documents:

// we might want to reject only those documents that contain NULs.

// But with luck that will never be necessary.

    // No BOM, or mangled BOM, is allowed.

    // No NUL bytes are allowed.

#[cfg(test)]

mod test {

    // @@ begin test lint list maintained by maint/add_warning @@

    #![allow(clippy::bool_assert_comparison)]

    #![allow(clippy::clone_on_copy)]

    #![allow(clippy::dbg_macro)]

    #![allow(clippy::mixed_attributes_style)]

    #![allow(clippy::print_stderr)]

    #![allow(clippy::print_stdout)]

    #![allow(clippy::single_char_pattern)]

    #![allow(clippy::unwrap_used)]

    #![allow(clippy::unchecked_time_subtraction)]

    #![allow(clippy::useless_vec)]

    #![allow(clippy::needless_pass_by_value)]

    //! <!-- @@ end test lint list maintained by maint/add_warning @@ -->

    #![allow(clippy::cognitive_complexity)]

    use super::*;

    use crate::parse::macros::test::Fruit;

    use crate::{NetdocErrorKind as EK, Pos, Result};

    #[test]

    fn read_simple() {

        use Fruit::*;

        let s = "\

@tasty very much so

opt apple 77

banana 60

cherry 6

-----BEGIN CHERRY SYNOPSIS-----

8J+NkvCfjZLwn42S8J+NkvCfjZLwn42S

-----END CHERRY SYNOPSIS-----

plum hello there

";

        let mut r: NetDocReader<'_, Fruit> = NetDocReader::new(s).unwrap();

        assert_eq!(r.str(), s);

        assert!(r.should_be_exhausted().is_err()); // it's not exhausted.

        let toks: Result<Vec<_>> = r.by_ref().collect();

        assert!(r.should_be_exhausted().is_ok());

        let toks = toks.unwrap();

        assert_eq!(toks.len(), 5);

        assert_eq!(toks[0].kwd(), ANN_TASTY);

        assert_eq!(toks[0].n_args(), 3);

        assert_eq!(toks[0].args_as_str(), "very much so");

        assert_eq!(toks[0].arg(1), Some("much"));

        {

            let a: Vec<_> = toks[0].args().collect();

            assert_eq!(a, vec!["very", "much", "so"]);

        }

        assert!(toks[0].parse_arg::<usize>(0).is_err());

        assert!(toks[0].parse_arg::<usize>(10).is_err());

        assert!(!toks[0].has_obj());

        assert_eq!(toks[0].obj_tag(), None);

        assert_eq!(toks[2].pos().within(s), Pos::from_line(3, 1));

        assert_eq!(toks[2].arg_pos(0).within(s), Pos::from_line(3, 8));

        assert_eq!(toks[2].last_arg_end_pos().within(s), Pos::from_line(3, 10));

        assert_eq!(toks[2].end_pos().within(s), Pos::from_line(3, 10));

        assert_eq!(toks[3].kwd(), STONEFRUIT);

        assert_eq!(toks[3].kwd_str(), "cherry"); // not cherry/plum!

        assert_eq!(toks[3].n_args(), 1);

        assert_eq!(toks[3].required_arg(0), Ok("6"));

        assert_eq!(toks[3].parse_arg::<usize>(0), Ok(6));

        assert_eq!(toks[3].parse_optional_arg::<usize>(0), Ok(Some(6)));

        assert_eq!(toks[3].parse_optional_arg::<usize>(3), Ok(None));

        assert!(toks[3].has_obj());

        assert_eq!(toks[3].obj_tag(), Some("CHERRY SYNOPSIS"));

        assert_eq!(

            &toks[3].obj("CHERRY SYNOPSIS").unwrap()[..],

            "🍒🍒🍒🍒🍒🍒".as_bytes()

        );

        assert!(toks[3].obj("PLUOT SYNOPSIS").is_err());

        // this "end-pos" value is questionable!

        assert_eq!(toks[3].end_pos().within(s), Pos::from_line(7, 30));

    }

    #[test]

    fn test_badtoks() {

        use Fruit::*;

        let s = "\

-foobar 9090

apple 3.14159

$hello

unrecognized 127.0.0.1 foo

plum

-----BEGIN WHATEVER-----

8J+NkvCfjZLwn42S8J+NkvCfjZLwn42S

-----END SOMETHING ELSE-----

orange

orange

-----BEGIN WHATEVER-----

not! base64!

-----END WHATEVER-----

guava paste

opt @annotation

orange

-----BEGIN LOBSTER

8J+NkvCfjZLwn42S8J+NkvCfjZLwn42S

-----END SOMETHING ELSE-----

orange

-----BEGIN !!!!!!-----

8J+NkvCfjZLwn42S8J+NkvCfjZLwn42S

-----END !!!!!!-----

cherry

-----BEGIN CHERRY SYNOPSIS-----

8J+NkvCfjZLwn42S8J+NkvCfjZLwn42S

-----END CHERRY SYNOPSIS

truncated line";

        let r: NetDocReader<'_, Fruit> = NetDocReader::new(s).unwrap();

        let toks: Vec<_> = r.collect();

        assert!(toks[0].is_err());

        assert_eq!(

            toks[0].as_ref().err().unwrap(),

            &EK::BadKeyword.at_pos(Pos::from_line(1, 1))

        );

        assert!(toks[1].is_ok());

        assert!(toks[1].is_ok_with_non_annotation());

        assert!(!toks[1].is_ok_with_annotation());

        assert!(toks[1].is_ok_with_kwd_in(&[APPLE, ORANGE]));

        assert!(toks[1].is_ok_with_kwd_not_in(&[ORANGE, UNRECOGNIZED]));

        let t = toks[1].as_ref().unwrap();

        assert_eq!(t.kwd(), APPLE);

        assert_eq!(t.arg(0), Some("3.14159"));

        assert!(toks[2].is_err());

        assert!(!toks[2].is_ok_with_non_annotation());

        assert!(!toks[2].is_ok_with_annotation());

        assert!(!toks[2].is_ok_with_kwd_in(&[APPLE, ORANGE]));

        assert!(!toks[2].is_ok_with_kwd_not_in(&[ORANGE, UNRECOGNIZED]));

        assert_eq!(

            toks[2].as_ref().err().unwrap(),

            &EK::BadKeyword.at_pos(Pos::from_line(3, 1))

        );

        assert!(toks[3].is_ok());

        let t = toks[3].as_ref().unwrap();

        assert_eq!(t.kwd(), UNRECOGNIZED);

        assert_eq!(t.arg(1), Some("foo"));

        assert!(toks[4].is_err());

        assert_eq!(

            toks[4].as_ref().err().unwrap(),

            &EK::BadObjectMismatchedTag.at_pos(Pos::from_line(8, 1))

        );

        assert!(toks[5].is_ok());

        let t = toks[5].as_ref().unwrap();

        assert_eq!(t.kwd(), ORANGE);

        assert_eq!(t.args_as_str(), "");

        // This blob counts as two errors: a bad base64 blob, and

        // then an end line.

        assert!(toks[6].is_err());

        assert_eq!(

            toks[6].as_ref().err().unwrap(),

            &EK::BadObjectBase64.at_pos(Pos::from_line(12, 1))

        );

        assert!(toks[7].is_err());

        assert_eq!(

            toks[7].as_ref().err().unwrap(),

            &EK::BadKeyword.at_pos(Pos::from_line(13, 1))

        );

        assert!(toks[8].is_ok());

        let t = toks[8].as_ref().unwrap();

        assert_eq!(t.kwd(), GUAVA);

        // this is an error because you can't use opt with annotations.

        assert!(toks[9].is_err());

        assert_eq!(

            toks[9].as_ref().err().unwrap(),

            &EK::BadKeyword.at_pos(Pos::from_line(15, 1))

        );

        // this looks like a few errors.

        assert!(toks[10].is_err());

        assert_eq!(

            toks[10].as_ref().err().unwrap(),

            &EK::BadObjectBeginTag.at_pos(Pos::from_line(17, 1))

        );

        assert!(toks[11].is_err());

        assert_eq!(

            toks[11].as_ref().err().unwrap(),

            &EK::BadKeyword.at_pos(Pos::from_line(18, 1))

        );

        assert!(toks[12].is_err());

        assert_eq!(

            toks[12].as_ref().err().unwrap(),

            &EK::BadKeyword.at_pos(Pos::from_line(19, 1))

        );

        // so does this.

        assert!(toks[13].is_err());

        assert_eq!(

            toks[13].as_ref().err().unwrap(),

            &EK::BadObjectBeginTag.at_pos(Pos::from_line(21, 1))

        );

        assert!(toks[14].is_err());

        assert_eq!(

            toks[14].as_ref().err().unwrap(),

            &EK::BadKeyword.at_pos(Pos::from_line(22, 1))

        );

        assert!(toks[15].is_err());

        assert_eq!(

            toks[15].as_ref().err().unwrap(),

            &EK::BadKeyword.at_pos(Pos::from_line(23, 1))

        );

        // not this.

        assert!(toks[16].is_err());

        assert_eq!(

            toks[16].as_ref().err().unwrap(),

            &EK::BadObjectEndTag.at_pos(Pos::from_line(27, 1))

        );

        assert!(toks[17].is_err());

        assert_eq!(

            toks[17].as_ref().err().unwrap(),

            &EK::EmptyLine.at_pos(Pos::from_line(28, 1))

        );

        assert!(toks[18].is_err());

        assert_eq!(

            toks[18].as_ref().err().unwrap(),

            &EK::TruncatedLine.at_pos(Pos::from_line(29, 15))

        );

    }

    #[test]

    fn test_leading_space_forbidden() {

        // We need to make sure that items with a leading space aren't accepted:

        // the spec forbids it, and it can provide a vector for inflating the size

        // of downloaded hsdescs (see prop360).

        // Try a simple item with a space at the front.

        let s = "    guava space\n";

        let r: NetDocReader<'_, Fruit> = NetDocReader::new(s).unwrap();

        let toks: Vec<_> = r.collect();

        // No space allowed at the start of a line.

        assert_eq!(

            toks[0].as_ref().err().unwrap(),

            &EK::BadKeyword.at_pos(Pos::from_line(1, 1))

        );

        // Try an item with an object, inserting space at the start of each ine in turn.

        let s = "cherry

-----BEGIN WHATEVER-----

8J+NkvCfjZLwn42S8J+NkvCfjZLwn42S

-----END WHATEVER-----

";

        let orig_lines = s

            .split_terminator('\n')

            .map(str::to_string)

            .collect::<Vec<_>>();

        assert_eq!(orig_lines.len(), 4);

        let expected_kinds = [

            EK::BadKeyword,

            EK::BadKeyword,

            EK::BadObjectBase64,

            EK::BadObjectBase64,

        ];

        for pos in 0..orig_lines.len() {

            let mut lines = orig_lines.clone();

            lines[pos] = format!(" {}", lines[pos]);

            let joined = format!("{}\n", lines.join("\n"));

            let r: NetDocReader<'_, Fruit> = NetDocReader::new(&joined).unwrap();

            let toks: Result<Vec<_>> = r.collect();

            assert_eq!(toks.unwrap_err().netdoc_error_kind(), expected_kinds[pos]);

        }

    }

    #[test]

    fn test_validate_strings() {

        use validate_utf_8_rules as v;

        assert_eq!(v(""), Ok(""));

        assert_eq!(v("hello world"), Ok("hello world"));

        // We don't have to test a lot more valid cases, since this function is called before

        // parsing any string.

        for s in ["\u{feff}", "\u{feff}hello world", "\u{fffe}hello world"] {

            let e = v(s).unwrap_err();

            assert_eq!(e.netdoc_error_kind(), EK::BomMarkerFound);

            assert_eq!(e.pos().offset_within(s), Some(0));

        }

        for s in [

            "\0hello world",

            "\0",

            "\0\0\0",

            "hello\0world",

            "hello world\0",

        ] {

            let e = v(s).unwrap_err();

            assert_eq!(e.netdoc_error_kind(), EK::NulFound);

            let nul_pos = e.pos().offset_within(s).unwrap();

            assert_eq!(s.as_bytes()[nul_pos], 0);

        }

    }

    fn single_fruit(s: &str) -> Item<'_, Fruit> {

        NetDocReader::<Fruit>::new(s)

            .unwrap()

            .next()

            .unwrap()

            .unwrap()

    }

    #[test]

    fn end_of_item() {

        let s = "guava friends 123   \n";

        let item = single_fruit(s);

        assert_eq!(

            item.end_pos().within(s),

            Pos::from_byte(s.find('\n').unwrap()).within(s)

        );

        let s = "cherry

-----BEGIN WHATEVER-----

8J+NkvCfjZLwn42S8J+NkvCfjZLwn42S

-----END WHATEVER-----\n";

        let item = single_fruit(s);

        dbg!(&item);

        assert_eq!(

            item.end_pos().within(s),

            Pos::from_byte(s.rfind('\n').unwrap()).within(s)

        );

    }

}