1
//! Break a string into a set of directory-object Items.
2
//!
3
//! This module defines Item, which represents a basic entry in a
4
//! directory document, and NetDocReader, which is used to break a
5
//! string into Items.
6

            
7
use crate::parse::keyword::Keyword;
8
use crate::types::misc::FromBytes;
9
use crate::util::PeekableIterator;
10
use crate::{Error, NetdocErrorKind as EK, Pos, Result};
11
use base64ct::{Base64, Encoding};
12
use itertools::Itertools;
13
use std::cell::{Ref, RefCell};
14
use std::iter::Peekable;
15
use std::str::FromStr;
16
use tor_error::internal;
17

            
18
/// Useful constants for netdoc object syntax
19
pub(crate) mod object {
20
    /// indicates the start of an object
21
    pub(crate) const BEGIN_STR: &str = "-----BEGIN ";
22
    /// indicates the end of an object
23
    pub(crate) const END_STR: &str = "-----END ";
24
    /// indicates the end of a begin or end tag.
25
    pub(crate) const TAG_END: &str = "-----";
26
    /// Maximum PEM base64 line length (not enforced during parsing)
27
    pub(crate) const BASE64_PEM_MAX_LINE: usize = 64;
28
}
29

            
30
/// Return true iff a given character is "space" according to the rules
31
/// of dir-spec.txt
32
39500922
pub(crate) fn is_sp(c: char) -> bool {
33
39500922
    c == ' ' || c == '\t'
34
39500922
}
35
/// Check that all the characters in `s` are valid base64.
36
///
37
/// This is not a perfect check for base64ness -- it is mainly meant
38
/// to help us recover after unterminated base64.
39
249339
fn b64check(s: &str) -> Result<()> {
40
15589425
    for b in s.bytes() {
41
15589425
        match b {
42
23864
            b'=' => (),
43
6286417
            b'a'..=b'z' => (),
44
6423155
            b'A'..=b'Z' => (),
45
2388373
            b'0'..=b'9' => (),
46
467613
            b'/' | b'+' => (),
47
            _ => {
48
3
                return Err(EK::BadObjectBase64.at_pos(Pos::at(s)));
49
            }
50
        };
51
    }
52
249336
    Ok(())
53
249339
}
54

            
55
/// A tagged object that is part of a directory Item.
56
///
57
/// This represents a single blob within a pair of "-----BEGIN
58
/// FOO-----" and "-----END FOO-----".  The data is not guaranteed to
59
/// be actual base64 when this object is created: doing so would
60
/// require either that we parse the base64 twice, or that we allocate
61
/// a buffer to hold the data before it's needed.
62
#[derive(Clone, Copy, Debug)]
63
pub(crate) struct Object<'a> {
64
    /// Reference to the "tag" string (the 'foo') in the BEGIN line.
65
    tag: &'a str,
66
    /// Reference to the allegedly base64-encoded data.  This may or
67
    /// may not actually be base64 at this point.
68
    data: &'a str,
69
    /// Reference to the END line for this object.  This doesn't
70
    /// need to be parsed, but it's used to find where this object
71
    /// ends.
72
    endline: &'a str,
73
}
74

            
75
/// A single part of a directory object.
76
///
77
/// Each Item -- called an "entry" in dir-spec.txt -- has a keyword, a
78
/// (possibly empty) set of arguments, and an optional object.
79
///
80
/// This is a zero-copy implementation that points to slices within a
81
/// containing string.
82
#[derive(Clone, Debug)]
83
pub(crate) struct Item<'a, K: Keyword> {
84
    /// The keyword that determines the type of this item.
85
    kwd: K,
86
    /// A reference to the actual string that defines the keyword for
87
    /// this item.
88
    kwd_str: &'a str,
89
    /// Reference to the arguments that appear in the same line after the
90
    /// keyword.  Does not include the terminating newline or the
91
    /// space that separates the keyword for its arguments.
92
    args: &'a str,
93
    /// The arguments, split by whitespace.  This vector is constructed
94
    /// as needed, using interior mutability.
95
    split_args: RefCell<Option<Vec<&'a str>>>,
96
    /// If present, a base-64-encoded object that appeared at the end
97
    /// of this item.
98
    object: Option<Object<'a>>,
99
}
100

            
101
/// A cursor into a string that returns Items one by one.
102
///
103
/// (This type isn't used directly, but is returned wrapped in a Peekable.)
104
#[derive(Debug)]
105
struct NetDocReaderBase<'a, K: Keyword> {
106
    /// The string we're parsing.
107
    s: &'a str,
108
    /// Our position within the string.
109
    off: usize,
110
    /// Tells Rust it's okay that we are parameterizing on K.
111
    _k: std::marker::PhantomData<K>,
112
}
113

            
114
impl<'a, K: Keyword> NetDocReaderBase<'a, K> {
115
    /// Create a new NetDocReader to split a string into tokens.
116
5246
    fn new(s: &'a str) -> Result<Self> {
117
        Ok(NetDocReaderBase {
118
5246
            s: validate_utf_8_rules(s)?,
119
            off: 0,
120
5246
            _k: std::marker::PhantomData,
121
        })
122
5246
    }
123
    /// Return the current Pos within the string.
124
183
    fn pos(&self, pos: usize) -> Pos {
125
183
        Pos::from_offset(self.s, pos)
126
183
    }
127
    /// Skip forward by n bytes.
128
    ///
129
    /// (Note that standard caveats with byte-oriented processing of
130
    /// UTF-8 strings apply.)
131
395275
    fn advance(&mut self, n: usize) -> Result<()> {
132
395275
        if n > self.remaining() {
133
            return Err(
134
                Error::from(internal!("tried to advance past end of document"))
135
                    .at_pos(Pos::from_offset(self.s, self.off)),
136
            );
137
395275
        }
138
395275
        self.off += n;
139
395275
        Ok(())
140
395275
    }
141
    /// Return the remaining number of bytes in this reader.
142
502188
    fn remaining(&self) -> usize {
143
502188
        self.s.len() - self.off
144
502188
    }
145

            
146
    /// Return true if the next characters in this reader are `s`
147
    #[allow(clippy::string_slice)] // TODO
148
101612
    fn starts_with(&self, s: &str) -> bool {
149
101612
        self.s[self.off..].starts_with(s)
150
101612
    }
151
    /// Try to extract a NL-terminated line from this reader.  Always
152
    /// remove data if the reader is nonempty.
153
    #[allow(clippy::string_slice)] // TODO
154
395275
    fn line(&mut self) -> Result<&'a str> {
155
395275
        let remainder = &self.s[self.off..];
156
395275
        if let Some(nl_pos) = remainder.find('\n') {
157
395115
            self.advance(nl_pos + 1)?;
158
395115
            let line = &remainder[..nl_pos];
159

            
160
            // TODO: we should probably detect \r and do something about it.
161
            // Just ignoring it isn't the right answer, though.
162
395115
            Ok(line)
163
        } else {
164
160
            self.advance(remainder.len())?; // drain everything.
165
160
            Err(EK::TruncatedLine.at_pos(self.pos(self.s.len())))
166
        }
167
395275
    }
168

            
169
    /// Try to extract a line that begins with a keyword from this reader.
170
    ///
171
    /// Returns a (kwd, args) tuple on success.
172
    #[allow(clippy::string_slice)] // TODO
173
101791
    fn kwdline(&mut self) -> Result<(&'a str, &'a str)> {
174
101791
        let pos = self.off;
175
101791
        let line = self.line()?;
176
101631
        if line.is_empty() {
177
7
            return Err(EK::EmptyLine.at_pos(self.pos(pos)));
178
101624
        }
179
101624
        let (line, anno_ok) = if let Some(rem) = line.strip_prefix("opt ") {
180
3
            (rem.trim_start_matches(is_sp), false)
181
        } else {
182
101621
            (line, true)
183
        };
184
101624
        let parts = line.split_once(is_sp);
185
101624
        let (kwd, args) = match parts {
186
80275
            Some((left, right)) => (left, right.trim_start_matches(is_sp)),
187
21349
            None => (line, &line[line.len()..]),
188
        };
189
101624
        if !keyword_ok(kwd, anno_ok) {
190
12
            return Err(EK::BadKeyword.at_pos(self.pos(pos)));
191
101612
        }
192
101612
        Ok((kwd, args))
193
101791
    }
194

            
195
    /// Try to extract an Object beginning wrapped within BEGIN/END tags.
196
    ///
197
    /// Returns Ok(Some(Object(...))) on success if an object is
198
    /// found, Ok(None) if no object is found, and Err only if a
199
    /// corrupt object is found.
200
    #[allow(clippy::string_slice)] // TODO
201
101612
    fn object(&mut self) -> Result<Option<Object<'a>>> {
202
        use object::*;
203

            
204
101612
        let pos = self.off;
205
101612
        if !self.starts_with(BEGIN_STR) {
206
79537
            return Ok(None);
207
22075
        }
208
22075
        let line = self.line()?;
209
22075
        if !line.ends_with(TAG_END) {
210
1
            return Err(EK::BadObjectBeginTag.at_pos(self.pos(pos)));
211
22074
        }
212
22074
        let tag = &line[BEGIN_STR.len()..(line.len() - TAG_END.len())];
213
22074
        if !tag_keywords_ok(tag) {
214
1
            return Err(EK::BadObjectBeginTag.at_pos(self.pos(pos)));
215
22073
        }
216
22073
        let datapos = self.off;
217
22070
        let (endlinepos, endline) = loop {
218
271409
            let p = self.off;
219
271409
            let line = self.line()?;
220
271409
            if line.starts_with(END_STR) {
221
22070
                break (p, line);
222
249339
            }
223
            // Exit if this line isn't plausible base64.  Otherwise,
224
            // an unterminated base64 block could potentially
225
            // "consume" all the rest of the string, which would stop
226
            // us from recovering.
227
249339
            b64check(line).map_err(|e| e.within(self.s))?;
228
        };
229
22070
        let data = &self.s[datapos..endlinepos];
230
22070
        if !endline.ends_with(TAG_END) {
231
1
            return Err(EK::BadObjectEndTag.at_pos(self.pos(endlinepos)));
232
22069
        }
233
22069
        let endtag = &endline[END_STR.len()..(endline.len() - TAG_END.len())];
234
22069
        if endtag != tag {
235
1
            return Err(EK::BadObjectMismatchedTag.at_pos(self.pos(endlinepos)));
236
22068
        }
237
22068
        Ok(Some(Object { tag, data, endline }))
238
101612
    }
239

            
240
    /// Read the next Item from this NetDocReaderBase.
241
    ///
242
    /// If successful, returns Ok(Some(Item)), or Ok(None) if exhausted.
243
    /// Returns Err on failure.
244
    ///
245
    /// Always consumes at least one line if possible; always ends on a
246
    /// line boundary if one exists.
247
106913
    fn item(&mut self) -> Result<Option<Item<'a, K>>> {
248
106913
        if self.remaining() == 0 {
249
5122
            return Ok(None);
250
101791
        }
251
101791
        let (kwd_str, args) = self.kwdline()?;
252
101612
        let object = self.object()?;
253
101605
        let split_args = RefCell::new(None);
254
101605
        let kwd = K::from_str(kwd_str);
255
101605
        Ok(Some(Item {
256
101605
            kwd,
257
101605
            kwd_str,
258
101605
            args,
259
101605
            split_args,
260
101605
            object,
261
101605
        }))
262
106913
    }
263
}
264

            
265
/// Return true iff 's' is a valid keyword or annotation.
266
///
267
/// (Only allow annotations if `anno_ok` is true.`
268
#[allow(clippy::string_slice)] // TODO
269
205292
fn keyword_ok(mut s: &str, anno_ok: bool) -> bool {
270
    /// Helper: return true if this character can appear in keywords.
271
1697906
    fn kwd_char_ok(c: char) -> bool {
272
1697906
        matches!(c,'A'..='Z' | 'a'..='z' |'0'..='9' | '-')
273
1697906
    }
274

            
275
205292
    if s.is_empty() {
276
3
        return false;
277
205289
    }
278
205289
    if anno_ok && s.starts_with('@') {
279
15
        s = &s[1..];
280
205274
    }
281
205289
    if s.starts_with('-') {
282
4
        return false;
283
205285
    }
284
205285
    s.chars().all(kwd_char_ok)
285
205292
}
286

            
287
/// Return true iff 's' is a valid keywords string for a BEGIN/END tag.
288
56622
pub(crate) fn tag_keywords_ok(s: &str) -> bool {
289
103668
    s.split(' ').all(|w| keyword_ok(w, false))
290
56622
}
291

            
292
/// When used as an Iterator, returns a sequence of `Result<Item>`.
293
impl<'a, K: Keyword> Iterator for NetDocReaderBase<'a, K> {
294
    type Item = Result<Item<'a, K>>;
295
106913
    fn next(&mut self) -> Option<Self::Item> {
296
106913
        self.item().transpose()
297
106913
    }
298
}
299

            
300
/// Helper: as base64::decode(), but allows newlines in the middle of the
301
/// encoded object.
302
223761
pub(crate) fn base64_decode_multiline(s: &str) -> std::result::Result<Vec<u8>, base64ct::Error> {
303
    // base64 module hates whitespace.
304
223761
    let mut s = s.to_string();
305
78702502
    s.retain(|ch| ch != '\n');
306
223761
    let v = Base64::decode_vec(&s)?;
307
223760
    Ok(v)
308
223761
}
309

            
310
impl<'a, K: Keyword> Item<'a, K> {
311
    /// Return the parsed keyword part of this item.
312
170880
    pub(crate) fn kwd(&self) -> K {
313
170880
        self.kwd
314
170880
    }
315
    /// Return the keyword part of this item, as a string.
316
2678
    pub(crate) fn kwd_str(&self) -> &'a str {
317
2678
        self.kwd_str
318
2678
    }
319
    /// Return true if the keyword for this item is in 'ks'.
320
74829
    pub(crate) fn has_kwd_in(&self, ks: &[K]) -> bool {
321
74829
        ks.contains(&self.kwd)
322
74829
    }
323
    /// Return the arguments of this item, as a single string.
324
23032
    pub(crate) fn args_as_str(&self) -> &'a str {
325
23032
        self.args
326
23032
    }
327
    /// Return the arguments of this item as a vector.
328
102219
    fn args_as_vec(&self) -> Ref<'_, Vec<&'a str>> {
329
        // We're using an interior mutability pattern here to lazily
330
        // construct the vector.
331
102219
        if self.split_args.borrow().is_none() {
332
47198
            self.split_args.replace(Some(self.args().collect()));
333
55021
        }
334
102219
        Ref::map(self.split_args.borrow(), |opt| match opt {
335
102219
            Some(v) => v,
336
            None => panic!(),
337
102219
        })
338
102219
    }
339
    /// Return an iterator over the arguments of this item.
340
151510
    pub(crate) fn args(&self) -> impl Iterator<Item = &'a str> + use<'a, K> {
341
404698
        self.args.split(is_sp).filter(|s| !s.is_empty())
342
151510
    }
343
    /// Return the nth argument of this item, if there is one.
344
102215
    pub(crate) fn arg(&self, idx: usize) -> Option<&'a str> {
345
102215
        self.args_as_vec().get(idx).copied()
346
102215
    }
347
    /// Return the nth argument of this item, or an error if it isn't there.
348
28128
    pub(crate) fn required_arg(&self, idx: usize) -> Result<&'a str> {
349
28128
        self.arg(idx)
350
28128
            .ok_or_else(|| EK::MissingArgument.at_pos(Pos::at(self.args)))
351
28128
    }
352
    /// Try to parse the nth argument (if it exists) into some type
353
    /// that supports FromStr.
354
    ///
355
    /// Returns Ok(None) if the argument doesn't exist.
356
68632
    pub(crate) fn parse_optional_arg<V: FromStr>(&self, idx: usize) -> Result<Option<V>>
357
68632
    where
358
68632
        Error: From<V::Err>,
359
    {
360
68632
        match self.arg(idx) {
361
3
            None => Ok(None),
362
68629
            Some(s) => match s.parse() {
363
68627
                Ok(r) => Ok(Some(r)),
364
2
                Err(e) => {
365
2
                    let e: Error = e.into();
366
2
                    Err(e.or_at_pos(Pos::at(s)))
367
                }
368
            },
369
        }
370
68632
    }
371
    /// Try to parse the nth argument (if it exists) into some type
372
    /// that supports FromStr.
373
    ///
374
    /// Return an error if the argument doesn't exist.
375
68628
    pub(crate) fn parse_arg<V: FromStr>(&self, idx: usize) -> Result<V>
376
68628
    where
377
68628
        Error: From<V::Err>,
378
    {
379
68628
        match self.parse_optional_arg(idx) {
380
68625
            Ok(Some(v)) => Ok(v),
381
1
            Ok(None) => Err(EK::MissingArgument.at_pos(self.arg_pos(idx))),
382
2
            Err(e) => Err(e),
383
        }
384
68628
    }
385
    /// Return the number of arguments for this Item
386
101561
    pub(crate) fn n_args(&self) -> usize {
387
101561
        self.args().count()
388
101561
    }
389
    /// Return true iff this Item has an associated object.
390
100878
    pub(crate) fn has_obj(&self) -> bool {
391
100878
        self.object.is_some()
392
100878
    }
393
    /// Return the tag of this item's associated object, if it has one.
394
173
    pub(crate) fn obj_tag(&self) -> Option<&'a str> {
395
173
        self.object.map(|o| o.tag)
396
173
    }
397
    /// Try to decode the base64 contents of this Item's associated object.
398
    ///
399
    /// On success, return the object's tag and decoded contents.
400
24274
    pub(crate) fn obj_raw(&self) -> Result<Option<(&'a str, Vec<u8>)>> {
401
24274
        match self.object {
402
2252
            None => Ok(None),
403
22022
            Some(obj) => {
404
22022
                let decoded = base64_decode_multiline(obj.data)
405
22022
                    .map_err(|_| EK::BadObjectBase64.at_pos(Pos::at(obj.data)))?;
406
22022
                Ok(Some((obj.tag, decoded)))
407
            }
408
        }
409
24274
    }
410
    /// Try to decode the base64 contents of this Item's associated object,
411
    /// and make sure that its tag matches 'want_tag'.
412
17543
    pub(crate) fn obj(&self, want_tag: &str) -> Result<Vec<u8>> {
413
17543
        match self.obj_raw()? {
414
1
            None => Err(EK::MissingObject
415
1
                .with_msg(self.kwd.to_str())
416
1
                .at_pos(self.end_pos())),
417
17542
            Some((tag, decoded)) => {
418
17542
                if tag != want_tag {
419
2
                    Err(EK::WrongObject.at_pos(Pos::at(tag)))
420
                } else {
421
17540
                    Ok(decoded)
422
                }
423
            }
424
        }
425
17543
    }
426
    /// Try to decode the base64 contents of this item's associated object
427
    /// as a given type that implements FromBytes.
428
12448
    pub(crate) fn parse_obj<V: FromBytes>(&self, want_tag: &str) -> Result<V> {
429
12448
        let bytes = self.obj(want_tag)?;
430
        // Unwrap may be safe because above `.obj()` should return an Error if
431
        // wanted tag was not present
432
        #[allow(clippy::unwrap_used)]
433
12448
        let p = Pos::at(self.object.unwrap().data);
434
12448
        V::from_vec(bytes, p).map_err(|e| e.at_pos(p))
435
12448
    }
436
    /// Return the position of this item.
437
    ///
438
    /// This position won't be useful unless it is later contextualized
439
    /// with the containing string.
440
4742
    pub(crate) fn pos(&self) -> Pos {
441
4742
        Pos::at(self.kwd_str)
442
4742
    }
443
    /// Return the position of this Item in a string.
444
    ///
445
    /// Returns None if this item doesn't actually belong to the string.
446
10047
    pub(crate) fn offset_in(&self, s: &str) -> Option<usize> {
447
10047
        crate::util::str::str_offset(s, self.kwd_str)
448
10047
    }
449
    /// Return the position of the n'th argument of this item.
450
    ///
451
    /// If this item does not have a n'th argument, return the
452
    /// position of the end of the final argument.
453
4
    pub(crate) fn arg_pos(&self, n: usize) -> Pos {
454
4
        let args = self.args_as_vec();
455
4
        if n < args.len() {
456
3
            Pos::at(args[n])
457
        } else {
458
1
            self.last_arg_end_pos()
459
        }
460
4
    }
461
    /// Return the position at the end of the last argument.  (This will
462
    /// point to a newline.)
463
1139
    fn last_arg_end_pos(&self) -> Pos {
464
1139
        Pos::at_end_of(self.args)
465
1139
    }
466
    /// Return the position of the end of this object. (This will point to a
467
    /// newline.)
468
1313
    pub(crate) fn end_pos(&self) -> Pos {
469
1313
        match self.object {
470
176
            Some(o) => Pos::at_end_of(o.endline),
471
1137
            None => self.last_arg_end_pos(),
472
        }
473
1313
    }
474
    /// If this item occurs within s, return the byte offset
475
    /// immediately after the end of this item.
476
433
    pub(crate) fn offset_after(&self, s: &str) -> Option<usize> {
477
433
        self.end_pos().offset_within(s).map(|nl_pos| nl_pos + 1)
478
433
    }
479

            
480
    /// Return the text of this item, if it originated within `str`,
481
    /// from the start of its keyword up to and including its final newline.
482
    #[allow(dead_code)] // unused when hsdesc not enabled.
483
701
    pub(crate) fn text_within<'b>(&self, s: &'b str) -> Option<&'b str> {
484
701
        let start = self.pos().offset_within(s)?;
485
701
        let end = self.end_pos().offset_within(s)?;
486
701
        s.get(start..=end)
487
701
    }
488
}
489

            
490
/// Represents an Item that might not be present, whose arguments we
491
/// want to inspect.  If the Item is there, this acts like a proxy to the
492
/// item; otherwise, it treats the item as having no arguments.
493
pub(crate) struct MaybeItem<'a, 'b, K: Keyword>(Option<&'a Item<'b, K>>);
494

            
495
// All methods here are as for Item.
496
impl<'a, 'b, K: Keyword> MaybeItem<'a, 'b, K> {
497
    /// Return the position of this item, if it has one.
498
3
    pub(crate) fn pos(&self) -> Pos {
499
3
        match self.0 {
500
3
            Some(item) => item.pos(),
501
            None => Pos::None,
502
        }
503
3
    }
504
    /// Construct a MaybeItem from an Option reference to an item.
505
12183
    pub(crate) fn from_option(opt: Option<&'a Item<'b, K>>) -> Self {
506
12183
        MaybeItem(opt)
507
12183
    }
508

            
509
    /// If this item is present, parse its argument at position `idx`.
510
    /// Treat the absence or malformedness of the argument as an error,
511
    /// but treat the absence of this item as acceptable.
512
2239
    pub(crate) fn parse_arg<V: FromStr>(&self, idx: usize) -> Result<Option<V>>
513
2239
    where
514
2239
        Error: From<V::Err>,
515
    {
516
2239
        match self.0 {
517
2238
            Some(item) => match item.parse_arg(idx) {
518
2237
                Ok(v) => Ok(Some(v)),
519
1
                Err(e) => Err(e.or_at_pos(self.pos())),
520
            },
521
1
            None => Ok(None),
522
        }
523
2239
    }
524
    /// If this item is present, return its arguments as a single string.
525
3997
    pub(crate) fn args_as_str(&self) -> Option<&str> {
526
3997
        self.0.map(|item| item.args_as_str())
527
3997
    }
528
    /// If this item is present, parse all of its arguments as a
529
    /// single string.
530
5947
    pub(crate) fn parse_args_as_str<V: FromStr>(&self) -> Result<Option<V>>
531
5947
    where
532
5947
        Error: From<V::Err>,
533
    {
534
5947
        match self.0 {
535
2518
            Some(item) => match item.args_as_str().parse::<V>() {
536
2516
                Ok(v) => Ok(Some(v)),
537
2
                Err(e) => {
538
2
                    let e: Error = e.into();
539
2
                    Err(e.or_at_pos(self.pos()))
540
                }
541
            },
542
3429
            None => Ok(None),
543
        }
544
5947
    }
545
}
546

            
547
/// Extension trait for `Result<Item>` -- makes it convenient to implement
548
/// PauseAt predicates
549
pub(crate) trait ItemResult<K: Keyword> {
550
    /// Return true if this is an ok result with an annotation.
551
    fn is_ok_with_annotation(&self) -> bool;
552
    /// Return true if this is an ok result with a non-annotation.
553
    fn is_ok_with_non_annotation(&self) -> bool;
554
    /// Return true if this is an ok result with the keyword 'k'
555
19232
    fn is_ok_with_kwd(&self, k: K) -> bool {
556
19232
        self.is_ok_with_kwd_in(&[k])
557
19232
    }
558
    /// Return true if this is an ok result with a keyword in the slice 'ks'
559
    fn is_ok_with_kwd_in(&self, ks: &[K]) -> bool;
560
    /// Return true if this is an ok result with a keyword not in the slice 'ks'
561
    fn is_ok_with_kwd_not_in(&self, ks: &[K]) -> bool;
562
    /// Return true if this is an empty-line error.
563
    fn is_empty_line(&self) -> bool;
564
}
565

            
566
impl<'a, K: Keyword> ItemResult<K> for Result<Item<'a, K>> {
567
4491
    fn is_ok_with_annotation(&self) -> bool {
568
4491
        match self {
569
4484
            Ok(item) => item.kwd().is_annotation(),
570
7
            Err(_) => false,
571
        }
572
4491
    }
573
21
    fn is_ok_with_non_annotation(&self) -> bool {
574
21
        match self {
575
19
            Ok(item) => !item.kwd().is_annotation(),
576
2
            Err(_) => false,
577
        }
578
21
    }
579
68162
    fn is_ok_with_kwd_in(&self, ks: &[K]) -> bool {
580
68162
        match self {
581
68102
            Ok(item) => item.has_kwd_in(ks),
582
60
            Err(_) => false,
583
        }
584
68162
    }
585
6834
    fn is_ok_with_kwd_not_in(&self, ks: &[K]) -> bool {
586
6834
        match self {
587
6727
            Ok(item) => !item.has_kwd_in(ks),
588
107
            Err(_) => false,
589
        }
590
6834
    }
591
4488
    fn is_empty_line(&self) -> bool {
592
6
        matches!(
593
6
            self,
594
6
            Err(e) if e.netdoc_error_kind() == crate::err::NetdocErrorKind::EmptyLine
595
        )
596
4488
    }
597
}
598

            
599
/// A peekable cursor into a string that returns Items one by one.
600
///
601
/// This is an [`Iterator`], yielding [`Item`]s.
602
#[derive(Debug)]
603
pub(crate) struct NetDocReader<'a, K: Keyword> {
604
    // TODO: I wish there were some way around having this string
605
    // reference, since we already need one inside NetDocReaderBase.
606
    /// The underlying string being parsed.
607
    s: &'a str,
608
    /// A stream of tokens being parsed by this NetDocReader.
609
    tokens: Peekable<NetDocReaderBase<'a, K>>,
610
}
611

            
612
impl<'a, K: Keyword> NetDocReader<'a, K> {
613
    /// Construct a new NetDocReader to read tokens from `s`.
614
5246
    pub(crate) fn new(s: &'a str) -> Result<Self> {
615
        Ok(NetDocReader {
616
5246
            s,
617
5246
            tokens: NetDocReaderBase::new(s)?.peekable(),
618
        })
619
5246
    }
620
    /// Return a reference to the string used for this NetDocReader.
621
4992
    pub(crate) fn str(&self) -> &'a str {
622
4992
        self.s
623
4992
    }
624
    /// Return a wrapper around the peekable iterator in this
625
    /// NetDocReader that reads tokens until it reaches an element where
626
    /// 'f' is true.
627
11269
    pub(crate) fn pause_at<'f, 'r, F>(
628
11269
        &mut self,
629
11269
        mut f: F,
630
11269
    ) -> itertools::PeekingTakeWhile<
631
11269
        '_,
632
11269
        Self,
633
11269
        impl FnMut(&Result<Item<'a, K>>) -> bool + 'f + use<'a, 'f, F, K>,
634
11269
    >
635
11269
    where
636
11269
        'f: 'r,
637
11269
        F: FnMut(&Result<Item<'a, K>>) -> bool + 'f,
638
11269
        K: 'f,
639
    {
640
78527
        self.peeking_take_while(move |i| !f(i))
641
11269
    }
642

            
643
    /// Return true if there are no more items in this NetDocReader.
644
    // The implementation sadly needs to mutate the inner state, even if it's not *semantically*
645
    // mutated..  We don't want inner mutability just to placate clippy for an internal API.
646
    #[allow(clippy::wrong_self_convention)]
647
    #[allow(dead_code)] // TODO perhaps we should remove this ?
648
    pub(crate) fn is_exhausted(&mut self) -> bool {
649
        self.peek().is_none()
650
    }
651

            
652
    /// Give an error if there are remaining tokens in this NetDocReader.
653
2346
    pub(crate) fn should_be_exhausted(&mut self) -> Result<()> {
654
2346
        match self.peek() {
655
2345
            None => Ok(()),
656
1
            Some(Ok(t)) => Err(EK::UnexpectedToken
657
1
                .with_msg(t.kwd().to_str())
658
1
                .at_pos(t.pos())),
659
            Some(Err(e)) => Err(e.clone()),
660
        }
661
2346
    }
662

            
663
    /// Give an error if there are remaining tokens in this NetDocReader.
664
    ///
665
    /// Like [`should_be_exhausted`](Self::should_be_exhausted),
666
    /// but permit empty lines at the end of the document.
667
2229
    pub(crate) fn should_be_exhausted_but_for_empty_lines(&mut self) -> Result<()> {
668
        use crate::err::NetdocErrorKind as K;
669
2230
        while let Some(Err(e)) = self.peek() {
670
1
            if e.netdoc_error_kind() == K::EmptyLine {
671
1
                let _ignore = self.next();
672
1
            } else {
673
                break;
674
            }
675
        }
676
2229
        self.should_be_exhausted()
677
2229
    }
678

            
679
    /// Return the position from which the underlying reader is about to take
680
    /// the next token.  Use to make sure that the reader is progressing.
681
2435
    pub(crate) fn pos(&mut self) -> Pos {
682
2435
        match self.tokens.peek() {
683
2432
            Some(Ok(tok)) => tok.pos(),
684
1
            Some(Err(e)) => e.pos(),
685
2
            None => Pos::at_end_of(self.s),
686
        }
687
2435
    }
688
}
689

            
690
impl<'a, K: Keyword> Iterator for NetDocReader<'a, K> {
691
    type Item = Result<Item<'a, K>>;
692
103621
    fn next(&mut self) -> Option<Self::Item> {
693
103621
        self.tokens.next()
694
103621
    }
695
}
696

            
697
impl<'a, K: Keyword> PeekableIterator for NetDocReader<'a, K> {
698
110197
    fn peek(&mut self) -> Option<&Self::Item> {
699
110197
        self.tokens.peek()
700
110197
    }
701
}
702

            
703
impl<'a, K: Keyword> itertools::PeekingNext for NetDocReader<'a, K> {
704
80929
    fn peeking_next<F>(&mut self, f: F) -> Option<Self::Item>
705
80929
    where
706
80929
        F: FnOnce(&Self::Item) -> bool,
707
    {
708
80929
        if f(self.peek()?) { self.next() } else { None }
709
80929
    }
710
}
711

            
712
/// Check additional UTF-8 rules that the netdoc metaformat imposes on
713
/// our documents.
714
//
715
// NOTE: We might decide in the future to loosen our rules here
716
// for parsers that handle concatenated documents:
717
// we might want to reject only those documents that contain NULs.
718
// But with luck that will never be necessary.
719
5256
fn validate_utf_8_rules(s: &str) -> Result<&str> {
720
    // No BOM, or mangled BOM, is allowed.
721
5256
    let first_char = s.chars().next();
722
5256
    if [Some('\u{feff}'), Some('\u{fffe}')].contains(&first_char) {
723
3
        return Err(EK::BomMarkerFound.at_pos(Pos::at(s)));
724
5253
    }
725
    // No NUL bytes are allowed.
726
5253
    if let Some(nul_pos) = memchr::memchr(0, s.as_bytes()) {
727
5
        return Err(EK::NulFound.at_pos(Pos::from_byte(nul_pos)));
728
5248
    }
729
5248
    Ok(s)
730
5256
}
731

            
732
#[cfg(test)]
733
mod test {
734
    // @@ begin test lint list maintained by maint/add_warning @@
735
    #![allow(clippy::bool_assert_comparison)]
736
    #![allow(clippy::clone_on_copy)]
737
    #![allow(clippy::dbg_macro)]
738
    #![allow(clippy::mixed_attributes_style)]
739
    #![allow(clippy::print_stderr)]
740
    #![allow(clippy::print_stdout)]
741
    #![allow(clippy::single_char_pattern)]
742
    #![allow(clippy::unwrap_used)]
743
    #![allow(clippy::unchecked_time_subtraction)]
744
    #![allow(clippy::useless_vec)]
745
    #![allow(clippy::needless_pass_by_value)]
746
    #![allow(clippy::string_slice)] // See arti#2571
747
    //! <!-- @@ end test lint list maintained by maint/add_warning @@ -->
748
    use super::*;
749
    use crate::parse::macros::test::Fruit;
750
    use crate::{NetdocErrorKind as EK, Pos, Result};
751

            
752
    #[test]
753
    fn read_simple() {
754
        use Fruit::*;
755

            
756
        let s = "\
757
@tasty very much so
758
opt apple 77
759
banana 60
760
cherry 6
761
-----BEGIN CHERRY SYNOPSIS-----
762
8J+NkvCfjZLwn42S8J+NkvCfjZLwn42S
763
-----END CHERRY SYNOPSIS-----
764
plum hello there
765
orange    77
766
opt    lemon    tabs    and    spaces
767
";
768
        let mut r: NetDocReader<'_, Fruit> = NetDocReader::new(s).unwrap();
769

            
770
        assert_eq!(r.str(), s);
771
        assert!(r.should_be_exhausted().is_err()); // it's not exhausted.
772

            
773
        let toks: Result<Vec<_>> = r.by_ref().collect();
774
        assert!(r.should_be_exhausted().is_ok());
775

            
776
        let toks = toks.unwrap();
777
        assert_eq!(toks.len(), 7);
778
        assert_eq!(toks[0].kwd(), ANN_TASTY);
779
        assert_eq!(toks[0].n_args(), 3);
780
        assert_eq!(toks[0].args_as_str(), "very much so");
781
        assert_eq!(toks[0].arg(1), Some("much"));
782
        {
783
            let a: Vec<_> = toks[0].args().collect();
784
            assert_eq!(a, vec!["very", "much", "so"]);
785
        }
786
        assert!(toks[0].parse_arg::<usize>(0).is_err());
787
        assert!(toks[0].parse_arg::<usize>(10).is_err());
788
        assert!(!toks[0].has_obj());
789
        assert_eq!(toks[0].obj_tag(), None);
790

            
791
        assert_eq!(toks[2].pos().within(s), Pos::from_line(3, 1));
792
        assert_eq!(toks[2].arg_pos(0).within(s), Pos::from_line(3, 8));
793
        assert_eq!(toks[2].last_arg_end_pos().within(s), Pos::from_line(3, 10));
794
        assert_eq!(toks[2].end_pos().within(s), Pos::from_line(3, 10));
795

            
796
        assert_eq!(toks[3].kwd(), STONEFRUIT);
797
        assert_eq!(toks[3].kwd_str(), "cherry"); // not cherry/plum!
798
        assert_eq!(toks[3].n_args(), 1);
799
        assert_eq!(toks[3].required_arg(0), Ok("6"));
800
        assert_eq!(toks[3].parse_arg::<usize>(0), Ok(6));
801
        assert_eq!(toks[3].parse_optional_arg::<usize>(0), Ok(Some(6)));
802
        assert_eq!(toks[3].parse_optional_arg::<usize>(3), Ok(None));
803
        assert!(toks[3].has_obj());
804
        assert_eq!(toks[3].obj_tag(), Some("CHERRY SYNOPSIS"));
805
        assert_eq!(
806
            &toks[3].obj("CHERRY SYNOPSIS").unwrap()[..],
807
            "🍒🍒🍒🍒🍒🍒".as_bytes()
808
        );
809
        assert!(toks[3].obj("PLUOT SYNOPSIS").is_err());
810
        // this "end-pos" value is questionable!
811
        assert_eq!(toks[3].end_pos().within(s), Pos::from_line(7, 30));
812

            
813
        assert_eq!(toks[5].kwd(), ORANGE);
814
        assert_eq!(toks[5].n_args(), 1);
815
        assert_eq!(toks[5].args_as_str(), "77");
816

            
817
        assert_eq!(toks[6].kwd(), LEMON);
818
        assert_eq!(toks[6].n_args(), 3);
819
        assert_eq!(toks[6].args_as_str(), "tabs    and    spaces");
820
        assert_eq!(toks[6].required_arg(0), Ok("tabs"));
821
        assert_eq!(toks[6].required_arg(1), Ok("and"));
822
        assert_eq!(toks[6].required_arg(2), Ok("spaces"));
823
    }
824

            
825
    #[test]
826
    fn test_badtoks() {
827
        use Fruit::*;
828

            
829
        let s = "\
830
-foobar 9090
831
apple 3.14159
832
$hello
833
unrecognized 127.0.0.1 foo
834
plum
835
-----BEGIN WHATEVER-----
836
8J+NkvCfjZLwn42S8J+NkvCfjZLwn42S
837
-----END SOMETHING ELSE-----
838
orange
839
orange
840
-----BEGIN WHATEVER-----
841
not! base64!
842
-----END WHATEVER-----
843
guava paste
844
opt @annotation
845
orange
846
-----BEGIN LOBSTER
847
8J+NkvCfjZLwn42S8J+NkvCfjZLwn42S
848
-----END SOMETHING ELSE-----
849
orange
850
-----BEGIN !!!!!!-----
851
8J+NkvCfjZLwn42S8J+NkvCfjZLwn42S
852
-----END !!!!!!-----
853
cherry
854
-----BEGIN CHERRY SYNOPSIS-----
855
8J+NkvCfjZLwn42S8J+NkvCfjZLwn42S
856
-----END CHERRY SYNOPSIS
857

            
858
truncated line";
859

            
860
        let r: NetDocReader<'_, Fruit> = NetDocReader::new(s).unwrap();
861
        let toks: Vec<_> = r.collect();
862

            
863
        assert!(toks[0].is_err());
864
        assert_eq!(
865
            toks[0].as_ref().err().unwrap(),
866
            &EK::BadKeyword.at_pos(Pos::from_line(1, 1))
867
        );
868

            
869
        assert!(toks[1].is_ok());
870
        assert!(toks[1].is_ok_with_non_annotation());
871
        assert!(!toks[1].is_ok_with_annotation());
872
        assert!(toks[1].is_ok_with_kwd_in(&[APPLE, ORANGE]));
873
        assert!(toks[1].is_ok_with_kwd_not_in(&[ORANGE, UNRECOGNIZED]));
874
        let t = toks[1].as_ref().unwrap();
875
        assert_eq!(t.kwd(), APPLE);
876
        assert_eq!(t.arg(0), Some("3.14159"));
877

            
878
        assert!(toks[2].is_err());
879
        assert!(!toks[2].is_ok_with_non_annotation());
880
        assert!(!toks[2].is_ok_with_annotation());
881
        assert!(!toks[2].is_ok_with_kwd_in(&[APPLE, ORANGE]));
882
        assert!(!toks[2].is_ok_with_kwd_not_in(&[ORANGE, UNRECOGNIZED]));
883
        assert_eq!(
884
            toks[2].as_ref().err().unwrap(),
885
            &EK::BadKeyword.at_pos(Pos::from_line(3, 1))
886
        );
887

            
888
        assert!(toks[3].is_ok());
889
        let t = toks[3].as_ref().unwrap();
890
        assert_eq!(t.kwd(), UNRECOGNIZED);
891
        assert_eq!(t.arg(1), Some("foo"));
892

            
893
        assert!(toks[4].is_err());
894
        assert_eq!(
895
            toks[4].as_ref().err().unwrap(),
896
            &EK::BadObjectMismatchedTag.at_pos(Pos::from_line(8, 1))
897
        );
898

            
899
        assert!(toks[5].is_ok());
900
        let t = toks[5].as_ref().unwrap();
901
        assert_eq!(t.kwd(), ORANGE);
902
        assert_eq!(t.args_as_str(), "");
903

            
904
        // This blob counts as two errors: a bad base64 blob, and
905
        // then an end line.
906
        assert!(toks[6].is_err());
907
        assert_eq!(
908
            toks[6].as_ref().err().unwrap(),
909
            &EK::BadObjectBase64.at_pos(Pos::from_line(12, 1))
910
        );
911

            
912
        assert!(toks[7].is_err());
913
        assert_eq!(
914
            toks[7].as_ref().err().unwrap(),
915
            &EK::BadKeyword.at_pos(Pos::from_line(13, 1))
916
        );
917

            
918
        assert!(toks[8].is_ok());
919
        let t = toks[8].as_ref().unwrap();
920
        assert_eq!(t.kwd(), GUAVA);
921

            
922
        // this is an error because you can't use opt with annotations.
923
        assert!(toks[9].is_err());
924
        assert_eq!(
925
            toks[9].as_ref().err().unwrap(),
926
            &EK::BadKeyword.at_pos(Pos::from_line(15, 1))
927
        );
928

            
929
        // this looks like a few errors.
930
        assert!(toks[10].is_err());
931
        assert_eq!(
932
            toks[10].as_ref().err().unwrap(),
933
            &EK::BadObjectBeginTag.at_pos(Pos::from_line(17, 1))
934
        );
935
        assert!(toks[11].is_err());
936
        assert_eq!(
937
            toks[11].as_ref().err().unwrap(),
938
            &EK::BadKeyword.at_pos(Pos::from_line(18, 1))
939
        );
940
        assert!(toks[12].is_err());
941
        assert_eq!(
942
            toks[12].as_ref().err().unwrap(),
943
            &EK::BadKeyword.at_pos(Pos::from_line(19, 1))
944
        );
945

            
946
        // so does this.
947
        assert!(toks[13].is_err());
948
        assert_eq!(
949
            toks[13].as_ref().err().unwrap(),
950
            &EK::BadObjectBeginTag.at_pos(Pos::from_line(21, 1))
951
        );
952
        assert!(toks[14].is_err());
953
        assert_eq!(
954
            toks[14].as_ref().err().unwrap(),
955
            &EK::BadKeyword.at_pos(Pos::from_line(22, 1))
956
        );
957
        assert!(toks[15].is_err());
958
        assert_eq!(
959
            toks[15].as_ref().err().unwrap(),
960
            &EK::BadKeyword.at_pos(Pos::from_line(23, 1))
961
        );
962

            
963
        // not this.
964
        assert!(toks[16].is_err());
965
        assert_eq!(
966
            toks[16].as_ref().err().unwrap(),
967
            &EK::BadObjectEndTag.at_pos(Pos::from_line(27, 1))
968
        );
969

            
970
        assert!(toks[17].is_err());
971
        assert_eq!(
972
            toks[17].as_ref().err().unwrap(),
973
            &EK::EmptyLine.at_pos(Pos::from_line(28, 1))
974
        );
975

            
976
        assert!(toks[18].is_err());
977
        assert_eq!(
978
            toks[18].as_ref().err().unwrap(),
979
            &EK::TruncatedLine.at_pos(Pos::from_line(29, 15))
980
        );
981
    }
982

            
983
    #[test]
984
    fn test_leading_space_forbidden() {
985
        // We need to make sure that items with a leading space aren't accepted:
986
        // the spec forbids it, and it can provide a vector for inflating the size
987
        // of downloaded hsdescs (see prop360).
988

            
989
        // Try a simple item with a space at the front.
990
        let s = "    guava space\n";
991
        let r: NetDocReader<'_, Fruit> = NetDocReader::new(s).unwrap();
992
        let toks: Vec<_> = r.collect();
993

            
994
        // No space allowed at the start of a line.
995
        assert_eq!(
996
            toks[0].as_ref().err().unwrap(),
997
            &EK::BadKeyword.at_pos(Pos::from_line(1, 1))
998
        );
999

            
        // Try an item with an object, inserting space at the start of each ine in turn.
        let s = "cherry
-----BEGIN WHATEVER-----
8J+NkvCfjZLwn42S8J+NkvCfjZLwn42S
-----END WHATEVER-----
";
        let orig_lines = s
            .split_terminator('\n')
            .map(str::to_string)
            .collect::<Vec<_>>();
        assert_eq!(orig_lines.len(), 4);
        let expected_kinds = [
            EK::BadKeyword,
            EK::BadKeyword,
            EK::BadObjectBase64,
            EK::BadObjectBase64,
        ];
        for pos in 0..orig_lines.len() {
            let mut lines = orig_lines.clone();
            lines[pos] = format!(" {}", lines[pos]);
            let joined = format!("{}\n", lines.join("\n"));
            let r: NetDocReader<'_, Fruit> = NetDocReader::new(&joined).unwrap();
            let toks: Result<Vec<_>> = r.collect();
            assert_eq!(toks.unwrap_err().netdoc_error_kind(), expected_kinds[pos]);
        }
    }
    #[test]
    fn test_validate_strings() {
        use validate_utf_8_rules as v;
        assert_eq!(v(""), Ok(""));
        assert_eq!(v("hello world"), Ok("hello world"));
        // We don't have to test a lot more valid cases, since this function is called before
        // parsing any string.
        for s in ["\u{feff}", "\u{feff}hello world", "\u{fffe}hello world"] {
            let e = v(s).unwrap_err();
            assert_eq!(e.netdoc_error_kind(), EK::BomMarkerFound);
            assert_eq!(e.pos().offset_within(s), Some(0));
        }
        for s in [
            "\0hello world",
            "\0",
            "\0\0\0",
            "hello\0world",
            "hello world\0",
        ] {
            let e = v(s).unwrap_err();
            assert_eq!(e.netdoc_error_kind(), EK::NulFound);
            let nul_pos = e.pos().offset_within(s).unwrap();
            assert_eq!(s.as_bytes()[nul_pos], 0);
        }
    }
    fn single_fruit(s: &str) -> Item<'_, Fruit> {
        NetDocReader::<Fruit>::new(s)
            .unwrap()
            .next()
            .unwrap()
            .unwrap()
    }
    #[test]
    fn end_of_item() {
        let s = "guava friends 123   \n";
        let item = single_fruit(s);
        assert_eq!(
            item.end_pos().within(s),
            Pos::from_byte(s.find('\n').unwrap()).within(s)
        );
        let s = "cherry
-----BEGIN WHATEVER-----
8J+NkvCfjZLwn42S8J+NkvCfjZLwn42S
-----END WHATEVER-----\n";
        let item = single_fruit(s);
        dbg!(&item);
        assert_eq!(
            item.end_pos().within(s),
            Pos::from_byte(s.rfind('\n').unwrap()).within(s)
        );
    }
}