1
//! Code for managing multiple [`Keystore`](crate::Keystore)s.
2
//!
3
//! See the [`KeyMgr`] docs for more details.
4

            
5
use crate::raw::{RawEntryId, RawKeystoreEntry};
6
use crate::{
7
    ArtiPath, BoxedKeystore, KeyPath, KeyPathError, KeyPathInfo, KeyPathInfoExtractor,
8
    KeyPathPattern, KeySpecifier, KeystoreCorruptionError, KeystoreEntryResult, KeystoreId,
9
    KeystoreSelector, Result,
10
};
11

            
12
use itertools::Itertools;
13
use std::iter;
14
use std::result::Result as StdResult;
15
use tor_error::{bad_api_usage, internal, into_bad_api_usage};
16
use tor_key_forge::{
17
    ItemType, Keygen, KeygenRng, KeystoreItemType, ToEncodableCert, ToEncodableKey,
18
};
19

            
20
#[cfg(feature = "experimental-api")]
21
use crate::KeyCertificateSpecifier;
22

            
23
/// A key manager that acts as a frontend to a primary [`Keystore`](crate::Keystore) and
24
/// any number of secondary [`Keystore`](crate::Keystore)s.
25
///
26
/// Note: [`KeyMgr`] is a low-level utility and does not implement caching (the key stores are
27
/// accessed for every read/write).
28
///
29
/// The `KeyMgr` accessors - currently just [`get()`](KeyMgr::get) -
30
/// search the configured key stores in order: first the primary key store,
31
/// and then the secondary stores, in order.
32
///
33
///
34
/// ## Concurrent key store access
35
///
36
/// The key stores will allow concurrent modification by different processes. In
37
/// order to implement this safely without locking, the key store operations (get,
38
/// insert, remove) will need to be atomic.
39
///
40
/// **Note**: [`KeyMgr::generate`] and [`KeyMgr::get_or_generate`] should **not** be used
41
/// concurrently with any other `KeyMgr` operation that mutates the same key
42
/// (i.e. a key with the same `ArtiPath`), because
43
/// their outcome depends on whether the selected key store
44
/// [`contains`][crate::Keystore::contains]
45
/// the specified key (and thus suffers from a TOCTOU race).
46
#[derive(derive_builder::Builder)]
47
#[builder(pattern = "owned", build_fn(private, name = "build_unvalidated"))]
48
pub struct KeyMgr {
49
    /// The primary key store.
50
    primary_store: BoxedKeystore,
51
    /// The secondary key stores.
52
    #[builder(default, setter(custom))]
53
    secondary_stores: Vec<BoxedKeystore>,
54
    /// The key info extractors.
55
    ///
56
    /// These are initialized internally by [`KeyMgrBuilder::build`], using the values collected
57
    /// using `inventory`.
58
    #[builder(default, setter(skip))]
59
    key_info_extractors: Vec<&'static dyn KeyPathInfoExtractor>,
60
}
61

            
62
/// A keystore entry descriptor.
63
///
64
/// This identifies a key entry from a specific keystore.
65
/// The key entry can be retrieved, using [`KeyMgr::get_entry`],
66
/// or removed, using [`KeyMgr::remove_entry`].
67
///
68
/// Returned from [`KeyMgr::list_matching`].
69
#[derive(Clone, Debug, PartialEq, amplify::Getters)]
70
pub struct KeystoreEntry<'a> {
71
    /// The [`KeyPath`] of the key.
72
    key_path: KeyPath,
73
    /// The [`KeystoreItemType`] of the key.
74
    key_type: KeystoreItemType,
75
    /// The [`KeystoreId`] of the keystore where the key was found.
76
    #[getter(as_copy)]
77
    keystore_id: &'a KeystoreId,
78
    /// The [`RawEntryId`] of the key, an identifier used in
79
    /// `arti raw` operations.
80
    #[getter(skip)]
81
    raw_id: RawEntryId,
82
}
83

            
84
impl<'a> KeystoreEntry<'a> {
85
    /// Create a new `KeystoreEntry`
86
31164
    pub(crate) fn new(
87
31164
        key_path: KeyPath,
88
31164
        key_type: KeystoreItemType,
89
31164
        keystore_id: &'a KeystoreId,
90
31164
        raw_id: RawEntryId,
91
31164
    ) -> Self {
92
31164
        Self {
93
31164
            key_path,
94
31164
            key_type,
95
31164
            keystore_id,
96
31164
            raw_id,
97
31164
        }
98
31164
    }
99

            
100
    /// Return an instance of [`RawKeystoreEntry`]
101
    #[cfg(feature = "onion-service-cli-extra")]
102
624
    pub fn raw_entry(&self) -> RawKeystoreEntry {
103
624
        RawKeystoreEntry::new(self.raw_id.clone(), self.keystore_id.clone())
104
624
    }
105
}
106

            
107
// NOTE: Some methods require a `KeystoreEntryResult<KeystoreEntry>` as an
108
// argument (e.g.: `KeyMgr::raw_keystore_entry`). For this reason  implementing
109
// `From<KeystoreEntry<'a>> for KeystoreEntryResult<KeystoreEntry<'a>>` makes
110
// `KeystoreEntry` more ergonomic.
111
impl<'a> From<KeystoreEntry<'a>> for KeystoreEntryResult<KeystoreEntry<'a>> {
112
    fn from(val: KeystoreEntry<'a>) -> Self {
113
        Ok(val)
114
    }
115
}
116

            
117
impl KeyMgrBuilder {
118
    /// Construct a [`KeyMgr`] from this builder.
119
4082
    pub fn build(self) -> StdResult<KeyMgr, KeyMgrBuilderError> {
120
        use itertools::Itertools as _;
121

            
122
4082
        let mut keymgr = self.build_unvalidated()?;
123

            
124
5251
        if !keymgr.all_stores().map(|s| s.id()).all_unique() {
125
            return Err(KeyMgrBuilderError::ValidationError(
126
                "the keystore IDs are not pairwise unique".into(),
127
            ));
128
4082
        }
129

            
130
4082
        keymgr.key_info_extractors = inventory::iter::<&'static dyn KeyPathInfoExtractor>
131
4082
            .into_iter()
132
4082
            .copied()
133
4082
            .collect();
134

            
135
4082
        Ok(keymgr)
136
4082
    }
137
}
138

            
139
// TODO: auto-generate using define_list_builder_accessors/define_list_builder_helper
140
// when that becomes possible.
141
//
142
// See https://gitlab.torproject.org/tpo/core/arti/-/merge_requests/1760#note_2969841
143
impl KeyMgrBuilder {
144
    /// Access the being-built list of secondary stores (resolving default)
145
    ///
146
    /// If the field has not yet been set or accessed, the default list will be
147
    /// constructed and a mutable reference to the now-defaulted list of builders
148
    /// will be returned.
149
1060
    pub fn secondary_stores(&mut self) -> &mut Vec<BoxedKeystore> {
150
1060
        self.secondary_stores.get_or_insert(Default::default())
151
1060
    }
152

            
153
    /// Set the whole list (overriding the default)
154
    pub fn set_secondary_stores(mut self, list: Vec<BoxedKeystore>) -> Self {
155
        self.secondary_stores = Some(list);
156
        self
157
    }
158

            
159
    /// Inspect the being-built list (with default unresolved)
160
    ///
161
    /// If the list has not yet been set, or accessed, `&None` is returned.
162
    pub fn opt_secondary_stores(&self) -> &Option<Vec<BoxedKeystore>> {
163
        &self.secondary_stores
164
    }
165

            
166
    /// Mutably access the being-built list (with default unresolved)
167
    ///
168
    /// If the list has not yet been set, or accessed, `&mut None` is returned.
169
    pub fn opt_secondary_stores_mut(&mut self) -> &mut Option<Vec<BoxedKeystore>> {
170
        &mut self.secondary_stores
171
    }
172
}
173

            
174
inventory::collect!(&'static dyn crate::KeyPathInfoExtractor);
175

            
176
impl KeyMgr {
177
    /// Read a key from one of the key stores, and try to deserialize it as `K::Key`.
178
    ///
179
    /// The key returned is retrieved from the first key store that contains an entry for the given
180
    /// specifier.
181
    ///
182
    /// Returns `Ok(None)` if none of the key stores have the requested key.
183
1788
    pub fn get<K: ToEncodableKey>(&self, key_spec: &dyn KeySpecifier) -> Result<Option<K>> {
184
1788
        self.get_from_store(key_spec, &K::Key::item_type(), self.all_stores())
185
1788
    }
186

            
187
    /// Retrieve the specified keystore entry, and try to deserialize it as `K::Key`.
188
    ///
189
    /// The key returned is retrieved from the key store specified in the [`KeystoreEntry`].
190
    ///
191
    /// Returns `Ok(None)` if the key store does not contain the requested entry.
192
    ///
193
    /// Returns an error if the specified `key_type` does not match `K::Key::item_type()`.
194
76
    pub fn get_entry<K: ToEncodableKey>(&self, entry: &KeystoreEntry) -> Result<Option<K>> {
195
76
        let selector = entry.keystore_id().into();
196
76
        let store = self.select_keystore(&selector)?;
197
76
        self.get_from_store(entry.key_path(), entry.key_type(), [store].into_iter())
198
76
    }
199

            
200
    /// Retrieve the specified keystore certificate entry and the corresponding
201
    /// subject and signing keys, deserializing the subject key as `K::Key`,
202
    /// the cert as `C::Cert`, and the signing key as `C::SigningKey`.
203
    ///
204
    /// The `S` type parameter is the [`KeyCertificateSpecifier`] of the certificate.
205
    ///
206
    /// The key returned is retrieved from the key store specified in the [`KeystoreEntry`].
207
    ///
208
    /// Returns `Ok(None)` if the key store does not contain the requested entry.
209
    ///
210
    /// Returns an error if the item type of the [`KeystoreEntry`] does not match `C::item_type()`,
211
    /// or if the certificate is not valid according to [`ToEncodableCert::validate`],
212
    /// or if the [`ArtiPath`] of the entry cannot be converted to a certificate specifier
213
    /// of type `S`.
214
    #[cfg(feature = "experimental-api")]
215
22
    pub fn get_cert_entry<
216
22
        S: KeyCertificateSpecifier + for<'a> TryFrom<&'a KeyPath, Error = KeyPathError>,
217
22
        K: ToEncodableKey,
218
22
        C: ToEncodableCert<K>,
219
22
    >(
220
22
        &self,
221
22
        entry: &KeystoreEntry,
222
22
        signing_key_spec: &dyn KeySpecifier,
223
22
    ) -> Result<Option<C>> {
224
22
        let selector = entry.keystore_id().into();
225
22
        let store = self.select_keystore(&selector)?;
226
22
        let cert_spec = S::try_from(entry.key_path())
227
22
            .map_err(into_bad_api_usage!("wrong cert specifier for entry?!"))?;
228
22
        let subject_key_spec = cert_spec.subject_key_specifier();
229

            
230
22
        self.get_cert_from_store(
231
22
            entry.key_path(),
232
22
            entry.key_type(),
233
22
            signing_key_spec,
234
22
            subject_key_spec,
235
22
            [store].into_iter(),
236
        )
237
22
    }
238

            
239
    /// Read the key identified by `key_spec`.
240
    ///
241
    /// The key returned is retrieved from the first key store that contains an entry for the given
242
    /// specifier.
243
    ///
244
    /// If the requested key does not exist in any of the key stores, this generates a new key of
245
    /// type `K` from the key created using using `K::Key`'s [`Keygen`] implementation, and inserts
246
    /// it into the specified keystore, returning the newly inserted value.
247
    ///
248
    /// This is a convenience wrapper around [`get()`](KeyMgr::get) and
249
    /// [`generate()`](KeyMgr::generate).
250
164
    pub fn get_or_generate<K>(
251
164
        &self,
252
164
        key_spec: &dyn KeySpecifier,
253
164
        selector: KeystoreSelector,
254
164
        rng: &mut dyn KeygenRng,
255
164
    ) -> Result<K>
256
164
    where
257
164
        K: ToEncodableKey,
258
164
        K::Key: Keygen,
259
    {
260
164
        match self.get(key_spec)? {
261
130
            Some(k) => Ok(k),
262
34
            None => self.generate(key_spec, selector, rng, false),
263
        }
264
164
    }
265

            
266
    /// Read a key from one of the key stores specified, and try to deserialize it as `K::Key`.
267
    ///
268
    /// Returns `Ok(None)` if none of the key stores have the requested key.
269
    ///
270
    /// Returns an error if the specified keystore does not exist.
271
    // TODO: The function takes `&KeystoreId`, but it would be better to accept a
272
    // `KeystoreSelector`.
273
    // This way, the caller can pass `KeystoreSelector::Primary` directly without
274
    // needing to know the specific `KeystoreId` of the primary keystore.
275
    #[cfg(feature = "onion-service-cli-extra")]
276
76
    pub fn get_from<K: ToEncodableKey>(
277
76
        &self,
278
76
        key_spec: &dyn KeySpecifier,
279
76
        keystore_id: &KeystoreId,
280
76
    ) -> Result<Option<K>> {
281
76
        let store = std::iter::once(self.find_keystore(keystore_id)?);
282
76
        self.get_from_store(key_spec, &K::Key::item_type(), store)
283
76
    }
284

            
285
    /// Validates the integrity of a [`KeystoreEntry`].
286
    ///
287
    /// This retrieves the key corresponding to the provided [`KeystoreEntry`],
288
    /// and checks if its contents are valid (i.e. that the key can be parsed).
289
    /// The [`KeyPath`] of the entry is further validated using [`describe`](KeyMgr::describe).
290
    ///
291
    /// Returns `Ok(())` if the specified keystore entry is valid, and `Err` otherwise.
292
    ///
293
    /// NOTE: If the specified entry does not exist, this will only validate its [`KeyPath`].
294
    #[cfg(feature = "onion-service-cli-extra")]
295
    pub fn validate_entry_integrity(&self, entry: &KeystoreEntry) -> Result<()> {
296
        let selector = entry.keystore_id().into();
297
        let store = self.select_keystore(&selector)?;
298
        // Ignore the parsed key, only checking if it parses correctly
299
        let _ = store.get(entry.key_path(), entry.key_type())?;
300

            
301
        let path = entry.key_path();
302
        // Ignore the result, just checking if the path is recognized
303
        let _ = self
304
            .describe(path)
305
            .ok_or_else(|| KeystoreCorruptionError::Unrecognized(path.clone()))?;
306

            
307
        Ok(())
308
    }
309

            
310
    /// Generate a new key of type `K`, and insert it into the key store specified by `selector`.
311
    ///
312
    /// If the key already exists in the specified key store, the `overwrite` flag is used to
313
    /// decide whether to overwrite it with a newly generated key.
314
    ///
315
    /// On success, this function returns the newly generated key.
316
    ///
317
    /// Returns [`Error::KeyAlreadyExists`](crate::Error::KeyAlreadyExists)
318
    /// if the key already exists in the specified key store and `overwrite` is `false`.
319
    ///
320
    /// **IMPORTANT**: using this function concurrently with any other `KeyMgr` operation that
321
    /// mutates the key store state is **not** recommended, as it can yield surprising results! The
322
    /// outcome of [`KeyMgr::generate`] depends on whether the selected key store
323
    /// [`contains`][crate::Keystore::contains] the specified key, and thus suffers from a TOCTOU race.
324
    //
325
    // TODO (#1119): can we make this less racy without a lock? Perhaps we should say we'll always
326
    // overwrite any existing keys.
327
    //
328
    // TODO: consider replacing the overwrite boolean with a GenerateOptions type
329
    // (sort of like std::fs::OpenOptions)
330
318
    pub fn generate<K>(
331
318
        &self,
332
318
        key_spec: &dyn KeySpecifier,
333
318
        selector: KeystoreSelector,
334
318
        rng: &mut dyn KeygenRng,
335
318
        overwrite: bool,
336
318
    ) -> Result<K>
337
318
    where
338
318
        K: ToEncodableKey,
339
318
        K::Key: Keygen,
340
    {
341
318
        let store = self.select_keystore(&selector)?;
342

            
343
318
        if overwrite || !store.contains(key_spec, &K::Key::item_type())? {
344
316
            let key = K::Key::generate(rng)?;
345
316
            store.insert(&key, key_spec)?;
346

            
347
316
            Ok(K::from_encodable_key(key))
348
        } else {
349
2
            Err(crate::Error::KeyAlreadyExists)
350
        }
351
318
    }
352

            
353
    /// Insert `key` into the [`Keystore`](crate::Keystore) specified by `selector`.
354
    ///
355
    /// If the key already exists in the specified key store, the `overwrite` flag is used to
356
    /// decide whether to overwrite it with the provided key.
357
    ///
358
    /// If this key is not already in the keystore, `None` is returned.
359
    ///
360
    /// If this key already exists in the keystore, its value is updated
361
    /// and the old value is returned.
362
    ///
363
    /// Returns an error if the selected keystore is not the primary keystore or one of the
364
    /// configured secondary stores.
365
302
    pub fn insert<K: ToEncodableKey>(
366
302
        &self,
367
302
        key: K,
368
302
        key_spec: &dyn KeySpecifier,
369
302
        selector: KeystoreSelector,
370
302
        overwrite: bool,
371
302
    ) -> Result<Option<K>> {
372
302
        let key = key.to_encodable_key();
373
302
        let store = self.select_keystore(&selector)?;
374
302
        let key_type = K::Key::item_type();
375
302
        let old_key: Option<K> = self.get_from_store(key_spec, &key_type, [store].into_iter())?;
376

            
377
302
        if old_key.is_some() && !overwrite {
378
2
            Err(crate::Error::KeyAlreadyExists)
379
        } else {
380
300
            let () = store.insert(&key, key_spec)?;
381
300
            Ok(old_key)
382
        }
383
302
    }
384

            
385
    /// Remove the key identified by `key_spec` from the [`Keystore`](crate::Keystore)
386
    /// specified by `selector`.
387
    ///
388
    /// Returns an error if the selected keystore is not the primary keystore or one of the
389
    /// configured secondary stores.
390
    ///
391
    /// Returns the value of the removed key,
392
    /// or `Ok(None)` if the key does not exist in the requested keystore.
393
    ///
394
    /// Returns `Err` if an error occurred while trying to remove the key.
395
42
    pub fn remove<K: ToEncodableKey>(
396
42
        &self,
397
42
        key_spec: &dyn KeySpecifier,
398
42
        selector: KeystoreSelector,
399
42
    ) -> Result<Option<K>> {
400
42
        let store = self.select_keystore(&selector)?;
401
40
        let key_type = K::Key::item_type();
402
40
        let old_key: Option<K> = self.get_from_store(key_spec, &key_type, [store].into_iter())?;
403

            
404
40
        store.remove(key_spec, &key_type)?;
405

            
406
40
        Ok(old_key)
407
42
    }
408

            
409
    /// Remove the specified keystore entry.
410
    ///
411
    /// Like [`KeyMgr::remove`], except this function does not return the value of the removed key.
412
    ///
413
    /// A return value of `Ok(None)` indicates the key was not found in the specified key store,
414
    /// whereas `Ok(Some(())` means the key was successfully removed.
415
    //
416
    // TODO: We should be consistent and return the removed key.
417
    //
418
    // This probably will involve changing the return type of Keystore::remove
419
    // to Result<Option<ErasedKey>>.
420
1824
    pub fn remove_entry(&self, entry: &KeystoreEntry) -> Result<Option<()>> {
421
1824
        let selector = entry.keystore_id().into();
422
1824
        let store = self.select_keystore(&selector)?;
423

            
424
1824
        store.remove(entry.key_path(), entry.key_type())
425
1824
    }
426

            
427
    /// Remove the specified keystore entry.
428
    ///
429
    /// Similar to [`KeyMgr::remove_entry`], except this method accepts both recognized and
430
    /// unrecognized entries, identified by a raw id (in the form of a `&str`) and a
431
    /// [`KeystoreId`].
432
    ///
433
    /// Returns an error if the entry could not be removed, or if the entry doesn't exist.
434
    #[cfg(feature = "onion-service-cli-extra")]
435
4
    pub fn remove_unchecked(&self, raw_id: &str, keystore_id: &KeystoreId) -> Result<()> {
436
4
        let selector = KeystoreSelector::from(keystore_id);
437
4
        let store = self.select_keystore(&selector)?;
438
4
        let raw_id = store.raw_entry_id(raw_id)?;
439
4
        let store = self.select_keystore(&selector)?;
440
4
        store.remove_unchecked(&raw_id)
441
4
    }
442

            
443
    /// Return the keystore entry descriptors of the keys matching the specified [`KeyPathPattern`].
444
    ///
445
    /// NOTE: This searches for matching keys in _all_ keystores.
446
    ///
447
    /// NOTE: This function only returns the *recognized* entries that match the provided pattern.
448
    /// The unrecognized entries (i.e. those that do not have a valid [`KeyPath`]) will be filtered out,
449
    /// even if they match the specified pattern.
450
7392
    pub fn list_matching(&self, pat: &KeyPathPattern) -> Result<Vec<KeystoreEntry>> {
451
7392
        self.all_stores()
452
7854
            .map(|store| -> Result<Vec<_>> {
453
7708
                Ok(store
454
7708
                    .list()?
455
7708
                    .into_iter()
456
30638
                    .filter_map(|entry| entry.ok())
457
30012
                    .filter(|entry| entry.key_path().matches(pat))
458
7708
                    .collect::<Vec<_>>())
459
7708
            })
460
7392
            .flatten_ok()
461
7392
            .collect::<Result<Vec<_>>>()
462
7392
    }
463

            
464
    /// List keys and certificates of the specified keystore.
465
    #[cfg(feature = "onion-service-cli-extra")]
466
524
    pub fn list_by_id(&self, id: &KeystoreId) -> Result<Vec<KeystoreEntryResult<KeystoreEntry>>> {
467
524
        self.find_keystore(id)?.list()
468
524
    }
469

            
470
    /// List keys and certificates of all the keystores.
471
    #[cfg(feature = "onion-service-cli-extra")]
472
158
    pub fn list(&self) -> Result<Vec<KeystoreEntryResult<KeystoreEntry>>> {
473
158
        self.all_stores()
474
270
            .map(|store| -> Result<Vec<_>> { store.list() })
475
158
            .flatten_ok()
476
158
            .collect::<Result<Vec<_>>>()
477
158
    }
478

            
479
    /// List all the configured keystore.
480
    #[cfg(feature = "onion-service-cli-extra")]
481
158
    pub fn list_keystores(&self) -> Vec<KeystoreId> {
482
158
        self.all_stores()
483
218
            .map(|store| store.id().to_owned())
484
158
            .collect()
485
158
    }
486

            
487
    /// Describe the specified key.
488
    ///
489
    /// Returns `None` if none of the registered
490
    /// [`KeyPathInfoExtractor`]s is able to parse the specified [`KeyPath`].
491
    ///
492
    /// This function uses the [`KeyPathInfoExtractor`]s registered using
493
    /// [`register_key_info_extractor`](crate::register_key_info_extractor),
494
    /// or by [`DefaultKeySpecifier`](crate::derive_deftly_template_KeySpecifier).
495
1040
    pub fn describe(&self, path: &KeyPath) -> Option<KeyPathInfo> {
496
11284
        for info_extractor in &self.key_info_extractors {
497
11180
            if let Ok(info) = info_extractor.describe(path) {
498
936
                return Some(info);
499
10244
            }
500
        }
501

            
502
104
        None
503
1040
    }
504

            
505
    /// Attempt to retrieve a key from one of the specified `stores`.
506
    ///
507
    /// Returns the `<K as ToEncodableKey>::Key` representation of the key.
508
    ///
509
    /// See [`KeyMgr::get`] for more details.
510
3163
    fn get_from_store_raw<'a, K: ItemType>(
511
3163
        &self,
512
3163
        key_spec: &dyn KeySpecifier,
513
3163
        key_type: &KeystoreItemType,
514
3163
        stores: impl Iterator<Item = &'a BoxedKeystore>,
515
3163
    ) -> Result<Option<K>> {
516
3163
        let static_key_type = K::item_type();
517
3163
        if key_type != &static_key_type {
518
            return Err(internal!(
519
                "key type {:?} does not match the key type {:?} of requested key K::Key",
520
                key_type,
521
                static_key_type
522
            )
523
            .into());
524
3163
        }
525

            
526
4461
        for store in stores {
527
3327
            let key = match store.get(key_spec, &K::item_type()) {
528
                Ok(None) => {
529
                    // The key doesn't exist in this store, so we check the next one...
530
1298
                    continue;
531
                }
532
2029
                Ok(Some(k)) => k,
533
                Err(e) => {
534
                    // Note: we immediately return if one of the keystores is inaccessible.
535
                    return Err(e);
536
                }
537
            };
538

            
539
            // Found it! Now try to downcast it to the right type (this should _not_ fail)...
540
2029
            let key: K = key
541
2029
                .downcast::<K>()
542
2029
                .map(|k| *k)
543
2029
                .map_err(|_| internal!("failed to downcast key to requested type"))?;
544

            
545
2029
            return Ok(Some(key));
546
        }
547

            
548
1134
        Ok(None)
549
3163
    }
550

            
551
    /// Attempt to retrieve a certificate from one of the specified `stores`.
552
    #[cfg(feature = "experimental-api")]
553
22
    fn get_cert_from_store<'a, K: ToEncodableKey, C: ToEncodableCert<K>>(
554
22
        &self,
555
22
        cert_spec: &dyn KeySpecifier,
556
22
        cert_type: &KeystoreItemType,
557
22
        signing_cert_spec: &dyn KeySpecifier,
558
22
        subject_cert_spec: &dyn KeySpecifier,
559
22
        stores: impl Iterator<Item = &'a BoxedKeystore>,
560
22
    ) -> Result<Option<C>> {
561
22
        let Some(cert) = self.get_from_store_raw::<C::ParsedCert>(cert_spec, cert_type, stores)?
562
        else {
563
            return Ok(None);
564
        };
565

            
566
        // Get the subject key...
567
22
        let Some(subject) =
568
22
            self.get_from_store::<K>(subject_cert_spec, &K::Key::item_type(), self.all_stores())?
569
        else {
570
            return Ok(None);
571
        };
572
22
        let signed_with = self.get_cert_signing_key::<K, C>(signing_cert_spec)?;
573
22
        let cert = C::validate(cert, &subject, &signed_with)?;
574

            
575
20
        Ok(Some(cert))
576
22
    }
577

            
578
    /// Attempt to retrieve a key from one of the specified `stores`.
579
    ///
580
    /// See [`KeyMgr::get`] for more details.
581
2872
    fn get_from_store<'a, K: ToEncodableKey>(
582
2872
        &self,
583
2872
        key_spec: &dyn KeySpecifier,
584
2872
        key_type: &KeystoreItemType,
585
2872
        stores: impl Iterator<Item = &'a BoxedKeystore> + Clone,
586
2872
    ) -> Result<Option<K>> {
587
2872
        let Some(key) = self.get_from_store_raw::<K::Key>(key_spec, key_type, stores.clone())?
588
        else {
589
            // If the key_spec is the specifier for the public part of a keypair,
590
            // try getting the pair and extracting the public portion from it.
591
989
            let Some(key_pair_spec) = key_spec.keypair_specifier() else {
592
663
                return Ok(None);
593
            };
594

            
595
326
            let key_type = <K::KeyPair as ToEncodableKey>::Key::item_type();
596
326
            return Ok(self
597
326
                .get_from_store::<K::KeyPair>(&*key_pair_spec, &key_type, stores)?
598
326
                .map(|k| k.into()));
599
        };
600

            
601
1883
        Ok(Some(K::from_encodable_key(key)))
602
2872
    }
603

            
604
    /// Read the specified key and certificate from one of the key stores,
605
    /// deserializing the subject key as `K::Key`, the cert as `C::Cert`,
606
    /// and the signing key as `C::SigningKey`.
607
    ///
608
    /// Returns `Ok(None)` if none of the key stores have the requested key.
609
    ///
610
    // Note: the behavior of this function is a bit inconsistent with
611
    // get_or_generate_key_and_cert: here, if the cert is absent but
612
    // its subject key is not, we return Ok(None).
613
    // In get_or_generate_key_and_cert, OTOH< we return an error in that case
614
    // (because we can't possibly generate the missing subject key
615
    // without overwriting the cert of the missing key).
616
    ///
617
    /// This function validates the certificate using [`ToEncodableCert::validate`],
618
    /// returning an error if it is invalid or missing.
619
    #[cfg(feature = "experimental-api")]
620
    pub fn get_key_and_cert<K, C>(
621
        &self,
622
        spec: &dyn KeyCertificateSpecifier,
623
        signing_key_spec: &dyn KeySpecifier,
624
    ) -> Result<Option<(K, C)>>
625
    where
626
        K: ToEncodableKey,
627
        C: ToEncodableCert<K>,
628
    {
629
        let subject_key_spec = spec.subject_key_specifier();
630
        // Get the subject key...
631
        let Some(key) =
632
            self.get_from_store::<K>(subject_key_spec, &K::Key::item_type(), self.all_stores())?
633
        else {
634
            return Ok(None);
635
        };
636

            
637
        let cert_spec = spec
638
            .arti_path()
639
            .map_err(into_bad_api_usage!("invalid key certificate specifier"))?;
640

            
641
        let Some(cert) = self.get_from_store_raw::<C::ParsedCert>(
642
            &cert_spec,
643
            &<C::ParsedCert as ItemType>::item_type(),
644
            self.all_stores(),
645
        )?
646
        else {
647
            return Err(KeystoreCorruptionError::MissingCertificate.into());
648
        };
649

            
650
        // Finally, get the signing key and validate the cert
651
        let signed_with = self.get_cert_signing_key::<K, C>(signing_key_spec)?;
652
        let cert = C::validate(cert, &key, &signed_with)?;
653

            
654
        Ok(Some((key, cert)))
655
    }
656

            
657
    /// Like [`KeyMgr::get_key_and_cert`], except this function also generates the subject key
658
    /// and its corresponding certificate if they don't already exist.
659
    ///
660
    /// If the key certificate is missing, it will be generated
661
    /// from the subject key and signing key using the provided `make_certificate` callback.
662
    ///
663
    /// Generates the missing key and/or certificate as follows:
664
    ///
665
    /// ```text
666
    /// | Subject Key exists | Signing Key exists | Cert exists | Action                                 |
667
    /// |--------------------|--------------------|-------------|----------------------------------------|
668
    /// | Y                  | Y                  | Y           | Validate cert, return key and cert     |
669
    /// |                    |                    |             | if valid, error otherwise              |
670
    /// |--------------------|--------------------|-------------|----------------------------------------|
671
    /// | N                  | Y                  | N           | Generate subject key and               |
672
    /// |                    |                    |             | a new cert signed with signing key     |
673
    /// |--------------------|--------------------|-------------|----------------------------------------|
674
    /// | Y                  | Y                  | N           | Generate cert signed with signing key  |
675
    /// |--------------------|--------------------|-------------|----------------------------------------|
676
    /// | Y                  | N                  | N           | Error - cannot validate cert           |
677
    /// |                    |                    |             | if signing key is not available        |
678
    /// |--------------------|--------------------|-------------|----------------------------------------|
679
    /// | Y/N                | N                  | N           | Error - cannot generate cert           |
680
    /// |                    |                    |             | if signing key is not available        |
681
    /// |--------------------|--------------------|-------------|----------------------------------------|
682
    /// | N                  | Y/N                | Y           | Error - subject key was removed?       |
683
    /// |                    |                    |             | (we found the cert,                    |
684
    /// |                    |                    |             | but the subject key is missing)        |
685
    /// ```
686
    ///
687
    //
688
    // Note; the table above isn't a markdown table because CommonMark-flavor markdown
689
    // doesn't support multiline text in tables. Even if we trim down the text,
690
    // the resulting markdown table would be pretty unreadable in raw form
691
    // (it would have several excessively long lines, over 120 chars in len).
692
    #[cfg(feature = "experimental-api")]
693
52
    pub fn get_or_generate_key_and_cert<K, C>(
694
52
        &self,
695
52
        spec: &dyn KeyCertificateSpecifier,
696
52
        signing_key_spec: &dyn KeySpecifier,
697
52
        make_certificate: impl FnOnce(&K, &<C as ToEncodableCert<K>>::SigningKey) -> C,
698
52
        selector: KeystoreSelector,
699
52
        rng: &mut dyn KeygenRng,
700
52
    ) -> Result<(K, C)>
701
52
    where
702
52
        K: ToEncodableKey,
703
52
        K::Key: Keygen,
704
52
        C: ToEncodableCert<K>,
705
    {
706
52
        let subject_key_spec = spec.subject_key_specifier();
707
52
        let subject_key_arti_path = subject_key_spec
708
52
            .arti_path()
709
52
            .map_err(|_| bad_api_usage!("subject key does not have an ArtiPath?!"))?;
710

            
711
52
        let cert_specifier =
712
52
            ArtiPath::from_path_and_denotators(subject_key_arti_path, &spec.cert_denotators())
713
52
                .map_err(into_bad_api_usage!("invalid certificate specifier"))?;
714

            
715
52
        let maybe_cert = self.get_from_store_raw::<C::ParsedCert>(
716
52
            &cert_specifier,
717
52
            &C::ParsedCert::item_type(),
718
52
            self.all_stores(),
719
        )?;
720

            
721
52
        let maybe_subject_key = self.get::<K>(subject_key_spec)?;
722

            
723
52
        match (&maybe_cert, &maybe_subject_key) {
724
            (Some(_), None) => {
725
                return Err(KeystoreCorruptionError::MissingSubjectKey.into());
726
            }
727
52
            _ => {
728
52
                // generate key and/or cert
729
52
            }
730
        }
731
52
        let subject_key = match maybe_subject_key {
732
18
            Some(key) => key,
733
            _ => {
734
34
                let subject_keypair_spec =
735
34
                    subject_key_spec.keypair_specifier().ok_or_else(|| {
736
                        internal!(
737
                            "KeyCertificateSpecifier has no keypair specifier for the subject key?!"
738
                        )
739
                    })?;
740
34
                self.generate(&*subject_keypair_spec, selector, rng, false)?
741
            }
742
        };
743

            
744
52
        let signed_with = self.get_cert_signing_key::<K, C>(signing_key_spec)?;
745
48
        let cert = match maybe_cert {
746
            Some(cert) => C::validate(cert, &subject_key, &signed_with)?,
747
            None => {
748
48
                let cert = make_certificate(&subject_key, &signed_with);
749

            
750
48
                let () = self.insert_cert(cert.clone(), &cert_specifier, selector)?;
751

            
752
48
                cert
753
            }
754
        };
755

            
756
48
        Ok((subject_key, cert))
757
52
    }
758

            
759
    /// Return an iterator over all configured stores.
760
43352
    fn all_stores(&self) -> impl Iterator<Item = &BoxedKeystore> + Clone {
761
43352
        iter::once(&self.primary_store).chain(self.secondary_stores.iter())
762
43352
    }
763

            
764
    /// Return the [`Keystore`](crate::Keystore) matching the specified `selector`.
765
    ///
766
    /// Returns an error if the selected keystore is not the primary keystore or one of the
767
    /// configured secondary stores.
768
9188
    fn select_keystore(&self, selector: &KeystoreSelector) -> Result<&BoxedKeystore> {
769
9188
        match selector {
770
3102
            KeystoreSelector::Id(keystore_id) => self.find_keystore(keystore_id),
771
6086
            KeystoreSelector::Primary => Ok(&self.primary_store),
772
        }
773
9188
    }
774

            
775
    /// Return the [`Keystore`](crate::Keystore) with the specified `id`.
776
    ///
777
    /// Returns an error if the specified ID is not the ID of the primary keystore or
778
    /// the ID of one of the configured secondary stores.
779
4254
    fn find_keystore(&self, id: &KeystoreId) -> Result<&BoxedKeystore> {
780
4254
        self.all_stores()
781
5437
            .find(|keystore| keystore.id() == id)
782
4256
            .ok_or_else(|| crate::Error::KeystoreNotFound(id.clone()))
783
4254
    }
784

            
785
    /// Get the signing key of the certificate described by `spec`.
786
    ///
787
    /// Returns a [`KeystoreCorruptionError::MissingSigningKey`] error
788
    /// if the signing key doesn't exist in any of the keystores.
789
    #[cfg(feature = "experimental-api")]
790
74
    fn get_cert_signing_key<K, C>(
791
74
        &self,
792
74
        signing_key_spec: &dyn KeySpecifier,
793
74
    ) -> Result<C::SigningKey>
794
74
    where
795
74
        K: ToEncodableKey,
796
74
        C: ToEncodableCert<K>,
797
    {
798
74
        let Some(signing_key) = self.get_from_store::<C::SigningKey>(
799
74
            signing_key_spec,
800
74
            &<C::SigningKey as ToEncodableKey>::Key::item_type(),
801
74
            self.all_stores(),
802
        )?
803
        else {
804
4
            return Err(KeystoreCorruptionError::MissingSigningKey.into());
805
        };
806

            
807
70
        Ok(signing_key)
808
74
    }
809

            
810
    /// Insert `cert` into the [`Keystore`](crate::Keystore) specified by `selector`.
811
    ///
812
    /// If the key already exists in the specified key store, it will be overwritten.
813
    ///
814
    // NOTE: if we ever make this public we should rethink/improve its API.
815
    // TODO: maybe fold this into insert() somehow?
816
50
    fn insert_cert<K, C>(
817
50
        &self,
818
50
        cert: C,
819
50
        cert_spec: &dyn KeySpecifier,
820
50
        selector: KeystoreSelector,
821
50
    ) -> Result<()>
822
50
    where
823
50
        K: ToEncodableKey,
824
50
        K::Key: Keygen,
825
50
        C: ToEncodableCert<K>,
826
    {
827
50
        let cert = cert.to_encodable_cert();
828
50
        let store = self.select_keystore(&selector)?;
829

            
830
50
        let () = store.insert(&cert, cert_spec)?;
831
50
        Ok(())
832
50
    }
833
}
834

            
835
#[cfg(test)]
836
mod tests {
837
    // @@ begin test lint list maintained by maint/add_warning @@
838
    #![allow(clippy::bool_assert_comparison)]
839
    #![allow(clippy::clone_on_copy)]
840
    #![allow(clippy::dbg_macro)]
841
    #![allow(clippy::mixed_attributes_style)]
842
    #![allow(clippy::print_stderr)]
843
    #![allow(clippy::print_stdout)]
844
    #![allow(clippy::single_char_pattern)]
845
    #![allow(clippy::unwrap_used)]
846
    #![allow(clippy::unchecked_time_subtraction)]
847
    #![allow(clippy::useless_vec)]
848
    #![allow(clippy::needless_pass_by_value)]
849
    //! <!-- @@ end test lint list maintained by maint/add_warning @@ -->
850
    use super::*;
851
    use crate::keystore::arti::err::{ArtiNativeKeystoreError, MalformedPathError};
852
    use crate::raw::{RawEntryId, RawKeystoreEntry};
853
    use crate::test_utils::{TestDerivedKeySpecifier, TestDerivedKeypairSpecifier};
854
    use crate::{
855
        ArtiPath, ArtiPathUnavailableError, Error, KeyPath, KeystoreEntryResult, KeystoreError,
856
        UnrecognizedEntryError,
857
    };
858
    use std::path::PathBuf;
859
    use std::result::Result as StdResult;
860
    use std::str::FromStr;
861
    use std::sync::{Arc, RwLock};
862
    use tor_basic_utils::test_rng::testing_rng;
863
    use tor_cert::CertifiedKey;
864
    use tor_cert::Ed25519Cert;
865
    use tor_checkable::TimeValidityError;
866
    use tor_error::{ErrorKind, HasKind};
867
    use tor_key_forge::{
868
        CertData, CertType, EncodableItem, EncodedEd25519Cert, ErasedKey, InvalidCertError,
869
        KeyType, KeystoreItem,
870
    };
871
    use tor_llcrypto::pk::ed25519::{self, Ed25519PublicKey as _};
872
    use tor_llcrypto::rng::FakeEntropicRng;
873
    use web_time_compat::{Duration, SystemTime, SystemTimeExt};
874

            
875
    #[cfg(feature = "experimental-api")]
876
    use {
877
        crate::CertSpecifierPattern,
878
        crate::test_utils::{TestCertSpecifier, TestCertSpecifierPattern},
879
    };
880

            
881
    /// Metadata structure for tracking key operations in tests.
882
    #[derive(Clone, Debug, PartialEq)]
883
    struct KeyMetadata {
884
        /// The identifier for the item (e.g., "coot", "moorhen").
885
        item_id: String,
886
        /// The keystore from which the item was retrieved.
887
        ///
888
        /// Set by `Keystore::get`.
889
        retrieved_from: Option<KeystoreId>,
890
        /// Whether the item was generated via `Keygen::generate`.
891
        is_generated: bool,
892
    }
893

            
894
    /// Metadata structure for tracking certificate operations in tests.
895
    #[derive(Clone, Debug, PartialEq)]
896
    struct CertMetadata {
897
        /// The identifier for the subject key (e.g., "coot").
898
        subject_key_id: String,
899
        /// The identifier for the signing key (e.g., "moorhen").
900
        signing_key_id: String,
901
        /// The keystore from which the certificate was retrieved.
902
        ///
903
        /// Set by `Keystore::get`.
904
        retrieved_from: Option<KeystoreId>,
905
        /// Whether the certificate was freshly generated (i.e. returned from the "or generate"
906
        /// branch of `get_or_generate()`) or retrieved from a keystore.
907
        is_generated: bool,
908
    }
909

            
910
    /// Metadata structure for tracking item operations in tests.
911
    #[derive(Clone, Debug, PartialEq, derive_more::From)]
912
    enum ItemMetadata {
913
        /// Metadata about a key.
914
        Key(KeyMetadata),
915
        /// Metadata about a certificate.
916
        Cert(CertMetadata),
917
    }
918

            
919
    impl ItemMetadata {
920
        /// Get the item ID.
921
        ///
922
        /// For keys, this returns the key's ID.
923
        /// For certificates, this returns a formatted string identifying the subject key.
924
        fn item_id(&self) -> &str {
925
            match self {
926
                ItemMetadata::Key(k) => &k.item_id,
927
                ItemMetadata::Cert(c) => &c.subject_key_id,
928
            }
929
        }
930

            
931
        /// Get retrieved_from.
932
        fn retrieved_from(&self) -> Option<&KeystoreId> {
933
            match self {
934
                ItemMetadata::Key(k) => k.retrieved_from.as_ref(),
935
                ItemMetadata::Cert(c) => c.retrieved_from.as_ref(),
936
            }
937
        }
938

            
939
        /// Get is_generated.
940
        fn is_generated(&self) -> bool {
941
            match self {
942
                ItemMetadata::Key(k) => k.is_generated,
943
                ItemMetadata::Cert(c) => c.is_generated,
944
            }
945
        }
946

            
947
        /// Set the retrieved_from field to the specified keystore ID.
948
        fn set_retrieved_from(&mut self, id: KeystoreId) {
949
            match self {
950
                ItemMetadata::Key(meta) => meta.retrieved_from = Some(id),
951
                ItemMetadata::Cert(meta) => meta.retrieved_from = Some(id),
952
            }
953
        }
954

            
955
        /// Returns a reference to key metadata if this is a Key variant.
956
        fn as_key(&self) -> Option<&KeyMetadata> {
957
            match self {
958
                ItemMetadata::Key(meta) => Some(meta),
959
                _ => None,
960
            }
961
        }
962

            
963
        /// Returns a reference to certificate metadata if this is a Cert variant.
964
        fn as_cert(&self) -> Option<&CertMetadata> {
965
            match self {
966
                ItemMetadata::Cert(meta) => Some(meta),
967
                _ => None,
968
            }
969
        }
970
    }
971

            
972
    /// The type of "key" stored in the test key stores.
973
    #[derive(Clone, Debug)]
974
    struct TestItem {
975
        /// The underlying key.
976
        item: KeystoreItem,
977
        /// Metadata about the key.
978
        meta: ItemMetadata,
979
    }
980

            
981
    /// The type of certificate stored in the test key stores.
982
    struct TestCert(TestItem);
983

            
984
    impl ItemType for TestCert {
985
        fn item_type() -> KeystoreItemType
986
        where
987
            Self: Sized,
988
        {
989
            CertType::Ed25519TorCert.into()
990
        }
991
    }
992

            
993
    /// A "certificate" used for testing purposes.
994
    #[derive(Clone, Debug)]
995
    struct AlwaysValidCert(TestItem);
996

            
997
    /// An expired "certificate" used for testing purposes.
998
    #[derive(Clone, Debug)]
999
    struct AlwaysExpiredCert(TestItem);
    /// The corresponding fake public key type.
    #[derive(Clone, Debug)]
    struct TestPublicKey {
        /// The underlying key.
        key: KeystoreItem,
        /// Metadata about the key.
        meta: ItemMetadata,
    }
    impl From<TestItem> for TestPublicKey {
        fn from(tk: TestItem) -> TestPublicKey {
            TestPublicKey {
                key: tk.item,
                meta: tk.meta,
            }
        }
    }
    impl TestItem {
        /// Create a new test key with the specified metadata.
        fn new(item_id: &str) -> Self {
            let mut rng = testing_rng();
            TestItem {
                item: ed25519::Keypair::generate(&mut rng)
                    .as_keystore_item()
                    .unwrap(),
                meta: ItemMetadata::Key(KeyMetadata {
                    item_id: item_id.to_string(),
                    retrieved_from: None,
                    is_generated: false,
                }),
            }
        }
    }
    impl Keygen for TestItem {
        fn generate(mut rng: &mut dyn KeygenRng) -> tor_key_forge::Result<Self>
        where
            Self: Sized,
        {
            Ok(TestItem {
                item: ed25519::Keypair::generate(&mut rng).as_keystore_item()?,
                meta: ItemMetadata::Key(KeyMetadata {
                    item_id: "generated_test_key".to_string(),
                    retrieved_from: None,
                    is_generated: true,
                }),
            })
        }
    }
    impl ItemType for TestItem {
        fn item_type() -> KeystoreItemType
        where
            Self: Sized,
        {
            // Dummy value
            KeyType::Ed25519Keypair.into()
        }
    }
    impl EncodableItem for TestItem {
        fn as_keystore_item(&self) -> tor_key_forge::Result<KeystoreItem> {
            Ok(self.item.clone())
        }
    }
    impl ToEncodableKey for TestItem {
        type Key = Self;
        type KeyPair = Self;
        fn to_encodable_key(self) -> Self::Key {
            self
        }
        fn from_encodable_key(key: Self::Key) -> Self {
            key
        }
    }
    impl ItemType for TestPublicKey {
        fn item_type() -> KeystoreItemType
        where
            Self: Sized,
        {
            KeyType::Ed25519PublicKey.into()
        }
    }
    impl EncodableItem for TestPublicKey {
        fn as_keystore_item(&self) -> tor_key_forge::Result<KeystoreItem> {
            Ok(self.key.clone())
        }
    }
    impl ToEncodableKey for TestPublicKey {
        type Key = Self;
        type KeyPair = TestItem;
        fn to_encodable_key(self) -> Self::Key {
            self
        }
        fn from_encodable_key(key: Self::Key) -> Self {
            key
        }
    }
    impl ToEncodableCert<TestItem> for AlwaysValidCert {
        type ParsedCert = TestCert;
        type EncodableCert = TestItem;
        type SigningKey = TestItem;
        fn validate(
            cert: Self::ParsedCert,
            _subject: &TestItem,
            _signed_with: &Self::SigningKey,
        ) -> StdResult<Self, InvalidCertError> {
            // AlwaysValidCert is always valid
            Ok(Self(cert.0))
        }
        /// Convert this cert to a type that implements [`EncodableKey`].
        fn to_encodable_cert(self) -> Self::EncodableCert {
            self.0
        }
    }
    impl ToEncodableCert<TestItem> for AlwaysExpiredCert {
        type ParsedCert = TestCert;
        type EncodableCert = TestItem;
        type SigningKey = TestItem;
        fn validate(
            _cert: Self::ParsedCert,
            _subject: &TestItem,
            _signed_with: &Self::SigningKey,
        ) -> StdResult<Self, InvalidCertError> {
            Err(InvalidCertError::TimeValidity(TimeValidityError::Expired(
                Duration::from_secs(60),
            )))
        }
        /// Convert this cert to a type that implements [`EncodableKey`].
        fn to_encodable_cert(self) -> Self::EncodableCert {
            self.0
        }
    }
    #[derive(thiserror::Error, Debug, Clone, derive_more::Display)]
    enum MockKeystoreError {
        NotFound,
    }
    impl KeystoreError for MockKeystoreError {}
    impl HasKind for MockKeystoreError {
        fn kind(&self) -> ErrorKind {
            // Return a dummy ErrorKind for the purposes of this test
            tor_error::ErrorKind::Other
        }
    }
    fn build_raw_id_path<T: ToString>(key_path: &T, key_type: &KeystoreItemType) -> RawEntryId {
        let mut path = key_path.to_string();
        path.push('.');
        path.push_str(&key_type.arti_extension());
        RawEntryId::Path(PathBuf::from(&path))
    }
    struct Keystore {
        inner: RwLock<Vec<KeystoreEntryResult<(ArtiPath, KeystoreItemType, TestItem)>>>,
        id: KeystoreId,
    }
    impl Keystore {
        fn new(id: &str) -> Self {
            let id = KeystoreId::from_str(id).unwrap();
            Self {
                inner: Default::default(),
                id,
            }
        }
        fn new_boxed(id: &str) -> BoxedKeystore {
            Box::new(Self::new(id))
        }
    }
    impl crate::Keystore for Keystore {
        fn contains(
            &self,
            key_spec: &dyn KeySpecifier,
            item_type: &KeystoreItemType,
        ) -> Result<bool> {
            let wanted_arti_path = key_spec.arti_path().unwrap();
            Ok(self.inner.read().unwrap().iter().any(|res| match res {
                Ok((spec, ty, _)) => spec == &wanted_arti_path && ty == item_type,
                Err(_) => false,
            }))
        }
        fn id(&self) -> &KeystoreId {
            &self.id
        }
        fn get(
            &self,
            key_spec: &dyn KeySpecifier,
            item_type: &KeystoreItemType,
        ) -> Result<Option<ErasedKey>> {
            let key_spec = key_spec.arti_path().unwrap();
            Ok(self.inner.read().unwrap().iter().find_map(|res| {
                if let Ok((arti_path, ty, k)) = res {
                    if arti_path == &key_spec && ty == item_type {
                        let mut k = k.clone();
                        k.meta.set_retrieved_from(self.id().clone());
                        match item_type {
                            KeystoreItemType::Key(_) => {
                                return Some(Box::new(k) as Box<dyn ItemType>);
                            }
                            KeystoreItemType::Cert(_) => {
                                // Hack: the KeyMgr code will want to downcast cert types
                                // to C::ParsedCert, so we need to avoid returning the bare
                                // TestItem here
                                return Some(Box::new(TestCert(k)) as Box<dyn ItemType>);
                            }
                            _ => panic!("unknown item type?!"),
                        }
                    }
                }
                None
            }))
        }
        #[cfg(feature = "onion-service-cli-extra")]
        fn raw_entry_id(&self, raw_id: &str) -> Result<RawEntryId> {
            Ok(RawEntryId::Path(PathBuf::from(raw_id.to_string())))
        }
        fn insert(&self, key: &dyn EncodableItem, key_spec: &dyn KeySpecifier) -> Result<()> {
            let key = key.downcast_ref::<TestItem>().unwrap();
            let item = key.as_keystore_item()?;
            let item_type = item.item_type()?;
            self.inner
                .write()
                .unwrap()
                // TODO: `insert` is used instead of `push`, because some of the
                // tests (mainly `insert_and_get` and `keygen`) fail otherwise.
                // It could be a good idea to use `push` and adapt the tests,
                // in order to reduce cognitive complexity.
                .insert(
                    0,
                    Ok((key_spec.arti_path().unwrap(), item_type, key.clone())),
                );
            Ok(())
        }
        fn remove(
            &self,
            key_spec: &dyn KeySpecifier,
            item_type: &KeystoreItemType,
        ) -> Result<Option<()>> {
            let wanted_arti_path = key_spec.arti_path().unwrap();
            let index = self.inner.read().unwrap().iter().position(|res| {
                if let Ok((arti_path, ty, _)) = res {
                    arti_path == &wanted_arti_path && ty == item_type
                } else {
                    false
                }
            });
            let Some(index) = index else {
                return Ok(None);
            };
            let _ = self.inner.write().unwrap().remove(index);
            Ok(Some(()))
        }
        #[cfg(feature = "onion-service-cli-extra")]
        fn remove_unchecked(&self, entry_id: &RawEntryId) -> Result<()> {
            let index = self.inner.read().unwrap().iter().position(|res| match res {
                Ok((spec, ty, _)) => {
                    let id = build_raw_id_path(spec, ty);
                    entry_id == &id
                }
                Err(e) => e.entry().raw_id() == entry_id,
            });
            let Some(index) = index else {
                return Err(Error::Keystore(Arc::new(MockKeystoreError::NotFound)));
            };
            let _ = self.inner.write().unwrap().remove(index);
            Ok(())
        }
        fn list(&self) -> Result<Vec<KeystoreEntryResult<KeystoreEntry>>> {
            Ok(self
                .inner
                .read()
                .unwrap()
                .iter()
                .map(|res| match res {
                    Ok((arti_path, ty, _)) => {
                        let raw_id = RawEntryId::Path(PathBuf::from(&arti_path.to_string()));
                        Ok(KeystoreEntry::new(
                            KeyPath::Arti(arti_path.clone()),
                            ty.clone(),
                            self.id(),
                            raw_id,
                        ))
                    }
                    Err(e) => Err(e.clone()),
                })
                .collect())
        }
    }
    // Populate `keystore` with the specified number of unrecognized entries.
    fn add_unrecognized_entries(keystore: &mut Keystore, count: usize) {
        for i in 0..count {
            let invalid_key_path = PathBuf::from(&format!("unrecognized_entry{}", i));
            let raw_id = RawEntryId::Path(invalid_key_path.clone());
            let entry = RawKeystoreEntry::new(raw_id, keystore.id.clone()).into();
            let entry = UnrecognizedEntryError::new(
                entry,
                Arc::new(ArtiNativeKeystoreError::MalformedPath {
                    path: invalid_key_path,
                    err: MalformedPathError::NoExtension,
                }),
            );
            keystore.inner.write().unwrap().push(Err(entry));
        }
    }
    macro_rules! impl_specifier {
        ($name:tt, $id:expr) => {
            struct $name;
            impl KeySpecifier for $name {
                fn arti_path(&self) -> StdResult<ArtiPath, ArtiPathUnavailableError> {
                    Ok(ArtiPath::new($id.into()).map_err(|e| tor_error::internal!("{e}"))?)
                }
                fn ctor_path(&self) -> Option<crate::CTorPath> {
                    None
                }
                fn keypair_specifier(&self) -> Option<Box<dyn KeySpecifier>> {
                    None
                }
            }
        };
    }
    impl_specifier!(TestKeySpecifier1, "spec1");
    impl_specifier!(TestKeySpecifier2, "spec2");
    impl_specifier!(TestKeySpecifier3, "spec3");
    impl_specifier!(TestKeySpecifier4, "spec4");
    impl_specifier!(TestPublicKeySpecifier1, "pub-spec1");
    /// Create a test `KeystoreEntry`.
    fn entry_descriptor(
        specifier: impl KeySpecifier,
        key_type: KeystoreItemType,
        keystore_id: &KeystoreId,
    ) -> KeystoreEntry {
        let arti_path = specifier.arti_path().unwrap();
        let raw_id = RawEntryId::Path(PathBuf::from(arti_path.as_ref()));
        KeystoreEntry {
            key_path: arti_path.into(),
            key_type,
            keystore_id,
            raw_id,
        }
    }
    #[test]
    #[allow(clippy::cognitive_complexity)]
    fn insert_and_get() {
        let mut builder = KeyMgrBuilder::default().primary_store(Keystore::new_boxed("keystore1"));
        builder.secondary_stores().extend([
            Keystore::new_boxed("keystore2"),
            Keystore::new_boxed("keystore3"),
        ]);
        let mgr = builder.build().unwrap();
        // Insert a key into Keystore2
        let old_key = mgr
            .insert(
                TestItem::new("coot"),
                &TestKeySpecifier1,
                KeystoreSelector::Id(&KeystoreId::from_str("keystore2").unwrap()),
                true,
            )
            .unwrap();
        assert!(old_key.is_none());
        let key = mgr.get::<TestItem>(&TestKeySpecifier1).unwrap().unwrap();
        assert_eq!(key.meta.item_id(), "coot");
        assert_eq!(
            key.meta.retrieved_from(),
            Some(&KeystoreId::from_str("keystore2").unwrap())
        );
        assert_eq!(key.meta.is_generated(), false);
        // Insert a different key using the _same_ key specifier.
        let old_key = mgr
            .insert(
                TestItem::new("gull"),
                &TestKeySpecifier1,
                KeystoreSelector::Id(&KeystoreId::from_str("keystore2").unwrap()),
                true,
            )
            .unwrap()
            .unwrap();
        assert_eq!(old_key.meta.item_id(), "coot");
        assert_eq!(
            old_key.meta.retrieved_from(),
            Some(&KeystoreId::from_str("keystore2").unwrap())
        );
        assert_eq!(old_key.meta.is_generated(), false);
        // Check that the original value was overwritten:
        let key = mgr.get::<TestItem>(&TestKeySpecifier1).unwrap().unwrap();
        assert_eq!(key.meta.item_id(), "gull");
        assert_eq!(
            key.meta.retrieved_from(),
            Some(&KeystoreId::from_str("keystore2").unwrap())
        );
        assert_eq!(key.meta.is_generated(), false);
        // Insert a different key using the _same_ key specifier (overwrite = false)
        let err = mgr
            .insert(
                TestItem::new("gull"),
                &TestKeySpecifier1,
                KeystoreSelector::Id(&KeystoreId::from_str("keystore2").unwrap()),
                false,
            )
            .unwrap_err();
        assert!(matches!(err, crate::Error::KeyAlreadyExists));
        // Insert a new key into Keystore2 (overwrite = false)
        let old_key = mgr
            .insert(
                TestItem::new("penguin"),
                &TestKeySpecifier2,
                KeystoreSelector::Id(&KeystoreId::from_str("keystore2").unwrap()),
                false,
            )
            .unwrap();
        assert!(old_key.is_none());
        // Insert a key into the primary keystore
        let old_key = mgr
            .insert(
                TestItem::new("moorhen"),
                &TestKeySpecifier3,
                KeystoreSelector::Primary,
                true,
            )
            .unwrap();
        assert!(old_key.is_none());
        let key = mgr.get::<TestItem>(&TestKeySpecifier3).unwrap().unwrap();
        assert_eq!(key.meta.item_id(), "moorhen");
        assert_eq!(
            key.meta.retrieved_from(),
            Some(&KeystoreId::from_str("keystore1").unwrap())
        );
        assert_eq!(key.meta.is_generated(), false);
        // The key doesn't exist in any of the stores yet.
        assert!(mgr.get::<TestItem>(&TestKeySpecifier4).unwrap().is_none());
        // Insert the same key into all 3 key stores, in reverse order of keystore priority
        // (otherwise KeyMgr::get will return the key from the primary store for each iteration and
        // we won't be able to see the key was actually inserted in each store).
        for store in ["keystore3", "keystore2", "keystore1"] {
            let old_key = mgr
                .insert(
                    TestItem::new("cormorant"),
                    &TestKeySpecifier4,
                    KeystoreSelector::Id(&KeystoreId::from_str(store).unwrap()),
                    true,
                )
                .unwrap();
            assert!(old_key.is_none());
            // Ensure the key now exists in `store`.
            let key = mgr.get::<TestItem>(&TestKeySpecifier4).unwrap().unwrap();
            assert_eq!(key.meta.item_id(), "cormorant");
            assert_eq!(
                key.meta.retrieved_from(),
                Some(&KeystoreId::from_str(store).unwrap())
            );
            assert_eq!(key.meta.is_generated(), false);
        }
        // The key exists in all key stores, but if no keystore_id is specified, we return the
        // value from the first key store it is found in (in this case, Keystore1)
        let key = mgr.get::<TestItem>(&TestKeySpecifier4).unwrap().unwrap();
        assert_eq!(key.meta.item_id(), "cormorant");
        assert_eq!(
            key.meta.retrieved_from(),
            Some(&KeystoreId::from_str("keystore1").unwrap())
        );
        assert_eq!(key.meta.is_generated(), false);
    }
    #[test]
    #[cfg(feature = "onion-service-cli-extra")]
    fn get_from() {
        let mut builder = KeyMgrBuilder::default().primary_store(Keystore::new_boxed("keystore1"));
        builder.secondary_stores().extend([
            Keystore::new_boxed("keystore2"),
            Keystore::new_boxed("keystore3"),
        ]);
        let mgr = builder.build().unwrap();
        let keystore1_id = KeystoreId::from_str("keystore1").unwrap();
        let keystore2_id = KeystoreId::from_str("keystore2").unwrap();
        let key_id_1 = "mantis shrimp";
        let key_id_2 = "tardigrade";
        // Insert a key into Keystore1
        let _ = mgr
            .insert(
                TestItem::new(key_id_1),
                &TestKeySpecifier1,
                KeystoreSelector::Id(&keystore1_id),
                true,
            )
            .unwrap();
        // Insert a key into Keystore2
        let _ = mgr
            .insert(
                TestItem::new(key_id_2),
                &TestKeySpecifier1,
                KeystoreSelector::Id(&keystore2_id),
                true,
            )
            .unwrap();
        // Retrieve key
        let key = mgr
            .get_from::<TestItem>(&TestKeySpecifier1, &keystore2_id)
            .unwrap()
            .unwrap();
        assert_eq!(key.meta.item_id(), key_id_2);
        assert_eq!(key.meta.retrieved_from(), Some(&keystore2_id));
    }
    #[test]
    fn get_from_keypair() {
        const KEYSTORE_ID1: &str = "keystore1";
        const KEYSTORE_ID2: &str = "keystore2";
        let mut builder = KeyMgrBuilder::default().primary_store(Keystore::new_boxed(KEYSTORE_ID1));
        builder
            .secondary_stores()
            .extend([Keystore::new_boxed(KEYSTORE_ID2)]);
        let mgr = builder.build().unwrap();
        let keystore2 = KeystoreId::from_str(KEYSTORE_ID2).unwrap();
        // Insert a key into Keystore2
        let _ = mgr
            .insert(
                TestItem::new("nightjar"),
                &TestDerivedKeypairSpecifier,
                KeystoreSelector::Id(&keystore2),
                true,
            )
            .unwrap();
        macro_rules! boxed {
            ($closure:expr) => {
                Box::new($closure) as _
            };
        }
        #[allow(clippy::type_complexity)]
        let getters: &[(&'static str, Box<dyn Fn() -> Result<Option<TestPublicKey>>>)] = &[
            (
                "get",
                boxed!(|| mgr.get::<TestPublicKey>(&TestDerivedKeySpecifier)),
            ),
            #[cfg(feature = "onion-service-cli-extra")]
            (
                "get_from",
                boxed!(|| mgr.get_from::<TestPublicKey>(&TestDerivedKeySpecifier, &keystore2)),
            ),
            (
                "remove",
                boxed!(|| mgr.remove::<TestPublicKey>(
                    &TestDerivedKeySpecifier,
                    KeystoreSelector::Id(&keystore2)
                )),
            ),
        ];
        for (test_name, getter) in getters {
            // Retrieve the public key (internally, the keymgr should be able
            // to extract it from the TestItem "keypair" type).
            let key = getter().unwrap().expect(test_name);
            assert_eq!(key.meta.item_id(), "nightjar", "{test_name}");
            assert_eq!(key.meta.retrieved_from(), Some(&keystore2), "{test_name}");
        }
    }
    #[test]
    fn remove() {
        let mut builder = KeyMgrBuilder::default().primary_store(Keystore::new_boxed("keystore1"));
        builder.secondary_stores().extend([
            Keystore::new_boxed("keystore2"),
            Keystore::new_boxed("keystore3"),
        ]);
        let mgr = builder.build().unwrap();
        assert!(
            !mgr.secondary_stores[0]
                .contains(&TestKeySpecifier1, &TestItem::item_type())
                .unwrap()
        );
        // Insert a key into Keystore2
        mgr.insert(
            TestItem::new("coot"),
            &TestKeySpecifier1,
            KeystoreSelector::Id(&KeystoreId::from_str("keystore2").unwrap()),
            true,
        )
        .unwrap();
        let key = mgr.get::<TestItem>(&TestKeySpecifier1).unwrap().unwrap();
        assert_eq!(key.meta.item_id(), "coot");
        assert_eq!(
            key.meta.retrieved_from(),
            Some(&KeystoreId::from_str("keystore2").unwrap())
        );
        assert_eq!(key.meta.is_generated(), false);
        // Try to remove the key from a non-existent key store
        assert!(
            mgr.remove::<TestItem>(
                &TestKeySpecifier1,
                KeystoreSelector::Id(&KeystoreId::from_str("not_an_id_we_know_of").unwrap())
            )
            .is_err()
        );
        // The key still exists in Keystore2
        assert!(
            mgr.secondary_stores[0]
                .contains(&TestKeySpecifier1, &TestItem::item_type())
                .unwrap()
        );
        // Try to remove the key from the primary key store
        assert!(
            mgr.remove::<TestItem>(&TestKeySpecifier1, KeystoreSelector::Primary)
                .unwrap()
                .is_none()
        );
        // The key still exists in Keystore2
        assert!(
            mgr.secondary_stores[0]
                .contains(&TestKeySpecifier1, &TestItem::item_type())
                .unwrap()
        );
        // Removing from Keystore2 should succeed.
        let removed_key = mgr
            .remove::<TestItem>(
                &TestKeySpecifier1,
                KeystoreSelector::Id(&KeystoreId::from_str("keystore2").unwrap()),
            )
            .unwrap()
            .unwrap();
        assert_eq!(removed_key.meta.item_id(), "coot");
        assert_eq!(
            removed_key.meta.retrieved_from(),
            Some(&KeystoreId::from_str("keystore2").unwrap())
        );
        assert_eq!(removed_key.meta.is_generated(), false);
        // The key doesn't exist in Keystore2 anymore
        assert!(
            !mgr.secondary_stores[0]
                .contains(&TestKeySpecifier1, &TestItem::item_type())
                .unwrap()
        );
    }
    #[test]
    fn keygen() {
        let mut rng = FakeEntropicRng(testing_rng());
        let mgr = KeyMgrBuilder::default()
            .primary_store(Keystore::new_boxed("keystore1"))
            .build()
            .unwrap();
        mgr.insert(
            TestItem::new("coot"),
            &TestKeySpecifier1,
            KeystoreSelector::Primary,
            true,
        )
        .unwrap();
        // There is no corresponding public key entry.
        assert!(
            mgr.get::<TestPublicKey>(&TestPublicKeySpecifier1)
                .unwrap()
                .is_none()
        );
        // Try to generate a new key (overwrite = false)
        let err = mgr
            .generate::<TestItem>(
                &TestKeySpecifier1,
                KeystoreSelector::Primary,
                &mut rng,
                false,
            )
            .unwrap_err();
        assert!(matches!(err, crate::Error::KeyAlreadyExists));
        // The previous entry was not overwritten because overwrite = false
        let key = mgr.get::<TestItem>(&TestKeySpecifier1).unwrap().unwrap();
        assert_eq!(key.meta.item_id(), "coot");
        assert_eq!(
            key.meta.retrieved_from(),
            Some(&KeystoreId::from_str("keystore1").unwrap())
        );
        assert_eq!(key.meta.is_generated(), false);
        // We don't store public keys in the keystore
        assert!(
            mgr.get::<TestPublicKey>(&TestPublicKeySpecifier1)
                .unwrap()
                .is_none()
        );
        // Try to generate a new key (overwrite = true)
        let generated_key = mgr
            .generate::<TestItem>(
                &TestKeySpecifier1,
                KeystoreSelector::Primary,
                &mut rng,
                true,
            )
            .unwrap();
        assert_eq!(generated_key.meta.item_id(), "generated_test_key");
        // Not set in a freshly generated key, because KeyMgr::generate()
        // returns it straight away, without going through Keystore::get()
        assert_eq!(generated_key.meta.retrieved_from(), None);
        assert_eq!(generated_key.meta.is_generated(), true);
        // Retrieve the inserted key
        let retrieved_key = mgr.get::<TestItem>(&TestKeySpecifier1).unwrap().unwrap();
        assert_eq!(retrieved_key.meta.item_id(), "generated_test_key");
        assert_eq!(
            retrieved_key.meta.retrieved_from(),
            Some(&KeystoreId::from_str("keystore1").unwrap())
        );
        assert_eq!(retrieved_key.meta.is_generated(), true);
        // We don't store public keys in the keystore
        assert!(
            mgr.get::<TestPublicKey>(&TestPublicKeySpecifier1)
                .unwrap()
                .is_none()
        );
    }
    #[test]
    fn get_or_generate() {
        let mut rng = FakeEntropicRng(testing_rng());
        let mut builder = KeyMgrBuilder::default().primary_store(Keystore::new_boxed("keystore1"));
        builder.secondary_stores().extend([
            Keystore::new_boxed("keystore2"),
            Keystore::new_boxed("keystore3"),
        ]);
        let mgr = builder.build().unwrap();
        let keystore2 = KeystoreId::from_str("keystore2").unwrap();
        let entry_desc1 = entry_descriptor(TestKeySpecifier1, TestItem::item_type(), &keystore2);
        assert!(mgr.get_entry::<TestItem>(&entry_desc1).unwrap().is_none());
        mgr.insert(
            TestItem::new("coot"),
            &TestKeySpecifier1,
            KeystoreSelector::Id(&keystore2),
            true,
        )
        .unwrap();
        // The key already exists in keystore 2 so it won't be auto-generated.
        let key = mgr
            .get_or_generate::<TestItem>(&TestKeySpecifier1, KeystoreSelector::Primary, &mut rng)
            .unwrap();
        assert_eq!(key.meta.item_id(), "coot");
        assert_eq!(
            key.meta.retrieved_from(),
            Some(&KeystoreId::from_str("keystore2").unwrap())
        );
        assert_eq!(key.meta.is_generated(), false);
        assert_eq!(
            mgr.get_entry::<TestItem>(&entry_desc1)
                .unwrap()
                .map(|k| k.meta),
            Some(ItemMetadata::Key(KeyMetadata {
                item_id: "coot".to_string(),
                retrieved_from: Some(keystore2.clone()),
                is_generated: false,
            }))
        );
        // This key doesn't exist in any of the keystores, so it will be auto-generated and
        // inserted into keystore 3.
        let keystore3 = KeystoreId::from_str("keystore3").unwrap();
        let generated_key = mgr
            .get_or_generate::<TestItem>(
                &TestKeySpecifier2,
                KeystoreSelector::Id(&keystore3),
                &mut rng,
            )
            .unwrap();
        assert_eq!(generated_key.meta.item_id(), "generated_test_key");
        // Not set in a freshly generated key, because KeyMgr::get_or_generate()
        // returns it straight away, without going through Keystore::get()
        assert_eq!(generated_key.meta.retrieved_from(), None);
        assert_eq!(generated_key.meta.is_generated(), true);
        // Retrieve the inserted key
        let retrieved_key = mgr.get::<TestItem>(&TestKeySpecifier2).unwrap().unwrap();
        assert_eq!(retrieved_key.meta.item_id(), "generated_test_key");
        assert_eq!(
            retrieved_key.meta.retrieved_from(),
            Some(&KeystoreId::from_str("keystore3").unwrap())
        );
        assert_eq!(retrieved_key.meta.is_generated(), true);
        let entry_desc2 = entry_descriptor(TestKeySpecifier2, TestItem::item_type(), &keystore3);
        assert_eq!(
            mgr.get_entry::<TestItem>(&entry_desc2)
                .unwrap()
                .map(|k| k.meta),
            Some(ItemMetadata::Key(KeyMetadata {
                item_id: "generated_test_key".to_string(),
                retrieved_from: Some(keystore3.clone()),
                is_generated: true,
            }))
        );
        let arti_pat = KeyPathPattern::Arti("*".to_string());
        let matching = mgr.list_matching(&arti_pat).unwrap();
        assert_eq!(matching.len(), 2);
        assert!(matching.contains(&entry_desc1));
        assert!(matching.contains(&entry_desc2));
        assert_eq!(mgr.remove_entry(&entry_desc2).unwrap(), Some(()));
        assert!(mgr.get_entry::<TestItem>(&entry_desc2).unwrap().is_none());
        assert!(mgr.remove_entry(&entry_desc2).unwrap().is_none());
    }
    #[test]
    fn list_matching_ignores_unrecognized_keys() {
        let mut keystore = Keystore::new("keystore1");
        add_unrecognized_entries(&mut keystore, 1);
        let builder = KeyMgrBuilder::default().primary_store(Box::new(keystore));
        let mgr = builder.build().unwrap();
        let keystore1 = KeystoreId::from_str("keystore1").unwrap();
        mgr.insert(
            TestItem::new("whale shark"),
            &TestKeySpecifier1,
            KeystoreSelector::Id(&keystore1),
            true,
        )
        .unwrap();
        let arti_pat = KeyPathPattern::Arti("*".to_string());
        let valid_key_path = KeyPath::Arti(TestKeySpecifier1.arti_path().unwrap());
        let matching = mgr.list_matching(&arti_pat).unwrap();
        // assert the unrecognized key has been filtered out
        assert_eq!(matching.len(), 1);
        assert_eq!(matching.first().unwrap().key_path(), &valid_key_path);
    }
    #[cfg(feature = "onion-service-cli-extra")]
    #[test]
    /// Test all `arti keys` subcommands
    // TODO: split this in different tests
    fn keys_subcommands() {
        let mut keystore = Keystore::new("keystore1");
        add_unrecognized_entries(&mut keystore, 1);
        let mut builder = KeyMgrBuilder::default().primary_store(Box::new(keystore));
        builder.secondary_stores().extend([
            Keystore::new_boxed("keystore2"),
            Keystore::new_boxed("keystore3"),
        ]);
        let mgr = builder.build().unwrap();
        let keystore1id = KeystoreId::from_str("keystore1").unwrap();
        let keystore2id = KeystoreId::from_str("keystore2").unwrap();
        let keystore3id = KeystoreId::from_str("keystore3").unwrap();
        // Insert a key into Keystore1
        let _ = mgr
            .insert(
                TestItem::new("pangolin"),
                &TestKeySpecifier1,
                KeystoreSelector::Id(&keystore1id),
                true,
            )
            .unwrap();
        // Insert a key into Keystore2
        let _ = mgr
            .insert(
                TestItem::new("coot"),
                &TestKeySpecifier2,
                KeystoreSelector::Id(&keystore2id),
                true,
            )
            .unwrap();
        // Insert a key into Keystore3
        let _ = mgr
            .insert(
                TestItem::new("penguin"),
                &TestKeySpecifier3,
                KeystoreSelector::Id(&keystore3id),
                true,
            )
            .unwrap();
        let assert_key = |path, ty, expected_path: &ArtiPath, expected_type| {
            assert_eq!(ty, expected_type);
            assert_eq!(path, &KeyPath::Arti(expected_path.clone()));
        };
        let item_type = TestItem::new("axolotl").item.item_type().unwrap();
        let unrecognized_entry_id = RawEntryId::Path(PathBuf::from("unrecognized_entry0"));
        // Test `list`
        let entries = mgr.list().unwrap();
        let expected_items = [
            (keystore1id, TestKeySpecifier1.arti_path().unwrap()),
            (keystore2id, TestKeySpecifier2.arti_path().unwrap()),
            (keystore3id, TestKeySpecifier3.arti_path().unwrap()),
        ];
        // Secondary keystores contain 1 valid key each
        let mut recognized_entries = 0;
        let mut unrecognized_entries = 0;
        for entry in entries.iter() {
            match entry {
                Ok(e) => {
                    if let Some((_, expected_arti_path)) = expected_items
                        .iter()
                        .find(|(keystore_id, _)| keystore_id == e.keystore_id())
                    {
                        assert_key(e.key_path(), e.key_type(), expected_arti_path, &item_type);
                        recognized_entries += 1;
                        continue;
                    }
                    panic!("Unexpected key encountered {:?}", e);
                }
                Err(u) => {
                    assert_eq!(u.entry().raw_id(), &unrecognized_entry_id);
                    unrecognized_entries += 1;
                }
            }
        }
        assert_eq!(recognized_entries, 3);
        assert_eq!(unrecognized_entries, 1);
        // Test `list_keystores`
        let keystores = mgr.list_keystores().iter().len();
        assert_eq!(keystores, 3);
        // Test `list_by_id`
        let primary_keystore_id = KeystoreId::from_str("keystore1").unwrap();
        let entries = mgr.list_by_id(&primary_keystore_id).unwrap();
        // Primary keystore contains a valid key and an unrecognized key
        let mut recognized_entries = 0;
        let mut unrecognized_entries = 0;
        // A list of entries, in a form that can be consumed by remove_unchecked
        let mut all_entries = vec![];
        for entry in entries.iter() {
            match entry {
                Ok(entry) => {
                    assert_key(
                        entry.key_path(),
                        entry.key_type(),
                        &TestKeySpecifier1.arti_path().unwrap(),
                        &item_type,
                    );
                    recognized_entries += 1;
                    all_entries.push(RawKeystoreEntry::new(
                        build_raw_id_path(entry.key_path(), entry.key_type()),
                        primary_keystore_id.clone(),
                    ));
                }
                Err(u) => {
                    assert_eq!(u.entry().raw_id(), &unrecognized_entry_id);
                    unrecognized_entries += 1;
                    all_entries.push(u.entry().into());
                }
            }
        }
        assert_eq!(recognized_entries, 1);
        assert_eq!(unrecognized_entries, 1);
        // Remove a recognized entry and an recognized one
        for entry in all_entries {
            mgr.remove_unchecked(&entry.raw_id().to_string(), entry.keystore_id())
                .unwrap();
        }
        // Check the keys have been removed
        let entries = mgr.list_by_id(&primary_keystore_id).unwrap();
        assert_eq!(entries.len(), 0);
    }
    /// Whether to generate a given item before running the `run_certificate_test`.
    #[cfg(feature = "experimental-api")]
    #[derive(Clone, Copy, Debug, PartialEq)]
    enum GenerateItem {
        Yes,
        No,
    }
    fn make_certificate(subject_key: &TestItem, signed_with: &TestItem) -> AlwaysValidCert {
        let subject_id = subject_key.meta.as_key().unwrap().item_id.clone();
        let signing_id = signed_with.meta.as_key().unwrap().item_id.clone();
        let meta = ItemMetadata::Cert(CertMetadata {
            subject_key_id: subject_id,
            signing_key_id: signing_id,
            retrieved_from: None,
            is_generated: true,
        });
        // Note: this is not really a cert for `subject_key` signed with the `signed_with`
        // key!. The two are `TestItem`s and not keys, so we can't really generate a real
        // cert from them. We can, however, pretend we did, for testing purposes.
        // Eventually we might want to rewrite these tests to use real items
        // (like the `ArtiNativeKeystore` tests)
        let mut rng = FakeEntropicRng(testing_rng());
        let keypair = ed25519::Keypair::generate(&mut rng);
        let encoded_cert = Ed25519Cert::constructor()
            .cert_type(tor_cert::CertType::IDENTITY_V_SIGNING)
            .expiration(SystemTime::get() + Duration::from_secs(180))
            .signing_key(keypair.public_key().into())
            .cert_key(CertifiedKey::Ed25519(keypair.public_key().into()))
            .encode_and_sign(&keypair)
            .unwrap();
        let test_cert = CertData::TorEd25519Cert(encoded_cert);
        AlwaysValidCert(TestItem {
            item: KeystoreItem::Cert(test_cert),
            meta,
        })
    }
    #[cfg(feature = "experimental-api")]
    macro_rules! run_certificate_test {
        (
            generate_subject_key = $generate_subject_key:expr,
            generate_signing_key = $generate_signing_key:expr,
            $($expected_err:tt)?
        ) => {{
            use GenerateItem::*;
            let mut rng = FakeEntropicRng(testing_rng());
            let mut builder = KeyMgrBuilder::default().primary_store(Keystore::new_boxed("keystore1"));
            builder
                .secondary_stores()
                .extend([Keystore::new_boxed("keystore2"), Keystore::new_boxed("keystore3")]);
            let mgr = builder.build().unwrap();
            let spec = crate::test_utils::TestCertSpecifier {
                subject_key_spec: TestDerivedKeySpecifier,
                denotator: "foo".into(),
            };
            if $generate_subject_key == Yes {
                let _ = mgr
                    .generate::<TestItem>(
                        &TestKeySpecifier1,
                        KeystoreSelector::Primary,
                        &mut rng,
                        false,
                    )
                    .unwrap();
            }
            if $generate_signing_key == Yes {
                let _ = mgr
                    .generate::<TestItem>(
                        &TestKeySpecifier2,
                        KeystoreSelector::Id(&KeystoreId::from_str("keystore2").unwrap()),
                        &mut rng,
                        false,
                    )
                    .unwrap();
            }
            let signing_key_spec = TestKeySpecifier2;
            let res = mgr
                .get_or_generate_key_and_cert::<TestItem, AlwaysValidCert>(
                    &spec,
                    &signing_key_spec,
                    &make_certificate,
                    KeystoreSelector::Primary,
                    &mut rng,
                );
            #[allow(unused_assignments)]
            #[allow(unused_mut)]
            let mut has_error = false;
            $(
                has_error = true;
                let err = res.clone().unwrap_err();
                assert!(
                    matches!(
                        err,
                        crate::Error::Corruption(KeystoreCorruptionError::$expected_err)
                    ),
                    "unexpected error: {err:?}",
                );
            )?
            if !has_error {
                let (key, cert) = res.unwrap();
                let expected_subj_key_id = if $generate_subject_key == Yes {
                    "generated_test_key"
                } else {
                    "generated_test_key"
                };
                assert_eq!(key.meta.item_id(), expected_subj_key_id);
                assert_eq!(
                    cert.0.meta.as_cert().unwrap().subject_key_id,
                    expected_subj_key_id
                );
                assert_eq!(
                    cert.0.meta.as_cert().unwrap().signing_key_id,
                    "generated_test_key"
                );
                assert_eq!(cert.0.meta.is_generated(), true);
            }
        }}
    }
    #[test]
    #[cfg(feature = "experimental-api")]
    #[rustfmt::skip] // preserve the layout for readability
    #[allow(clippy::cognitive_complexity)] // clippy seems confused here...
    fn get_certificate() {
        run_certificate_test!(
            generate_subject_key = No,
            generate_signing_key = No,
            MissingSigningKey
        );
        run_certificate_test!(
            generate_subject_key = Yes,
            generate_signing_key = No,
            MissingSigningKey
        );
        run_certificate_test!(
            generate_subject_key = No,
            generate_signing_key = Yes,
        );
        run_certificate_test!(
            generate_subject_key = Yes,
            generate_signing_key = Yes,
        );
    }
    #[test]
    #[cfg(feature = "experimental-api")]
    fn get_cert_entry() {
        let mut rng = FakeEntropicRng(testing_rng());
        let builder = KeyMgrBuilder::default().primary_store(Keystore::new_boxed("keystore1"));
        let mgr = builder.build().unwrap();
        // Generate the subject key
        let _ = mgr
            .generate::<TestItem>(
                &TestKeySpecifier1,
                KeystoreSelector::Primary,
                &mut rng,
                false,
            )
            .unwrap();
        // Generate the signing key
        let _ = mgr
            .generate::<TestItem>(
                &TestKeySpecifier2,
                KeystoreSelector::Primary,
                &mut rng,
                false,
            )
            .unwrap();
        // Generate multiple test certificates for the same subject key
        for cert_deno in 0..10 {
            let cert_spec = TestCertSpecifier {
                subject_key_spec: TestDerivedKeySpecifier,
                denotator: cert_deno.to_string(),
            };
            let res = mgr.get_or_generate_key_and_cert::<TestItem, AlwaysValidCert>(
                &cert_spec,
                &TestKeySpecifier2,
                &make_certificate,
                KeystoreSelector::Primary,
                &mut rng,
            );
            assert!(res.is_ok());
        }
        // Time to list all certs and retrieve them
        let any_pat = TestCertSpecifierPattern::new_any().arti_pattern().unwrap();
        // Ensure the pattern is what we expect it to be
        assert_eq!(
            any_pat,
            KeyPathPattern::Arti("test/simple_key+@*".to_string())
        );
        let certs = mgr.list_matching(&any_pat).unwrap();
        // We generated 10 certs, so there should be 10 matching entries
        assert_eq!(certs.len(), 10);
        // Ensure we collected all the expected paths
        let all_paths = certs
            .iter()
            .map(|entry| entry.key_path().arti().unwrap().as_str().to_string())
            .sorted()
            .collect::<Vec<_>>();
        let expected_paths = (0..10)
            .map(|i| format!("test/simple_key+@{i}"))
            .collect::<Vec<_>>();
        assert_eq!(all_paths, expected_paths);
        for entry in certs {
            let always_valid_cert = mgr
                .get_cert_entry::<TestCertSpecifier, TestItem, AlwaysValidCert>(
                    &entry,
                    &TestKeySpecifier2,
                )
                .unwrap();
            // Check that the cert was found...
            assert!(always_valid_cert.is_some());
        }
        /// A denotator for our expired cert specifier.
        const EXPIRED_DENO: &str = "expired";
        // Generate an invalid test certificate
        let cert_spec = TestCertSpecifier {
            subject_key_spec: TestDerivedKeySpecifier,
            denotator: EXPIRED_DENO.to_string(),
        };
        // Dummy metadata
        let meta = CertMetadata {
            subject_key_id: "foo".to_string(),
            signing_key_id: "bar".to_string(),
            retrieved_from: None,
            is_generated: false,
        };
        let test_cert =
            CertData::TorEd25519Cert(EncodedEd25519Cert::dangerously_from_bytes(b"foobar"));
        let cert = AlwaysExpiredCert(TestItem {
            item: KeystoreItem::Cert(test_cert),
            meta: ItemMetadata::Cert(meta),
        });
        let res = mgr.insert_cert::<TestItem, AlwaysExpiredCert>(
            cert,
            &cert_spec,
            KeystoreSelector::Primary,
        );
        assert!(res.is_ok());
        // Build a pattern for matching *only* the expired cert
        let pat = KeyPathPattern::Arti(format!("test/simple_key+@{EXPIRED_DENO}"));
        let certs = mgr.list_matching(&pat).unwrap();
        assert_eq!(certs.len(), 1);
        let entry = &certs[0];
        let err = mgr
            .get_cert_entry::<TestCertSpecifier, TestItem, AlwaysExpiredCert>(
                entry,
                &TestKeySpecifier2,
            )
            .unwrap_err();
        // Can't retrieve the cert because it's expired!
        assert!(
            matches!(err, Error::InvalidCert(InvalidCertError::TimeValidity(_))),
            "{err:?}"
        );
    }
}