//! Experimental support for vanguards.

//!

//! For more information, see the [vanguards spec].

//!

//! [vanguards spec]: https://spec.torproject.org/vanguards-spec/index.html.

pub mod config;

mod err;

mod set;

use std::sync::{Arc, RwLock, Weak};

use std::time::{Duration, SystemTime};

use futures::stream::BoxStream;

use futures::{FutureExt as _, future};

use futures::{StreamExt as _, select_biased};

use postage::stream::Stream as _;

use postage::watch;

use tor_rtcompat::SpawnExt as _;

use tor_async_utils::PostageWatchSenderExt as _;

use tor_config::ReconfigureError;

use tor_error::{error_report, internal, into_internal};

use tor_netdir::{DirEvent, NetDir, NetDirProvider, Timeliness};

use tor_persist::{DynStorageHandle, StateMgr};

use tor_relay_selection::RelaySelector;

use tor_rtcompat::Runtime;

use tracing::{debug, info, instrument};

use crate::{RetireCircuits, VanguardMode};

use set::VanguardSets;

use crate::VanguardConfig;

pub use config::VanguardParams;

pub use err::VanguardMgrError;

pub use set::Vanguard;

/// The key used for storing the vanguard sets to persistent storage using `StateMgr`.

const STORAGE_KEY: &str = "vanguards";

/// The vanguard manager.

pub struct VanguardMgr<R: Runtime> {

    /// The mutable state.

    inner: RwLock<Inner>,

    /// The runtime.

    runtime: R,

    /// The persistent storage handle, used for writing the vanguard sets to disk

    /// if full vanguards are enabled.

    storage: DynStorageHandle<VanguardSets>,

}

/// The mutable inner state of [`VanguardMgr`].

struct Inner {

    /// The current vanguard parameters.

    params: VanguardParams,

    /// Whether to use full, lite, or no vanguards.

    ///

    // TODO(#1382): we should derive the mode from the

    // vanguards-enabled and vanguards-hs-service consensus params.

    mode: VanguardMode,

    /// The L2 and L3 vanguards.

    ///

    /// The L3 vanguards are only used if we are running in

    /// [`Full`](VanguardMode::Full) vanguard mode.

    /// Otherwise, the L3 set is not populated, or read from.

    ///

    /// If [`Full`](VanguardMode::Full) vanguard mode is enabled,

    /// the vanguard sets will be persisted to disk whenever

    /// vanuards are rotated, added, or removed.

    ///

    /// The vanguard sets are updated and persisted to storage by

    /// [`update_vanguard_sets`](Inner::update_vanguard_sets).

    ///

    /// If the `VanguardSets` change while we are in "lite" mode,

    /// the changes will not *not* be written to storage.

    /// If we later switch to "full" vanguards, those previous changes still

    /// won't be persisted to storage: we only flush to storage if the

    /// [`VanguardSets`] change *while* we are in "full" mode

    /// (changing the [`VanguardMode`] does not constitute a change in the `VanguardSets`).

    //

    // TODO HS-VANGUARDS: the correct behaviour here might be to never switch back to lite mode

    // after enabling full vanguards. If we do that, persisting the vanguard sets will be simpler,

    // as we can just unconditionally flush to storage if the vanguard mode is switched to full.

    // Right now we can't do that, because we don't remember the "mode":

    // we derive it on the fly from `has_onion_svc` and the current `VanguardParams`.

    //

    ///

    /// This is initialized with the vanguard sets read from the vanguard state file,

    /// if the file exists, or with a [`Default`] `VanguardSets`, if it doesn't.

    ///

    /// Note: The `VanguardSets` are read from the vanguard state file

    /// even if full vanguards are not enabled. They are *not*, however, written

    /// to the state file unless full vanguards are in use.

    vanguard_sets: VanguardSets,

    /// Whether we're running an onion service.

    ///

    // TODO(#1382): This should be used for deciding whether to use the `vanguards_hs_service` or the

    // `vanguards_enabled` [`NetParameter`](tor_netdir::params::NetParameters).

    #[allow(unused)]

    has_onion_svc: bool,

    /// A channel for sending VanguardConfig changes to the vanguard maintenance task.

    config_tx: watch::Sender<VanguardConfig>,

}

/// Whether the [`VanguardMgr::maintain_vanguard_sets`] task

/// should continue running or shut down.

///

/// Returned from [`VanguardMgr::run_once`].

#[derive(Copy, Clone, Debug)]

enum ShutdownStatus {

    /// Continue calling `run_once`.

    Continue,

    /// The `VanguardMgr` was dropped, terminate the task.

    Terminate,

}

impl<R: Runtime> VanguardMgr<R> {

    /// Create a new `VanguardMgr`.

    ///

    /// The `state_mgr` handle is used for persisting the "vanguards-full" guard pools to disk.

    {

        // Note: we start out with default vanguard params, but we adjust them

        // as soon as we obtain a NetDir (see Self::run_once()).

                // Discard the now-expired the vanguards

            }

            None => {

                // Initially, all sets have a target size of 0.

                // This is OK because the target is only used for repopulating the vanguard sets,

                // and we can't repopulate the sets without a netdir.

                // The target gets adjusted once we obtain a netdir.

            }

        };

    /// Launch the vanguard pool management tasks.

    ///

    /// These run until the `VanguardMgr` is dropped.

    //

    // This spawns [`VanguardMgr::maintain_vanguard_sets`].

    #[instrument(level = "trace", skip_all)]

    {

            ))

    /// Replace the configuration in this `VanguardMgr` with the specified `config`.

        // TODO(#1382): abolish VanguardConfig and derive the mode from the VanguardParams

        // and has_onion_svc instead.

        //

        // TODO(#1382): update has_onion_svc if the new config enables onion svc usage

        //

        // Perhaps we should always escalate to Full if we start running an onion service,

        // but not decessarily downgrade to lite if we stop.

        // See <https://gitlab.torproject.org/tpo/core/arti/-/merge_requests/2083#note_3018173>

            // Wake up the maintenance task to replenish the vanguard pools.

        } else {

        }

    /// Return a [`Vanguard`] relay for use in the specified layer.

    ///

    /// The `relay_selector` must exclude the relays that would neighbor this vanguard

    /// in the path.

    ///

    /// Specifically, it should exclude

    ///   * the last relay in the path (the one immediately preceding the vanguard): the same relay

    ///     cannot be used in consecutive positions in the path (a relay won't let you extend the

    ///     circuit to itself).

    ///   * the penultimate relay of the path, if there is one: relays don't allow extending the

    ///     circuit to their previous hop

    ///

    /// If [`Full`](VanguardMode::Full) vanguards are in use, this function can be used

    /// for selecting both [`Layer2`](Layer::Layer2) and [`Layer3`](Layer::Layer3) vanguards.

    ///

    /// If [`Lite`](VanguardMode::Lite) vanguards are in use, this function can only be used

    /// for selecting [`Layer2`](Layer::Layer2) vanguards.

    /// It will return an error if a [`Layer3`](Layer::Layer3) is requested.

    ///

    /// Returns an error if vanguards are disabled.

    ///

    /// Returns a [`NoSuitableRelay`](VanguardMgrError::NoSuitableRelay) error

    /// if none of our vanguards satisfy the `layer` and `neighbor_exclusion` requirements.

    ///

    /// Returns a [`BootstrapRequired`](VanguardMgrError::BootstrapRequired) error

    /// if called before the vanguard manager has finished bootstrapping,

    /// or if all the vanguards have become unusable

    /// (by expiring or no longer being listed in the consensus)

    /// and we are unable to replenish them.

    ///

    ///  ### Example

    ///

    ///  If the partially built path is of the form `G - L2` and we are selecting the L3 vanguard,

    ///  the `RelayExclusion` should contain `G` and `L2` (to prevent building a path of the form

    ///  `G - L2 - G`, or `G - L2 - L2`).

    ///

    ///  If the path only contains the L1 guard (`G`), then the `RelayExclusion` should only

    ///  exclude `G`.

        use VanguardMode::*;

        // All our vanguard sets are empty. This means select_vanguards() was called before

        // maintain_vanguard_sets() managed to obtain a netdir and populate the vanguard sets,

        // or all the vanguards have become unusable and we have been unable to replenish them.

                (Layer::Layer3, Full) => {

                }

                _ => {

                }

            };

    /// The vanguard set management task.

    ///

    /// This is a background task that:

    /// * removes vanguards from the L2 and L3 vanguard sets when they expire

    /// * ensures the vanguard sets are repopulated with new vanguards

    ///   when the number of vanguards drops below a certain threshold

    /// * handles `NetDir` changes, updating the vanguard set sizes as needed

            None => {

            }

        };

        loop {

            )

            {

                Ok(ShutdownStatus::Terminate) => {

                }

                }

            }

        }

    /// Wait until a vanguard expires or until there is a new [`NetDir`].

    ///

    /// This populates the L2 and L3 vanguard sets,

    /// and rotates the vanguards when their lifetime expires.

    ///

    /// Note: the L3 set is only populated with vanguards if

    /// [`Full`](VanguardMode::Full) vanguards are enabled.

        };

        // A future that sleeps until the next vanguard expires

            } else {

            }

            },

                    // If we have a NetDir, replenish the vanguard sets that don't have enough vanguards.

                    //

                    // For example, if the config change enables full vanguards for the first time,

                    // this will cause the L3 vanguard set to be populated.

            },

                // A vanguard expired, time to run the cleanup

            },

        }

    /// Return a timely `NetDir`, if one is available.

    ///

    /// Returns `None` if no directory information is available.

        use tor_netdir::Error as NetDirError;

        }

    /// Rotate the vanguards that have expired,

    /// returning how long until the next vanguard will expire,

    /// or `None` if there are no vanguards in any of our sets.

            // If we have a NetDir, replenish the vanguard sets that don't have enough vanguards.

            // Both vanguard sets are empty

        };

    /// Get the current [`VanguardMode`].

}

impl Inner {

    /// Update the vanguard sets, handling any potential vanguard parameter changes.

    ///

    /// This updates the [`VanguardSets`]s based on the [`VanguardParams`]

    /// derived from the new `NetDir`, replenishing the sets if necessary.

    ///

    /// NOTE: if the new `VanguardParams` specify different lifetime ranges

    /// than the previous `VanguardParams`, the new lifetime requirements only

    /// apply to newly selected vanguards. They are **not** retroactively applied

    /// to our existing vanguards.

    //

    // TODO(#1352): we might want to revisit this decision.

    // We could, for example, adjust the lifetime of our existing vanguards

    // to comply with the new lifetime requirements.

        // Update our params with the new values.

        // If we loaded some vanguards from persistent storage but we still need more,

        // we select them here.

        //

        // If full vanguards are not enabled and we started with an empty (default)

        // vanguard set, we populate the sets here.

        //

        // If we have already populated the vanguard sets in a previous iteration,

        // this will ensure they have enough vanguards.

        // Flush the vanguard sets to disk.

    /// Update our vanguard params.

    /// Flush the vanguard sets to storage, if the mode is "vanguards-full".

            VanguardMode::Full => {

            }

        }

}

#[cfg(any(test, feature = "testing"))]

use {

    tor_config::ExplicitOrAuto, tor_netdir::testprovider::TestNetDirProvider,

    tor_persist::TestingStateMgr, tor_rtmock::MockRuntime,

};

/// Helpers for tests involving vanguards

#[cfg(any(test, feature = "testing"))]

impl VanguardMgr<MockRuntime> {

    /// Create a new VanguardMgr for testing.

        // TODO(#1382): has_onion_svc doesn't matter right now

    /// Wait until the vanguardmgr has populated its vanguard sets.

    ///

    /// Returns a [`TestNetDirProvider`] that can be used to notify

    /// the `VanguardMgr` of netdir changes.

        // Call set_netdir_and_notify to trigger an event

        // Wait until the vanguard mgr has finished handling the netdir event.

}

/// The vanguard layer.

#[derive(Debug, Clone, Copy, PartialEq)] //

#[derive(derive_more::Display)] //

#[non_exhaustive]

pub enum Layer {

    /// L2 vanguard.

    #[display("layer 2")]

    Layer2,

    /// L3 vanguard.

    #[display("layer 3")]

    Layer3,

}

#[cfg(test)]

mod test {

    // @@ begin test lint list maintained by maint/add_warning @@

    #![allow(clippy::bool_assert_comparison)]

    #![allow(clippy::clone_on_copy)]

    #![allow(clippy::dbg_macro)]

    #![allow(clippy::mixed_attributes_style)]

    #![allow(clippy::print_stderr)]

    #![allow(clippy::print_stdout)]

    #![allow(clippy::single_char_pattern)]

    #![allow(clippy::unwrap_used)]

    #![allow(clippy::unchecked_time_subtraction)]

    #![allow(clippy::useless_vec)]

    #![allow(clippy::needless_pass_by_value)]

    //! <!-- @@ end test lint list maintained by maint/add_warning @@ -->

    use std::{fmt, time};

    use set::TimeBoundVanguard;

    use tor_config::ExplicitOrAuto;

    use tor_relay_selection::RelayExclusion;

    use super::*;

    use Layer::*;

    use tor_basic_utils::test_rng::testing_rng;

    use tor_linkspec::{HasRelayIds, RelayIds};

    use tor_netdir::{

        testnet::{self, construct_custom_netdir_with_params},

        testprovider::TestNetDirProvider,

    };

    use tor_persist::FsStateMgr;

    use tor_rtmock::MockRuntime;

    use itertools::Itertools;

    /// Enable lite vanguards for onion services.

    const ENABLE_LITE_VANGUARDS: [(&str, i32); 1] = [("vanguards-hs-service", 1)];

    /// Enable full vanguards for hidden services.

    const ENABLE_FULL_VANGUARDS: [(&str, i32); 1] = [("vanguards-hs-service", 2)];

    /// A valid vanguard state file.

    const VANGUARDS_JSON: &str = include_str!("../testdata/vanguards.json");

    /// A invalid vanguard state file.

    const INVALID_VANGUARDS_JSON: &str = include_str!("../testdata/vanguards_invalid.json");

    /// Create the `StateMgr`, populating the vanguards.json state file with the specified JSON string.

    fn state_dir_with_vanguards(vanguards_json: &str) -> (FsStateMgr, tempfile::TempDir) {

        let dir = tempfile::TempDir::new().unwrap();

        std::fs::create_dir_all(dir.path().join("state")).unwrap();

        std::fs::write(dir.path().join("state/vanguards.json"), vanguards_json).unwrap();

        let statemgr = FsStateMgr::from_path_and_mistrust(

            dir.path(),

            &fs_mistrust::Mistrust::new_dangerously_trust_everyone(),

        )

        .unwrap();

        (statemgr, dir)

    }

    impl fmt::Debug for Vanguard<'_> {

        fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {

            f.debug_struct("Vanguard").finish()

        }

    }

    impl Inner {

        /// Return the L2 vanguard set.

        pub(super) fn l2_vanguards(&self) -> &Vec<TimeBoundVanguard> {

            self.vanguard_sets.l2_vanguards()

        }

        /// Return the L3 vanguard set.

        pub(super) fn l3_vanguards(&self) -> &Vec<TimeBoundVanguard> {

            self.vanguard_sets.l3_vanguards()

        }

    }

    /// Return a maximally permissive RelaySelector for a vanguard.

    fn permissive_selector() -> RelaySelector<'static> {

        RelaySelector::new(

            tor_relay_selection::RelayUsage::vanguard(),

            RelayExclusion::no_relays_excluded(),

        )

    }

    /// Look up the vanguard in the specified VanguardSet.

    fn find_in_set<R: Runtime>(

        relay_ids: &RelayIds,

        mgr: &VanguardMgr<R>,

        layer: Layer,

    ) -> Option<TimeBoundVanguard> {

        let inner = mgr.inner.read().unwrap();

        let vanguards = match layer {

            Layer2 => inner.l2_vanguards(),

            Layer3 => inner.l3_vanguards(),

        };

        // Look up the TimeBoundVanguard that corresponds to this Vanguard,

        // and figure out its expiry.

        vanguards.iter().find(|v| v.id == *relay_ids).cloned()

    }

    /// Get the total number of vanguard entries (L2 + L3).

    fn vanguard_count<R: Runtime>(mgr: &VanguardMgr<R>) -> usize {

        let inner = mgr.inner.read().unwrap();

        inner.l2_vanguards().len() + inner.l3_vanguards().len()

    }

    /// Return a `Duration` representing how long until this vanguard expires.

    fn duration_until_expiry<R: Runtime>(

        relay_ids: &RelayIds,

        mgr: &VanguardMgr<R>,

        runtime: &R,

        layer: Layer,

    ) -> Duration {

        // Look up the TimeBoundVanguard that corresponds to this Vanguard,

        // and figure out its expiry.

        let vanguard = find_in_set(relay_ids, mgr, layer).unwrap();

        vanguard

            .when

            .duration_since(runtime.wallclock())

            .unwrap_or_default()

    }

    /// Assert the lifetime of the specified `vanguard` is within the bounds of its `layer`.

    fn assert_expiry_in_bounds<R: Runtime>(

        vanguard: &Vanguard<'_>,

        mgr: &VanguardMgr<R>,

        runtime: &R,

        params: &VanguardParams,

        layer: Layer,

    ) {

        let (min, max) = match layer {

            Layer2 => (params.l2_lifetime_min(), params.l2_lifetime_max()),

            Layer3 => (params.l3_lifetime_min(), params.l3_lifetime_max()),

        };

        let vanguard = RelayIds::from_relay_ids(vanguard.relay());

        // This is not exactly the lifetime of the vanguard,

        // but rather the time left until it expires (but it's close enough for our purposes).

        let lifetime = duration_until_expiry(&vanguard, mgr, runtime, layer);

        assert!(

            lifetime >= min && lifetime <= max,

            "lifetime {lifetime:?} not between {min:?} and {max:?}",

        );

    }

    /// Assert that the vanguard manager's pools are empty.

    fn assert_sets_empty<R: Runtime>(vanguardmgr: &VanguardMgr<R>) {

        let inner = vanguardmgr.inner.read().unwrap();

        // The sets are initially empty, and the targets are set to 0

        assert_eq!(inner.vanguard_sets.l2_vanguards_deficit(), 0);

        assert_eq!(inner.vanguard_sets.l3_vanguards_deficit(), 0);

        assert_eq!(vanguard_count(vanguardmgr), 0);

    }

    /// Assert that the vanguard manager's pools have been filled.

    fn assert_sets_filled<R: Runtime>(vanguardmgr: &VanguardMgr<R>, params: &VanguardParams) {

        let inner = vanguardmgr.inner.read().unwrap();

        let l2_pool_size = params.l2_pool_size();

        // The sets are initially empty

        assert_eq!(inner.vanguard_sets.l2_vanguards_deficit(), 0);

        if inner.mode == VanguardMode::Full {

            assert_eq!(inner.vanguard_sets.l3_vanguards_deficit(), 0);

            let l3_pool_size = params.l3_pool_size();

            assert_eq!(vanguard_count(vanguardmgr), l2_pool_size + l3_pool_size);

        }

    }

    /// Assert the target size of the specified vanguard set matches the target from `params`.

    fn assert_set_vanguards_targets_match_params<R: Runtime>(

        mgr: &VanguardMgr<R>,

        params: &VanguardParams,

    ) {

        let inner = mgr.inner.read().unwrap();

        assert_eq!(

            inner.vanguard_sets.l2_vanguards_target(),

            params.l2_pool_size()

        );

        if inner.mode == VanguardMode::Full {

            assert_eq!(

                inner.vanguard_sets.l3_vanguards_target(),

                params.l3_pool_size()

            );

        }

    }

    #[test]

    fn full_vanguards_disabled() {

        MockRuntime::test_with_various(|rt| async move {

            let vanguardmgr = VanguardMgr::new_testing(&rt, VanguardMode::Lite).unwrap();

            let netdir = testnet::construct_netdir().unwrap_if_sufficient().unwrap();

            let mut rng = testing_rng();

            // Wait until the vanguard manager has bootstrapped

            // (otherwise we'll get a BootstrapRequired error)

            let _netdir_provider = vanguardmgr.init_vanguard_sets(&netdir).await.unwrap();

            // Cannot select an L3 vanguard when running in "Lite" mode.

            let err = vanguardmgr

                .select_vanguard(&mut rng, &netdir, Layer3, &permissive_selector())

                .unwrap_err();

            assert!(

                matches!(

                    err,

                    VanguardMgrError::LayerNotSupported {

                        layer: Layer::Layer3,

                        mode: VanguardMode::Lite

                    }

                ),

                "{err}"

            );

        });

    }

    #[test]

    fn background_task_not_spawned() {

        MockRuntime::test_with_various(|rt| async move {

            let vanguardmgr = VanguardMgr::new_testing(&rt, VanguardMode::Lite).unwrap();

            let netdir = testnet::construct_netdir().unwrap_if_sufficient().unwrap();

            let mut rng = testing_rng();

            // The sets are initially empty

            assert_sets_empty(&vanguardmgr);

            // VanguardMgr::launch_background tasks was not called, so select_vanguard will return

            // an error (because the vanguard sets are empty)

            let err = vanguardmgr

                .select_vanguard(&mut rng, &netdir, Layer2, &permissive_selector())

                .unwrap_err();

            assert!(

                matches!(

                    err,

                    VanguardMgrError::BootstrapRequired {

                        action: "select vanguard"

                    }

                ),

                "{err:?}"

            );

        });

    }

    #[test]

    fn select_vanguards() {

        MockRuntime::test_with_various(|rt| async move {

            let vanguardmgr = VanguardMgr::new_testing(&rt, VanguardMode::Full).unwrap();

            let netdir = testnet::construct_netdir().unwrap_if_sufficient().unwrap();

            let params = VanguardParams::try_from(netdir.params()).unwrap();

            let mut rng = testing_rng();

            // The sets are initially empty

            assert_sets_empty(&vanguardmgr);

            // Wait until the vanguard manager has bootstrapped

            let _netdir_provider = vanguardmgr.init_vanguard_sets(&netdir).await.unwrap();

            assert_sets_filled(&vanguardmgr, &params);

            let vanguard1 = vanguardmgr

                .select_vanguard(&mut rng, &netdir, Layer2, &permissive_selector())

                .unwrap();

            assert_expiry_in_bounds(&vanguard1, &vanguardmgr, &rt, &params, Layer2);

            let exclusion = RelayExclusion::exclude_identities(

                vanguard1

                    .relay()

                    .identities()

                    .map(|id| id.to_owned())

                    .collect(),

            );

            let selector =

                RelaySelector::new(tor_relay_selection::RelayUsage::vanguard(), exclusion);

            let vanguard2 = vanguardmgr

                .select_vanguard(&mut rng, &netdir, Layer3, &selector)

                .unwrap();

            assert_expiry_in_bounds(&vanguard2, &vanguardmgr, &rt, &params, Layer3);

            // Ensure we didn't select the same vanguard twice

            assert_ne!(

                vanguard1.relay().identities().collect_vec(),

                vanguard2.relay().identities().collect_vec()

            );

        });

    }

    /// Override the vanguard params from the netdir, returning the new VanguardParams.

    ///

    /// This also waits until the vanguard manager has had a chance to process the changes.

    async fn install_new_params(

        rt: &MockRuntime,

        netdir_provider: &TestNetDirProvider,

        params: impl IntoIterator<Item = (&str, i32)>,

    ) -> VanguardParams {

        let new_netdir = testnet::construct_custom_netdir_with_params(|_, _, _| {}, params, None)

            .unwrap()

            .unwrap_if_sufficient()

            .unwrap();

        let new_params = VanguardParams::try_from(new_netdir.params()).unwrap();

        netdir_provider.set_netdir_and_notify(new_netdir).await;

        // Wait until the vanguard mgr has finished handling the new netdir.

        rt.progress_until_stalled().await;

        new_params

    }

    /// Switch the vanguard "mode" of the VanguardMgr to `mode`,

    /// by setting the vanguards-hs-service parameter.

    //

    // TODO(#1382): use this instead of switch_hs_mode_config.

    #[allow(unused)]

    async fn switch_hs_mode(

        rt: &MockRuntime,

        vanguardmgr: &VanguardMgr<MockRuntime>,

        netdir_provider: &TestNetDirProvider,

        mode: VanguardMode,

    ) {

        use VanguardMode::*;

        let _params = match mode {

            Lite => install_new_params(rt, netdir_provider, ENABLE_LITE_VANGUARDS).await,

            Full => install_new_params(rt, netdir_provider, ENABLE_FULL_VANGUARDS).await,

            Disabled => panic!("cannot disable vanguards in the vanguard tests!"),

        };

        assert_eq!(vanguardmgr.mode(), mode);

    }

    /// Switch the vanguard "mode" of the VanguardMgr to `mode`,

    /// by calling `VanguardMgr::reconfigure`.

    fn switch_hs_mode_config(vanguardmgr: &VanguardMgr<MockRuntime>, mode: VanguardMode) {

        let _ = vanguardmgr

            .reconfigure(&VanguardConfig {

                mode: ExplicitOrAuto::Explicit(mode),

            })

            .unwrap();

        assert_eq!(vanguardmgr.mode(), mode);

    }

    /// Use a new NetDir that excludes one of our L2 vanguards

    async fn install_netdir_excluding_vanguard<'a>(

        runtime: &MockRuntime,

        vanguard: &Vanguard<'_>,

        params: impl IntoIterator<Item = (&'a str, i32)>,

        netdir_provider: &TestNetDirProvider,

    ) -> NetDir {

        let new_netdir = construct_custom_netdir_with_params(

            |_idx, bld, _| {

                let md_so_far = bld.md.testing_md().unwrap();

                if md_so_far.ed25519_id() == vanguard.relay().id() {

                    bld.omit_rs = true;

                }

            },

            params,

            None,

        )

        .unwrap()

        .unwrap_if_sufficient()

        .unwrap();

        netdir_provider

            .set_netdir_and_notify(new_netdir.clone())

            .await;

        // Wait until the vanguard mgr has finished handling the new netdir.

        runtime.progress_until_stalled().await;

        new_netdir

    }

    #[test]

    fn override_vanguard_set_size() {

        MockRuntime::test_with_various(|rt| async move {

            let vanguardmgr = VanguardMgr::new_testing(&rt, VanguardMode::Lite).unwrap();

            let netdir = testnet::construct_netdir().unwrap_if_sufficient().unwrap();

            // Wait until the vanguard manager has bootstrapped

            let netdir_provider = vanguardmgr.init_vanguard_sets(&netdir).await.unwrap();

            let params = VanguardParams::try_from(netdir.params()).unwrap();

            let old_size = params.l2_pool_size();

            assert_set_vanguards_targets_match_params(&vanguardmgr, &params);

            const PARAMS: [[(&str, i32); 2]; 2] = [

                [("guard-hs-l2-number", 1), ("guard-hs-l3-number", 10)],

                [("guard-hs-l2-number", 10), ("guard-hs-l3-number", 10)],

            ];

            for params in PARAMS {

                let new_params = install_new_params(&rt, &netdir_provider, params).await;

                // Ensure the target size was updated.

                assert_set_vanguards_targets_match_params(&vanguardmgr, &new_params);

                {

                    let inner = vanguardmgr.inner.read().unwrap();

                    let l2_vanguards = inner.l2_vanguards();

                    let l3_vanguards = inner.l3_vanguards();

                    let new_l2_size = params[0].1 as usize;

                    if new_l2_size < old_size {

                        // The actual size of the set hasn't changed: it's OK to have more vanguards than

                        // needed in the set (they extraneous ones will eventually expire).

                        assert_eq!(l2_vanguards.len(), old_size);

                    } else {

                        // The new size is greater, so we have more L2 vanguards now.

                        assert_eq!(l2_vanguards.len(), new_l2_size);

                    }

                    // There are no L3 vanguards because full vanguards are not in use.

                    assert_eq!(l3_vanguards.len(), 0);

                }

            }

        });

    }

    #[test]

    fn expire_vanguards() {

        MockRuntime::test_with_various(|rt| async move {

            let vanguardmgr = VanguardMgr::new_testing(&rt, VanguardMode::Lite).unwrap();

            let netdir = testnet::construct_netdir().unwrap_if_sufficient().unwrap();

            let params = VanguardParams::try_from(netdir.params()).unwrap();

            let initial_l2_number = params.l2_pool_size();

            // Wait until the vanguard manager has bootstrapped

            let netdir_provider = vanguardmgr.init_vanguard_sets(&netdir).await.unwrap();

            assert_eq!(vanguard_count(&vanguardmgr), params.l2_pool_size());

            // Find the RelayIds of the vanguard that is due to expire next

            let vanguard_id = {

                let inner = vanguardmgr.inner.read().unwrap();

                let next_expiry = inner.vanguard_sets.next_expiry().unwrap();

                inner

                    .l2_vanguards()

                    .iter()

                    .find(|v| v.when == next_expiry)

                    .cloned()

                    .unwrap()

                    .id

            };

            const FEWER_VANGUARDS_PARAM: [(&str, i32); 1] = [("guard-hs-l2-number", 1)];

            // Set the number of L2 vanguards to a lower value to ensure the vanguard that is about

            // to expire is not replaced. This allows us to test that it has indeed expired

            // (we can't simply check that the relay is no longer is the set,

            // because it's possible for the set to get replenished with the same relay).

            let new_params = install_new_params(&rt, &netdir_provider, FEWER_VANGUARDS_PARAM).await;

            // The vanguard has not expired yet.

            let timebound_vanguard = find_in_set(&vanguard_id, &vanguardmgr, Layer2);

            assert!(timebound_vanguard.is_some());

            assert_eq!(vanguard_count(&vanguardmgr), initial_l2_number);

            let lifetime = duration_until_expiry(&vanguard_id, &vanguardmgr, &rt, Layer2);

            // Wait until this vanguard expires

            rt.advance_by(lifetime).await.unwrap();

            rt.progress_until_stalled().await;

            let timebound_vanguard = find_in_set(&vanguard_id, &vanguardmgr, Layer2);

            // The vanguard expired, but was not replaced.

            assert!(timebound_vanguard.is_none());

            assert_eq!(vanguard_count(&vanguardmgr), initial_l2_number - 1);

            // Wait until more vanguards expire. This will reduce the set size to 1

            // (the new target size we set by overriding the params).

            for _ in 0..initial_l2_number - 1 {

                let vanguard_id = {

                    let inner = vanguardmgr.inner.read().unwrap();

                    let next_expiry = inner.vanguard_sets.next_expiry().unwrap();

                    inner

                        .l2_vanguards()

                        .iter()

                        .find(|v| v.when == next_expiry)

                        .cloned()

                        .unwrap()

                        .id

                };

                let lifetime = duration_until_expiry(&vanguard_id, &vanguardmgr, &rt, Layer2);

                rt.advance_by(lifetime).await.unwrap();

                rt.progress_until_stalled().await;

            }

            assert_eq!(vanguard_count(&vanguardmgr), new_params.l2_pool_size());

            // Update the L2 set size again, to force the vanguard manager to replenish the L2 set.

            const MORE_VANGUARDS_PARAM: [(&str, i32); 1] = [("guard-hs-l2-number", 5)];

            // Set the number of L2 vanguards to a lower value to ensure the vanguard that is about

            // to expire is not replaced. This allows us to test that it has indeed expired

            // (we can't simply check that the relay is no longer is the set,

            // because it's possible for the set to get replenished with the same relay).

            let new_params = install_new_params(&rt, &netdir_provider, MORE_VANGUARDS_PARAM).await;

            // Check that we replaced the expired vanguard with a new one:

            assert_eq!(vanguard_count(&vanguardmgr), new_params.l2_pool_size());

            {

                let inner = vanguardmgr.inner.read().unwrap();

                let l2_count = inner.l2_vanguards().len();

                assert_eq!(l2_count, new_params.l2_pool_size());

            }

        });

    }

    #[test]

    fn full_vanguards_persistence() {

        MockRuntime::test_with_various(|rt| async move {

            let vanguardmgr = VanguardMgr::new_testing(&rt, VanguardMode::Lite).unwrap();

            let netdir =

                construct_custom_netdir_with_params(|_, _, _| {}, ENABLE_LITE_VANGUARDS, None)

                    .unwrap()

                    .unwrap_if_sufficient()

                    .unwrap();

            let netdir_provider = vanguardmgr.init_vanguard_sets(&netdir).await.unwrap();

            // Full vanguards are not enabled, so we don't expect anything to be written

            // to persistent storage.

            assert_eq!(vanguardmgr.mode(), VanguardMode::Lite);

            assert!(vanguardmgr.storage.load().unwrap().is_none());

            let mut rng = testing_rng();

            assert!(

                vanguardmgr

                    .select_vanguard(&mut rng, &netdir, Layer3, &permissive_selector())

                    .is_err()

            );

            // Enable full vanguards again.

            //

            // We expect VanguardMgr to populate the L3 set, and write the VanguardSets to storage.

            switch_hs_mode_config(&vanguardmgr, VanguardMode::Full);

            rt.progress_until_stalled().await;

            let vanguard_sets_orig = vanguardmgr.storage.load().unwrap();

            assert!(

                vanguardmgr

                    .select_vanguard(&mut rng, &netdir, Layer3, &permissive_selector())

                    .is_ok()

            );

            // Switch to lite vanguards.

            switch_hs_mode_config(&vanguardmgr, VanguardMode::Lite);

            // The vanguard sets should not change when switching between lite and full vanguards.

            assert_eq!(vanguard_sets_orig, vanguardmgr.storage.load().unwrap());

            switch_hs_mode_config(&vanguardmgr, VanguardMode::Full);

            assert_eq!(vanguard_sets_orig, vanguardmgr.storage.load().unwrap());

            // TODO HS-VANGUARDS: we may want to disable the ability to switch back to lite

            // vanguards.

            // Switch to lite vanguards and remove a relay from the consensus.

            // The relay should *not* be persisted to storage until we switch back to full

            // vanguards.

            switch_hs_mode_config(&vanguardmgr, VanguardMode::Lite);

            let mut rng = testing_rng();

            let excluded_vanguard = vanguardmgr

                .select_vanguard(&mut rng, &netdir, Layer2, &permissive_selector())

                .unwrap();

            let _ = install_netdir_excluding_vanguard(

                &rt,

                &excluded_vanguard,

                ENABLE_LITE_VANGUARDS,

                &netdir_provider,

            )

            .await;

            // The vanguard sets from storage haven't changed, because we are in "lite" mode.

            assert_eq!(vanguard_sets_orig, vanguardmgr.storage.load().unwrap());

            let _ = install_netdir_excluding_vanguard(

                &rt,

                &excluded_vanguard,

                ENABLE_FULL_VANGUARDS,

                &netdir_provider,

            )

            .await;

        });

    }

    #[test]

    fn load_from_state_file() {

        MockRuntime::test_with_various(|rt| async move {

            // Set the wallclock to a time when some of the stored vanguards are still valid.

            let now = time::UNIX_EPOCH + Duration::from_secs(1610000000);

            rt.jump_wallclock(now);

            let config = VanguardConfig {

                mode: ExplicitOrAuto::Explicit(VanguardMode::Full),

            };

            // The state file contains no vanguards

            let (statemgr, _dir) =

                state_dir_with_vanguards(r#"{ "l2_vanguards": [], "l3_vanguards": [] }"#);

            let vanguardmgr = VanguardMgr::new(&config, rt.clone(), statemgr, false).unwrap();

            {

                let inner = vanguardmgr.inner.read().unwrap();

                // The vanguard sets should be empty too

                assert!(inner.vanguard_sets.l2().is_empty());

                assert!(inner.vanguard_sets.l3().is_empty());

            }

            let (statemgr, _dir) = state_dir_with_vanguards(VANGUARDS_JSON);

            let vanguardmgr =

                Arc::new(VanguardMgr::new(&config, rt.clone(), statemgr, false).unwrap());

            let (initial_l2, initial_l3) = {

                let inner = vanguardmgr.inner.read().unwrap();

                let l2_vanguards = inner.vanguard_sets.l2_vanguards().clone();

                let l3_vanguards = inner.vanguard_sets.l3_vanguards().clone();

                // The sets actually contain 4 and 5 vanguards, respectively,

                // but the expired ones are discarded.

                assert_eq!(l2_vanguards.len(), 3);

                assert_eq!(l3_vanguards.len(), 2);

                // We don't know how many vanguards we're going to need

                // until we fetch the consensus.

                assert_eq!(inner.vanguard_sets.l2_vanguards_target(), 0);

                assert_eq!(inner.vanguard_sets.l3_vanguards_target(), 0);

                assert_eq!(inner.vanguard_sets.l2_vanguards_deficit(), 0);

                assert_eq!(inner.vanguard_sets.l3_vanguards_deficit(), 0);

                (l2_vanguards, l3_vanguards)

            };

            let netdir = testnet::construct_netdir().unwrap_if_sufficient().unwrap();

            let _netdir_provider = vanguardmgr.init_vanguard_sets(&netdir).await.unwrap();

            {

                let inner = vanguardmgr.inner.read().unwrap();

                let l2_vanguards = inner.vanguard_sets.l2_vanguards();

                let l3_vanguards = inner.vanguard_sets.l3_vanguards();

                // The sets were replenished with more vanguards

                assert_eq!(l2_vanguards.len(), 4);

                assert_eq!(l3_vanguards.len(), 8);

                // We now know we need 4 L2 vanguards and 8 L3 ones.

                assert_eq!(inner.vanguard_sets.l2_vanguards_target(), 4);

                assert_eq!(inner.vanguard_sets.l3_vanguards_target(), 8);

                assert_eq!(inner.vanguard_sets.l2_vanguards_deficit(), 0);

                assert_eq!(inner.vanguard_sets.l3_vanguards_deficit(), 0);

                // All of the vanguards read from the state file should still be in the sets.

                assert!(initial_l2.iter().all(|v| l2_vanguards.contains(v)));

                assert!(initial_l3.iter().all(|v| l3_vanguards.contains(v)));

            }

        });

    }

    #[test]

    fn invalid_state_file() {

        MockRuntime::test_with_various(|rt| async move {

            let config = VanguardConfig {

                mode: ExplicitOrAuto::Explicit(VanguardMode::Full),

            };

            let (statemgr, _dir) = state_dir_with_vanguards(INVALID_VANGUARDS_JSON);

            let res = VanguardMgr::new(&config, rt.clone(), statemgr, false);

            assert!(matches!(res, Err(VanguardMgrError::State(_))));

        });

    }

}