1
//! Access to the database schema.
2
//!
3
//! This module is not intended to provide a high-level ORM, instead it serves
4
//! the purpose of initializing and upgrading the database, if necessary.
5
//!
6
//! # Synchronous or Asynchronous?
7
//!
8
//! The question on whether the database and access to it shall be synchronous
9
//! or asynchronous has been fairly long debate that eventually got settled
10
//! after realizing that an asynchronous approach does not work.  This comment
11
//! should serve as a reminder for future devs, wondering why we use certain
12
//! synchronous primitives in an otherwise asynchronous codebase.
13
//!
14
//! Early on, it was clear that we would need some sort of connection pool,
15
//! primarily for two reasons:
16
//! 1. Performing frequent open and close calls in every task would be costly.
17
//! 2. Sharing a single connection object with a Mutex would be a waste
18
//!
19
//! Because the application itself is primarily asynchronous, we decided to go
20
//! with an asynchronous connection pool as well, leading to the choose of
21
//! `deadpool` initially.
22
//!
23
//! However, soon thereafter, problems with `deadpool` became evident.  Those
24
//! problems mostly stemmed from the synchronous nature of SQLite itself.  In our
25
//! case, this problem was initially triggered by figuring out a way to solve
26
//! `SQLITE_BUSY` handling.  In the end, we decided to settle upon the following
27
//! approach: Set `PRAGMA busy_timeout` to a certain value and create write
28
//! transactions with `BEGIN EXCLUSIVE`.  This way, SQLite would try to obtain
29
//! a write transaction for `busy_timeout` milliseconds by blocking the current
30
//! thread.  Due to this blocking, async no longer made any sense and was in
31
//! fact quite counter-productive because those potential sleep could screw a
32
//! lot of things up, which became very evident while trying to test this.
33
//!
34
//! Besides, throughout refactoring the code base, we realized that, even while
35
//! still using `deadpool`, the actual "asynchronous" calls interfacing with the
36
//! database became smaller and smaller.  In the end, the asynchronous code just
37
//! involved parts of obtaining a connection and creating a transaction,
38
//! eventually resulting in a calling a synchronous function taking the
39
//! transaction handle to perform the lion's share of the operation.
40

            
41
// TODO DIRMIRROR: This could benefit from methods by wrapping the pool into a
42
// custom type.
43

            
44
use std::{
45
    collections::HashSet,
46
    fmt::Display,
47
    io::{Cursor, Write},
48
    marker::PhantomData,
49
    num::NonZero,
50
    ops::{Add, Sub},
51
    path::Path,
52
    rc::Rc,
53
    time::{Duration, SystemTime},
54
};
55

            
56
use digest::Digest;
57
use educe::Educe;
58
use flate2::write::{DeflateEncoder, GzEncoder};
59
use r2d2::Pool;
60
use r2d2_sqlite::SqliteConnectionManager;
61
use rand::Rng;
62
use rusqlite::{
63
    ToSql, Transaction, TransactionBehavior, named_params, params,
64
    types::{FromSql, FromSqlError, FromSqlResult, ToSqlOutput, Value, ValueRef},
65
};
66
use saturating_time::SaturatingTime;
67
use tor_basic_utils::RngExt;
68
use tor_error::{internal, into_internal};
69
use tor_netdoc::doc::{authcert::AuthCert, netstatus::ConsensusFlavor};
70

            
71
use crate::{
72
    err::DatabaseError,
73
    types::{FlavoredConsensusBody, FlavoredConsensusSignatures, FlavoredConsensusUnverified},
74
};
75

            
76
/// Version 1 of the database schema.
77
///
78
/// TODO DIRMIRROR: Before the release, figure out where to use rowid and where
79
/// to use docid.
80
const V1_SCHEMA: &str = include_str!("schema_v1.sql");
81

            
82
/// Global options set in every connection.
83
const GLOBAL_OPTIONS: &str = sql!(
84
    "
85
PRAGMA journal_mode=WAL;
86
PRAGMA foreign_keys=ON;
87
PRAGMA busy_timeout=1000;
88
"
89
);
90

            
91
/// Convenience macro for implementing a hash type in a rusqlite compatible fashion.
92
///
93
/// This macro accepts the following parameters:
94
/// 1. `name` for specifying an identifier of the type, such as [`Sha256`].
95
/// 2. `algo` for specifying the type from the rust-crypto [`digest`] ecosystem,
96
///    such as [`tor_llcrypto::d::Sha256`].
97
/// 3. The size in bytes of the hash output, such as `32` for [`Sha256`].
98
///     * Unfortunately, we cannot use something like [`Digest::output_size()`]
99
///       because it is not a constant.
100
///
101
/// It generates a struct with `name` as the identifier, which implements the
102
/// following methods:
103
/// * `digest` for wrapping around [`Digest::digest()`].
104
///
105
/// It also implements the following traits:
106
/// * [`Display`]
107
/// * [`FromSql`]
108
/// * [`ToSql`]
109
/// * [`PartialEq<&str>`] for base16 comparisons
110
/// * [`From<u8; $size>`]
111
macro_rules! impl_hash_wrapper {
112
    ($name:ident, $algo:ty, $size:literal) => {
113
        /// Database wrapper type for $name.
114
        ///
115
        /// Serves as a database friendly wrapper around [`tor_llcrypto::d`]
116
        /// with features such as SQL support.
117
        #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
118
        pub(crate) struct $name(pub [u8; $size]);
119

            
120
        impl $name {
121
            /// Computes the hash from arbitrary data.
122
562
            pub(crate) fn digest(data: &[u8]) -> Self {
123
562
                Self(<$algo>::digest(data).into())
124
562
            }
125
        }
126

            
127
        impl Display for $name {
128
            /// Formats the hash in uppercase hexadecimal.
129
2144
            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
130
2144
                write!(f, "{}", hex::encode_upper(self.0))
131
2144
            }
132
        }
133

            
134
        impl FromSql for $name {
135
195
            fn column_result(value: ValueRef<'_>) -> FromSqlResult<Self> {
136
                // We read the hash as a hexadecimal string from the database.
137
                // Convert it to binary data and check length afterwards.
138
195
                let data: [u8; $size] = value
139
195
                    .as_str()
140
195
                    .map(hex::decode)?
141
195
                    .map_err(|e| {
142
                        FromSqlError::Other(Box::new(tor_error::internal!(
143
                            "non hex data in database? {e}"
144
                        )))
145
                    })?
146
195
                    .try_into()
147
195
                    .map_err(|_| {
148
                        FromSqlError::Other(Box::new(tor_error::internal!(
149
                            "$name with invalid length in database?"
150
                        )))
151
                    })?;
152

            
153
195
                Ok(Self(data))
154
195
            }
155
        }
156

            
157
        impl ToSql for $name {
158
2121
            fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
159
                // Because Self is only constructed with FromSql and digest
160
                // data, it is safe to assume it is valid.
161
2121
                Ok(ToSqlOutput::from(self.to_string()))
162
2121
            }
163
        }
164

            
165
        impl From<$name> for Value {
166
22
            fn from(value: $name) -> Value {
167
22
                Value::Text(value.to_string())
168
22
            }
169
        }
170

            
171
        impl PartialEq<&str> for $name {
172
1
            fn eq(&self, other: &&str) -> bool {
173
1
                self.to_string() == other.to_uppercase()
174
1
            }
175
        }
176

            
177
        impl From<[u8; $size]> for $name {
178
783
            fn from(value: [u8; $size]) -> Self {
179
783
                Self(value)
180
783
            }
181
        }
182
    };
183
}
184

            
185
impl_hash_wrapper!(Sha1, tor_llcrypto::d::Sha1, 20);
186
impl_hash_wrapper!(Sha256, tor_llcrypto::d::Sha256, 32);
187
impl_hash_wrapper!(Sha3_256, tor_llcrypto::d::Sha3_256, 32);
188

            
189
/// The identifier for documents in the content-addressable cache.
190
///
191
/// Right now, this is a [`Sha256`] hash, but this may change in future.
192
pub(crate) type DocumentId = Sha256;
193

            
194
/// The supported content encodings.
195
#[derive(
196
    Debug, Clone, Copy, PartialEq, Eq, Hash, strum::EnumString, strum::Display, strum::EnumIter,
197
)]
198
#[strum(serialize_all = "kebab-case", ascii_case_insensitive)]
199
pub(crate) enum ContentEncoding {
200
    /// RFC2616 section 3.5.
201
    Identity,
202
    /// RFC2616 section 3.5.
203
    Deflate,
204
    /// RFC2616 section 3.5.
205
    Gzip,
206
    /// The zstandard compression algorithm (www.zstd.net).
207
    XZstd,
208
    /// The lzma compression algorithm with a "present" value no higher than 6.
209
    XTorLzma,
210
}
211

            
212
/// A wrapper around [`SystemTime`] with convenient features.
213
///
214
/// Please use this type throughout the crate internally, instead of
215
/// [`SystemTime`].
216
///
217
/// # Conversion
218
///
219
/// This type can be safely converted from and into a [`SystemTime`], because
220
/// it is just a wrapper type.
221
///
222
/// # Saturating Arithmetic
223
///
224
/// This type implements [`Add`] and [`Sub`] for [`Duration`] and [`Timestamp`]
225
/// ([`Sub`] only) using saturating arithmetic from the [`saturating_time`]
226
/// crate.  It means that addition and subtraction can be safely performed
227
/// without the potential risk of an unexpected panic, instead wrapping to
228
/// a local maximum/minimum or [`Duration::ZERO`] depending on the type.
229
///
230
/// Note that we don't provide a saturating version of [`Duration`], so addition
231
/// or subtraction of two [`Duration`]s still needs care to avoid panics.
232
///
233
/// # SQLite Interaction
234
///
235
/// This type implements [`FromSql`] and [`ToSql`], making it convenient to
236
/// integrate into SQL statements, as the database schema represents timestamps
237
/// internally using a non-negative [`i64`] storing the seconds since the epoch.
238
#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
239
pub(crate) struct Timestamp(SystemTime);
240

            
241
impl From<SystemTime> for Timestamp {
242
20173
    fn from(value: SystemTime) -> Self {
243
20173
        Self(value)
244
20173
    }
245
}
246

            
247
impl From<Timestamp> for SystemTime {
248
48
    fn from(value: Timestamp) -> Self {
249
48
        value.0
250
48
    }
251
}
252

            
253
impl Add<Duration> for Timestamp {
254
    type Output = Self;
255

            
256
    /// Performs a saturating addition wrapping to [`SystemTime::max_value()`].
257
10007
    fn add(self, rhs: Duration) -> Self::Output {
258
10007
        Self(self.0.saturating_add(rhs))
259
10007
    }
260
}
261

            
262
impl Sub<Duration> for Timestamp {
263
    type Output = Self;
264

            
265
    /// Performs a saturating subtraction wrapping to [`SystemTime::min_value()`].
266
    fn sub(self, rhs: Duration) -> Self::Output {
267
        Self(self.0.saturating_sub(rhs))
268
    }
269
}
270

            
271
impl Sub<Timestamp> for Timestamp {
272
    type Output = Duration;
273

            
274
    /// Performs a saturating duration_since wrapping to [`Duration::ZERO`].
275
10003
    fn sub(self, rhs: Timestamp) -> Self::Output {
276
        #[allow(unstable_name_collisions)]
277
10003
        self.0.saturating_duration_since(rhs.0)
278
10003
    }
279
}
280

            
281
impl FromSql for Timestamp {
282
118
    fn column_result(value: ValueRef<'_>) -> FromSqlResult<Self> {
283
118
        let mut res = SystemTime::UNIX_EPOCH;
284
118
        res = res.saturating_add(Duration::from_secs(value.as_i64()?.try_into().unwrap_or(0)));
285
118
        Ok(Self(res))
286
118
    }
287
}
288

            
289
impl ToSql for Timestamp {
290
154
    fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
291
        #[allow(unstable_name_collisions)]
292
154
        Ok(ToSqlOutput::from(
293
154
            self.0
294
154
                .saturating_duration_since(SystemTime::UNIX_EPOCH)
295
154
                .as_secs()
296
154
                .try_into()
297
154
                .unwrap_or(i64::MAX),
298
154
        ))
299
154
    }
300
}
301

            
302
/// Representation of consensus metadata from the database.
303
#[derive(Educe)]
304
#[educe(Debug, Clone, Copy, PartialEq, Eq)]
305
pub(crate) struct ConsensusMeta<T> {
306
    /// The document id uniquely identifying the consensus.
307
    pub docid: DocumentId,
308

            
309
    /// The SHA3 of the unsigned part of the consensus.
310
    pub sha3_256: Sha3_256,
311

            
312
    /// The time after which this consensus is valid.
313
    pub valid_after: Timestamp,
314

            
315
    /// The time after which this consensus stops being fresh.
316
    pub fresh_until: Timestamp,
317

            
318
    /// The time after which this consensus stops being valid.
319
    pub valid_until: Timestamp,
320

            
321
    /// The flavor of the consensus; determined at compile time.
322
    flavor: PhantomData<T>,
323
}
324

            
325
impl<T: FlavoredConsensusUnverified> ConsensusMeta<T> {
326
    /// Select the missing router descriptors.
327
    ///
328
    /// A router descriptor is considered missing if it exists in
329
    /// `consensus_descriptor_member` but not in `descriptor`
330
    /// because the first entry is added once the consensus got parsed,
331
    /// whereas the second entry is added once we have actually retrieved it.
332
    ///
333
    /// It works by doing a left join on descriptor and filtering for
334
    /// all entries where the join is NULL, as that implies we are aware of
335
    /// the descriptor but not have it stored.
336
    ///
337
    /// Parameters:
338
    /// :docid - The docid of the consensus.
339
    /// :exclude - The descriptors to skip.
340
    /// :limit - The maximum number of descriptors to return.
341
    //
342
    // TODO DIRMIRROR: Potentially constify more queries.
343
    const MISSING_ROUTERS_QUERY: &'static str = sql!(
344
        "
345
        SELECT cr.sha1, RANDOM() AS rand
346
        FROM consensus_descriptor_member AS cr
347
          LEFT JOIN descriptor AS router ON cr.sha1 = router.sha1
348
        WHERE
349
          cr.consensus_docid = :docid
350
          AND cr.sha1 IS NOT NULL
351
          AND cr.sha1 NOT IN rarray(:exclude)
352
          AND router.sha1 IS NULL
353
        ORDER BY rand
354
        LIMIT :limit
355
        "
356
    );
357

            
358
    /// Obtains all consensuses found in the database.
359
    ///
360
    /// This should be reasonable size-wise, given that a ConsensusMeta instance
361
    /// is very small and that the garbage collector removes old consensuses
362
    /// anyways.  If this becomes a problem, we may want to add an optional
363
    /// limit.
364
10
    pub(crate) fn query(tx: &Transaction) -> Result<Vec<Self>, DatabaseError> {
365
        // Select the most recent flavored consensus document from the database.
366
10
        let mut meta_stmt = tx.prepare_cached(sql!(
367
10
            "
368
10
            SELECT docid, sha3_256, valid_after, fresh_until, valid_until
369
10
            FROM consensus
370
10
            WHERE
371
10
              flavor = :flavor
372
10
            ORDER BY valid_after DESC
373
10
            "
374
10
        ))?;
375

            
376
        // Actually execute the query.
377
10
        let rows = meta_stmt.query_map(
378
10
            named_params! {
379
10
                ":flavor": T::flavor().name(),
380
            },
381
10
            |row| {
382
                Ok(Self {
383
10
                    docid: row.get(0)?,
384
10
                    sha3_256: row.get(1)?,
385
10
                    valid_after: row.get(2)?,
386
10
                    fresh_until: row.get(3)?,
387
10
                    valid_until: row.get(4)?,
388
10
                    flavor: Default::default(),
389
                })
390
10
            },
391
        )?;
392

            
393
10
        Ok(rows.collect::<Result<Vec<_>, _>>()?)
394
10
    }
395

            
396
    /// Queries the raw data of a [`ConsensusMeta`].
397
4
    pub(crate) fn data(&self, tx: &Transaction<'_>) -> Result<String, DatabaseError> {
398
4
        let mut stmt = tx.prepare_cached(sql!(
399
4
            "
400
4
            SELECT content
401
4
            FROM store
402
4
            WHERE docid = :docid
403
4
            "
404
4
        ))?;
405

            
406
4
        let raw = stmt.query_one(named_params! {":docid": self.docid}, |row| {
407
4
            row.get::<_, Vec<u8>>(0)
408
4
        })?;
409
4
        let raw = String::from_utf8(raw).map_err(into_internal!("utf-8 constraint violated?"))?;
410
4
        Ok(raw)
411
4
    }
412

            
413
    /// Calculates the [`Timestamp`] at which the authorities will be queried again.
414
    ///
415
    /// # Specifications
416
    ///
417
    /// * <https://spec.torproject.org/dir-spec/directory-cache-operation.html#download-ns-from-auth>
418
10002
    pub(crate) fn ttl<R: Rng>(&self, rng: &mut R) -> Timestamp {
419
10002
        assert!(self.fresh_until < self.valid_until);
420

            
421
10002
        let offset = rng
422
10002
            .gen_range_checked(0..=((self.valid_until - self.fresh_until).as_secs() / 2))
423
10002
            .expect("invalid range?");
424

            
425
10002
        self.fresh_until + Duration::from_secs(offset)
426
10002
    }
427

            
428
    /// Returns the missing router descriptors for this consensus.
429
    ///
430
    /// `limit` may be given to specify an optional upper limit, in which case
431
    /// the result will contain at most `limit` missing descriptors.
432
    ///
433
    /// # Performance
434
    ///
435
    /// The performance here is O(n * log n) for each 64-element batch, leading
436
    /// to an overall performance of O(n^2 * (log n)/64) per consensus period,
437
    /// which sould be acceptable.  See the link below for a further discussion
438
    /// on performance related matters:
439
    ///
440
    /// <https://gitlab.torproject.org/tpo/core/arti/-/merge_requests/4378#note_3467300>
441
    // TODO DIRMIRROR: This has a large overlap with missing_extras and
442
    // missing_micros, we should genericise or macrofy this at one point.
443
9
    pub(crate) fn missing_routers(
444
9
        &self,
445
9
        tx: &Transaction<'_>,
446
9
        exclude: impl Iterator<Item = Sha1>,
447
9
        limit: Option<u64>,
448
9
    ) -> Result<HashSet<Sha1>, DatabaseError> {
449
9
        if T::flavor() != ConsensusFlavor::Plain {
450
            return Ok(HashSet::new());
451
9
        }
452

            
453
9
        let mut stmt = tx.prepare_cached(Self::MISSING_ROUTERS_QUERY)?;
454

            
455
9
        let exclude = exclude.map(Value::from).collect();
456
9
        let exclude: Rc<Vec<Value>> = Rc::new(exclude);
457
9
        let limit = limit.map_or(-1, |n| n.try_into().unwrap_or(i64::MAX));
458
9
        let missing = stmt
459
9
            .query_map(
460
9
                named_params! {":docid": self.docid, ":limit": limit, ":exclude": exclude},
461
30
                |row| row.get(0),
462
            )?
463
9
            .collect::<Result<HashSet<_>, _>>()?;
464
9
        Ok(missing)
465
9
    }
466

            
467
    /// Returns the missing extra infos for this consensus to the best of our abilities.
468
    ///
469
    /// Keep in mind that this does not return **all** missing extra infos but
470
    /// only the missing extra infos of router descriptors we have.
471
    ///
472
    /// `limit` may be given to specify an optional upper limit, in which case
473
    /// the result will contain at most `limit` missing extra-infos.
474
    ///
475
    /// See [`ConsensusMeta::missing_routers()`] for a discussion on
476
    /// performance.
477
2
    pub(crate) fn missing_extras(
478
2
        &self,
479
2
        tx: &Transaction<'_>,
480
2
        exclude: impl Iterator<Item = Sha1>,
481
2
        limit: Option<u64>,
482
2
    ) -> Result<HashSet<Sha1>, DatabaseError> {
483
2
        if T::flavor() != ConsensusFlavor::Plain {
484
            return Ok(HashSet::new());
485
2
        }
486

            
487
        // Select the missing extra infos for this consensus.
488
        //
489
        // This return value is not complete because we only know the missing
490
        // extra-infos to the best of our abilities.  In other words: We are
491
        // only aware of a missing extra-info if we have parsed the respective
492
        // router descriptor.
493
        //
494
        // It works by doing an inner join from
495
        // `consensus_descriptor_member` to `descriptor` because
496
        // we can only know about the extra-infos of which we have the router
497
        // descriptors from.  Afterwards, we do a left join with the
498
        // `extra_info` table and filter for all results where the left
499
        // join result is null, hence where we have a router descriptor but not
500
        // the respective extra-info.
501
        //
502
        // Parameters:
503
        // :docid - The docid of the consensus.
504
        // :exclude - The descriptors to skip.
505
        // :limit - The maximum number of descriptors to return.
506
2
        let mut stmt = tx.prepare_cached(sql!(
507
2
            "
508
2
            SELECT router.extra_sha1, RANDOM() AS rand
509
2
            FROM consensus_descriptor_member AS cr
510
2
              INNER JOIN descriptor AS router ON cr.sha1 = router.sha1
511
2
              LEFT JOIN extra_info AS extra ON router.extra_sha1 = extra.sha1
512
2
            WHERE
513
2
              cr.consensus_docid = :docid
514
2
              AND router.extra_sha1 IS NOT NULL
515
2
              AND extra.sha1 IS NULL
516
2
              AND extra.sha1 NOT IN rarray(:exclude)
517
2
            ORDER BY rand
518
2
            LIMIT :limit
519
2
            "
520
2
        ))?;
521

            
522
2
        let exclude = exclude.map(Value::from).collect();
523
2
        let exclude: Rc<Vec<Value>> = Rc::new(exclude);
524
2
        let limit = limit.map_or(-1, |n| n.try_into().unwrap_or(i64::MAX));
525
2
        let missing = stmt
526
2
            .query_map(
527
2
                named_params! {":docid": self.docid, ":limit": limit, ":exclude": exclude},
528
                |row| row.get(0),
529
            )?
530
2
            .collect::<Result<HashSet<_>, _>>()?;
531
2
        Ok(missing)
532
2
    }
533

            
534
    /// Returns the missing micro descriptors for this consensus.
535
    ///
536
    /// `limit` may be given to specify an optional upper limit, in which case
537
    /// the result will contain at most `limit` missing descriptors.
538
    ///
539
    /// See [`ConsensusMeta::missing_routers()`] for a discussion on
540
    /// performance.
541
3
    pub(crate) fn missing_micros(
542
3
        &self,
543
3
        tx: &Transaction<'_>,
544
3
        exclude: impl Iterator<Item = Sha256>,
545
3
        limit: Option<u64>,
546
3
    ) -> Result<HashSet<Sha256>, DatabaseError> {
547
3
        if T::flavor() != ConsensusFlavor::Microdesc {
548
1
            return Ok(HashSet::new());
549
2
        }
550

            
551
        // Select the missing micro descriptors.
552
        //
553
        // A micro descriptor is considered missing if it exists in
554
        // `consensus_descriptor_member` but not in `descriptor`
555
        // because the first entry is added once the consensus got parsed,
556
        // whereas the second entry is added once we have actually retrieved it.
557
        //
558
        // It works by doing a left join on descriptor and filtering for
559
        // all entries where the join is NULL, as that implies we are aware of
560
        // the descriptor but not have it stored.
561
        //
562
        // Parameters:
563
        // :docid - The docid of the consensus.
564
        // :limit - The maximum number of descriptors to return.
565
2
        let mut stmt = tx.prepare_cached(sql!(
566
2
            "
567
2
            SELECT cr.sha2, RANDOM() AS rand
568
2
            FROM consensus_descriptor_member AS cr
569
2
              LEFT JOIN descriptor AS micro ON cr.sha2 = micro.sha2
570
2
            WHERE
571
2
              cr.consensus_docid = :docid
572
2
              AND cr.sha2 IS NOT NULL
573
2
              AND micro.sha2 IS NULL
574
2
              AND micro.sha2 NOT IN rarray(:exclude)
575
2
            ORDER BY rand
576
2
            LIMIT :limit
577
2
            "
578
2
        ))?;
579

            
580
2
        let exclude = exclude.map(Value::from).collect();
581
2
        let exclude: Rc<Vec<Value>> = Rc::new(exclude);
582
2
        let limit = limit.map_or(-1, |n| n.try_into().unwrap_or(i64::MAX));
583
2
        let missing = stmt
584
2
            .query_map(
585
2
                named_params! {":docid": self.docid, ":limit": limit, ":exclude": exclude},
586
8
                |row| row.get(0),
587
            )?
588
2
            .collect::<Result<HashSet<_>, _>>()?;
589
2
        Ok(missing)
590
3
    }
591

            
592
    /// Inserts a verified consensus into the database.
593
    ///
594
    /// This method *DOES NOT* compute any consensus diffs but populates
595
    /// associated meta tables such as `consensus_descriptor_member`
596
    /// and `consensus_authority_voter` properly.
597
22
    pub(crate) fn insert<I>(
598
22
        tx: &Transaction<'_>,
599
22
        encodings: I,
600
22
        (body, sigs): (&T::Body, &T::Signatures),
601
22
        data: &str,
602
22
    ) -> Result<ConsensusMeta<T>, DatabaseError>
603
22
    where
604
22
        I: Iterator<Item = ContentEncoding>,
605
    {
606
        // Insert a consensus into the consensus meta table.
607
        //
608
        // This requires the consensus to already be present in the `store`
609
        // table.
610
        //
611
        // Parameters:
612
        // :docid - The docid of the consensus.
613
        // :sha3_256 - The SHA3-256 sum of the part of the consensus
614
        //   without signatures but a trailing `\ndirectory-signature<SPACE>`.
615
        // :flavor - The consensus flavor to use.
616
        // :valid_after, :fresh_until, :valid_until - The respective validities.
617
22
        let mut cons_stmt = tx.prepare_cached(sql!(
618
22
            "
619
22
            INSERT INTO consensus
620
22
            (docid, sha3_256, flavor, valid_after, fresh_until, valid_until)
621
22
            VALUES
622
22
            (:docid, :sha3_256, :flavor, :valid_after, :fresh_until, :valid_until)
623
22
            ON CONFLICT DO NOTHING
624
22
            "
625
22
        ))?;
626

            
627
        // Insert the relationship between a router status to a consensus.
628
        //
629
        // Parameters:
630
        // :docid - The consensus docid.
631
        // :sha1 - The router descriptor digest (plain consensus only, NULL otherwise).
632
        // :sha2 - The micro descriptor digest (microdesc consensus only, NULL otherwise).
633
22
        let mut cons_rs_member_stmt = tx.prepare_cached(sql!(
634
22
            "
635
22
            INSERT INTO consensus_descriptor_member
636
22
            (consensus_docid, sha1, sha2)
637
22
            VALUES
638
22
            (:docid, :sha1, :sha2)
639
22
            ON CONFLICT DO NOTHING
640
22
            "
641
22
        ))?;
642

            
643
        // Insert the relationship between authority certificate to consensus votes.
644
        //
645
        // Parameters:
646
        // :cons - The consensus docid.
647
        // :auth - The authority certificate docid.
648
22
        let mut cons_auth_voter = tx.prepare_cached(sql!(
649
22
            "
650
22
            INSERT INTO consensus_authority_voter
651
22
            (consensus_docid, kp_auth_id_rsa_sha1)
652
22
            VALUES
653
22
            (:consensus_docid, :id_rsa)
654
22
            ON CONFLICT DO NOTHING
655
22
            "
656
22
        ))?;
657

            
658
        // First, insert the consensus as is.
659
22
        let docid = store_insert(tx, data.as_bytes(), encodings)?;
660

            
661
        // TODO DIRMIRROR: We *need* this in tor-netdoc!!!
662
22
        let sha3_256 = Sha3_256::digest(
663
22
            data.split_inclusive("\ndirectory-signature ")
664
22
                .next()
665
22
                .ok_or_else(|| internal!("verified document without signatures?"))?
666
22
                .as_bytes(),
667
        );
668

            
669
22
        let lifetime = body.lifetime();
670
22
        let meta = ConsensusMeta {
671
22
            docid,
672
22
            sha3_256,
673
22
            valid_after: lifetime.valid_after.0.into(),
674
22
            fresh_until: lifetime.fresh_until.0.into(),
675
22
            valid_until: lifetime.valid_until.0.into(),
676
22
            flavor: Default::default(),
677
22
        };
678
22
        cons_stmt.execute(named_params! {
679
22
            ":docid": meta.docid,
680
22
            ":sha3_256": meta.sha3_256,
681
22
            ":valid_after": meta.valid_after,
682
22
            ":fresh_until": meta.fresh_until,
683
22
            ":valid_until": meta.valid_until,
684
22
            ":flavor": T::flavor().name(),
685
22
        })?;
686

            
687
        // Insert all router descriptor member relationships.
688
        //
689
        // Many small queries are fast in sqlite, so this shouldn't be a perf problem.
690
        // https://sqlite.org/np1queryprob.html
691
        //
692
        // Depending on the flavor, we map the document digests to an iterator
693
        // of a tuple of Option's, where one of the values is always None.
694
        // This represents the fact that router descriptors use SHA-1 and micro
695
        // descriptors SHA-256.
696
154
        for (sha1, sha2) in body.doc_digests() {
697
154
            cons_rs_member_stmt.execute(named_params! {
698
154
                ":docid": docid,
699
154
                ":sha1": sha1,
700
154
                ":sha2": sha2,
701
154
            })?;
702
        }
703

            
704
88
        for sig in sigs.signatories() {
705
88
            cons_auth_voter.execute(named_params! {
706
88
                ":consensus_docid": docid,
707
88
                ":id_rsa": Sha1(sig.id_fingerprint.to_bytes()),
708
88
            })?;
709
        }
710

            
711
22
        Ok(meta)
712
22
    }
713
}
714

            
715
/// Representation of authority certificate metadata from the database.
716
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
717
pub(crate) struct AuthCertMeta {
718
    /// The document id uniquely identifying the consensus.
719
    pub docid: DocumentId,
720

            
721
    /// The SHA-1 fingerprint of the identity key.
722
    // TODO DIRMIRROR: Change this to RsaIdentity.
723
    pub kp_auth_id_rsa_sha1: Sha1,
724

            
725
    /// The SHA-1 fingerprint of the signign key.
726
    // TODO DIRMIRROR: Change this to RsaIdentity.
727
    pub kp_auth_sign_rsa_sha1: Sha1,
728

            
729
    /// The timestamp after which this certificate will be valid.
730
    pub dir_key_published: Timestamp,
731

            
732
    /// The timestamp until this certificate will be valid.
733
    pub dir_key_expires: Timestamp,
734
}
735

            
736
impl AuthCertMeta {
737
    /// Obtains the authority certificates from the database.
738
13
    pub(crate) fn query(tx: &Transaction) -> Result<Vec<Self>, DatabaseError> {
739
        // Obtain all certificates from the database.
740
        //
741
        // This is okay because the set is not very big.
742
        //
743
        // In the unlikely edge case of on identity-signing key pair having
744
        // multiple certificates, the most recently published certificate is
745
        // going to be used.
746
        //
747
        // TODO DIRMIRROR: Perhaps we should modify the auth_certs table to
748
        // add a UNIQUE constraint on that combination, while modifying the
749
        // insertion logic to replace with the newer one in the case of a
750
        // conflict.
751
13
        let mut stmt = tx.prepare_cached(sql!(
752
13
            "
753
13
            SELECT docid, kp_auth_id_rsa_sha1, kp_auth_sign_rsa_sha1,
754
13
              dir_key_published, dir_key_expires
755
13
            FROM authority_key_certificate
756
13
            GROUP BY kp_auth_id_rsa_sha1, kp_auth_sign_rsa_sha1
757
13
            ORDER BY MAX(dir_key_published)
758
13
            "
759
13
        ))?;
760

            
761
13
        let certs = stmt
762
44
            .query_map(params![], |row| {
763
                Ok(Self {
764
44
                    docid: row.get(0)?,
765
44
                    kp_auth_id_rsa_sha1: row.get(1)?,
766
44
                    kp_auth_sign_rsa_sha1: row.get(2)?,
767
44
                    dir_key_published: row.get(3)?,
768
44
                    dir_key_expires: row.get(4)?,
769
                })
770
44
            })?
771
13
            .collect::<Result<_, _>>()?;
772
13
        Ok(certs)
773
13
    }
774

            
775
    /// Queries the raw data of an [`AuthCertMeta`].
776
44
    pub(crate) fn data(&self, tx: &Transaction<'_>) -> Result<String, DatabaseError> {
777
44
        let mut stmt = tx.prepare_cached(sql!(
778
44
            "
779
44
            SELECT content
780
44
            FROM store
781
44
            WHERE docid = :docid
782
44
            "
783
44
        ))?;
784

            
785
44
        let raw = stmt.query_one(named_params! {":docid": self.docid}, |row| {
786
44
            row.get::<_, Vec<u8>>(0)
787
44
        })?;
788
44
        let raw = String::from_utf8(raw).map_err(into_internal!("utf-8 constraint violated?"))?;
789
44
        Ok(raw)
790
44
    }
791

            
792
    /// Inserts a new authority certificate into the database.
793
    ///
794
    /// Keep in mind that the data in the [`AuthCert`] should correspond to the
795
    /// data found in `data`, as this method performs no parsing.
796
44
    pub(crate) fn insert<I: Iterator<Item = ContentEncoding>>(
797
44
        tx: &Transaction<'_>,
798
44
        encodings: I,
799
44
        cert: &AuthCert,
800
44
        data: &str,
801
44
    ) -> Result<(), DatabaseError> {
802
        // Inserts a new certificate into the meta table.
803
        //
804
        // Parameters:
805
        // :docid - The document id.
806
        // :id_rsa - The identity key fingerprint.
807
        // :sign_rsa - The signing key fingerprint
808
        // :published - The published timestamp.
809
        // :expires - The expires timestamp.
810
44
        let mut stmt = tx.prepare_cached(sql!(
811
44
            "
812
44
            INSERT INTO authority_key_certificate
813
44
            (docid, kp_auth_id_rsa_sha1, kp_auth_sign_rsa_sha1, dir_key_published, dir_key_expires)
814
44
            VALUES
815
44
            (:docid, :id_rsa, :sign_rsa, :published, :expires)
816
44
            ON CONFLICT DO NOTHING
817
44
            "
818
44
        ))?;
819

            
820
44
        let docid = store_insert(tx, data.as_bytes(), encodings)?;
821
44
        stmt.execute(named_params! {
822
44
            ":docid": docid,
823
44
            ":id_rsa": cert.dir_identity_key.to_rsa_identity().as_hex_upper(),
824
44
            ":sign_rsa": cert.dir_signing_key.to_rsa_identity().as_hex_upper(),
825
44
            ":published": Timestamp::from(cert.dir_key_published.0),
826
44
            ":expires": Timestamp::from(cert.dir_key_expires.0),
827
44
        })?;
828

            
829
44
        Ok(())
830
44
    }
831
}
832

            
833
/// A no-op macro just returning the supplied.
834
///
835
/// The purpose of this macro is to semantically mark [`str`] literals to be
836
/// SQL statement.
837
///
838
/// Keep in mind that the compiler will not notice if you forget this macro.
839
/// Unfortunately, you have to ensure it yourself.
840
macro_rules! sql {
841
    ($s:literal) => {
842
        $s
843
    };
844
}
845

            
846
pub(crate) use sql;
847

            
848
/// Opens a database from disk, creating a [`Pool`] for it.
849
///
850
/// This function should be the entry point for all things requiring a database
851
/// handle, as this function prepares all necessary steps required for operating
852
/// on the database correctly, such as:
853
/// * Schema initialization.
854
/// * Schema upgrade.
855
/// * Setting connection specific settings.
856
///
857
/// # `SQLITE_BUSY` Caveat
858
///
859
/// There is a problem with the handling of `SQLITE_BUSY` when opening an
860
/// SQLite database.  In WAL, opening a database might acquire an exclusive lock
861
/// for a very short amount of time, in order to perform clean-up from previous
862
/// connections alongside other tasks for maintaining database integrity?  This
863
/// means, that opening multiple SQLite databases simultaneously will result in
864
/// a busy error regardless of a busy handler, as setting a busy handler will
865
/// require an existing connection, something we are unable to obtain in the
866
/// first place.
867
///
868
/// In order to mitigate this issue, the recommended way in the SQLite community
869
/// is to simply ensure that database connections are opened sequentially,
870
/// by urging calling applications to just use a single [`Pool`] instance.
871
///
872
/// Testing this is hard unfortunately.
873
21
pub(crate) fn open<P: AsRef<Path>>(
874
21
    path: P,
875
21
) -> Result<Pool<SqliteConnectionManager>, DatabaseError> {
876
21
    let num_cores = std::thread::available_parallelism()
877
21
        .unwrap_or(NonZero::new(8).expect("8 == 0?"))
878
21
        .get() as u32;
879

            
880
21
    let manager = r2d2_sqlite::SqliteConnectionManager::file(&path)
881
2016
        .with_init(|c| rusqlite::vtab::array::load_module(c));
882
21
    let pool = Pool::builder().max_size(num_cores).build(manager)?;
883

            
884
21
    rw_tx(&pool, |tx| {
885
        // Prepare the database, doing the following steps:
886
        // 1. Checking the database schema.
887
        // 2. Upgrading (in future) or initializing the database schema (if empty).
888

            
889
21
        let has_arti_dirserver_schema_version = match tx.query_one(
890
            sql!(
891
21
                "
892
21
                SELECT name
893
21
                FROM sqlite_master
894
21
                  WHERE type = 'table'
895
21
                    AND name = 'arti_dirserver_schema_version'
896
21
                "
897
            ),
898
21
            params![],
899
1
            |_| Ok(()),
900
        ) {
901
1
            Ok(()) => true,
902
20
            Err(rusqlite::Error::QueryReturnedNoRows) => false,
903
            Err(e) => return Err(DatabaseError::LowLevel(e)),
904
        };
905

            
906
21
        if has_arti_dirserver_schema_version {
907
1
            let version = tx.query_one(
908
1
                sql!("SELECT version FROM arti_dirserver_schema_version WHERE rowid = 1"),
909
1
                params![],
910
1
                |row| row.get::<_, String>(0),
911
            )?;
912

            
913
1
            match version.as_ref() {
914
1
                "1" => {}
915
1
                unknown => {
916
1
                    return Err(DatabaseError::IncompatibleSchema {
917
1
                        version: unknown.into(),
918
1
                    });
919
                }
920
            }
921
        } else {
922
20
            tx.execute_batch(V1_SCHEMA)?;
923
        }
924

            
925
20
        Ok::<_, DatabaseError>(())
926
21
    })??;
927

            
928
20
    Ok(pool)
929
21
}
930

            
931
/// Executes a closure `op` with a given read-only [`Transaction`].
932
///
933
/// The [`Transaction`] always gets rolled back the moment `op` returns.
934
///
935
/// The [`Transaction`] gets initialized with the global pragma options set.
936
///
937
/// **The closure shall not perform write operations!**
938
/// Not only do they get rolled back anyways, but upgrading the [`Transaction`]
939
/// from a read to a write transaction will lead to other simultaneous write upgrades
940
/// to fail.  Unfortunately, there is no real programmatic way to ensure this.
941
30
pub(crate) fn read_tx<U, F>(pool: &Pool<SqliteConnectionManager>, op: F) -> Result<U, DatabaseError>
942
30
where
943
30
    F: FnOnce(&Transaction<'_>) -> U,
944
{
945
30
    let mut conn = pool.get()?;
946
30
    conn.execute_batch(GLOBAL_OPTIONS)?;
947
30
    let tx = conn.transaction_with_behavior(TransactionBehavior::Deferred)?;
948
30
    let res = op(&tx);
949
30
    tx.rollback()?;
950
30
    Ok(res)
951
30
}
952

            
953
/// Executes a closure `op` with a given read-write [`Transaction`].
954
///
955
/// The [`Transaction`] always gets committed the moment `op` returns.
956
///
957
/// The [`Transaction`] gets initialized with the global pragma options set.
958
///
959
/// The [`Transaction`] gets created with [`TransactionBehavior::Immediate`],
960
/// meaning it will immediately exist as a write connection, retrying in the
961
/// case of a [`rusqlite::ErrorCode::DatabaseBusy`] until it failed after 1s.
962
35
pub(crate) fn rw_tx<U, F>(pool: &Pool<SqliteConnectionManager>, op: F) -> Result<U, DatabaseError>
963
35
where
964
35
    F: FnOnce(&Transaction<'_>) -> U,
965
{
966
35
    let mut conn = pool.get()?;
967
35
    conn.execute_batch(GLOBAL_OPTIONS)?;
968
35
    let tx = conn.transaction_with_behavior(TransactionBehavior::Exclusive)?;
969
34
    let res = op(&tx);
970
34
    tx.commit()?;
971
34
    Ok(res)
972
35
}
973

            
974
/// Inserts `data` into store while also compressing it with given encodings.
975
///
976
/// Returns the [`DocumentId`] of `data`.
977
///
978
/// This function inserts `data` into store and also compresses it into all
979
/// given compression formats.
980
///
981
/// Duplicates get re-encoded and replaced in the database, including
982
/// [`ContentEncoding::Identity`].
983
339
pub(crate) fn store_insert<I: Iterator<Item = ContentEncoding>>(
984
339
    tx: &Transaction,
985
339
    data: &[u8],
986
339
    encodings: I,
987
339
) -> Result<DocumentId, DatabaseError> {
988
    // The statement to insert some data into the store.
989
    //
990
    // Parameters:
991
    // :docid - The docid.
992
    // :content - The binary data.
993
339
    let mut store_stmt = tx.prepare_cached(sql!(
994
339
        "
995
339
        INSERT INTO store (docid, content)
996
339
        VALUES
997
339
        (:docid, :content)
998
339
        ON CONFLICT DO NOTHING
999
339
        "
339
    ))?;
    // The statement to insert a compressed document into the metatable.
    //
    // Parameters:
    // :algorithm - The name of the encoding algorithm.
    // :identity_docid - The docid of the plain-text document in the store.
    // :compressed_docid - The docid of the encoded document in the store.
339
    let mut compressed_stmt = tx.prepare_cached(sql!(
339
        "
339
        INSERT INTO compressed_document (algorithm, identity_docid, compressed_docid)
339
        VALUES
339
        (:algorithm, :identity_docid, :compressed_docid)
339
        ON CONFLICT DO NOTHING
339
        "
339
    ))?;
    // Insert the plain document into the store.
339
    let identity_docid = DocumentId::digest(data);
339
    store_stmt.execute(named_params! {
339
        ":docid": identity_docid,
339
        ":content": data
339
    })?;
    // Compress it into all formats and insert it into store and compressed.
371
    for encoding in encodings {
371
        if encoding == ContentEncoding::Identity {
            // Ignore identity because we inserted that above.
339
            continue;
32
        }
        // We map a compression error to a bug because there is no good reason
        // on why it should fail, given that we compress from memory data to
        // memory data.  Probably because it uses the std::io::Writer interface
        // which itself demands use of std::io::Result.
32
        let compressed = compress(data, encoding).map_err(into_internal!("{encoding} failed?"))?;
32
        let compressed_docid = DocumentId::digest(&compressed);
32
        store_stmt.execute(named_params! {
32
            ":docid": compressed_docid,
32
            ":content": compressed,
32
        })?;
32
        compressed_stmt.execute(named_params! {
32
            ":algorithm": encoding.to_string(),
32
            ":identity_docid": identity_docid,
32
            ":compressed_docid": compressed_docid,
32
        })?;
    }
339
    Ok(identity_docid)
339
}
/// Compresses `data` into a specified [`ContentEncoding`].
///
/// Returns a [`Vec`] containing the encoded data.
37
fn compress(data: &[u8], encoding: ContentEncoding) -> Result<Vec<u8>, std::io::Error> {
37
    match encoding {
1
        ContentEncoding::Identity => Ok(data.to_vec()),
        ContentEncoding::Deflate => {
9
            let mut w = DeflateEncoder::new(Vec::new(), Default::default());
9
            w.write_all(data)?;
9
            w.finish()
        }
        ContentEncoding::Gzip => {
9
            let mut w = GzEncoder::new(Vec::new(), Default::default());
9
            w.write_all(data)?;
9
            w.finish()
        }
9
        ContentEncoding::XZstd => zstd::encode_all(data, Default::default()),
        ContentEncoding::XTorLzma => {
9
            let mut res = Vec::new();
9
            lzma_rs::lzma_compress(&mut Cursor::new(data), &mut res)?;
9
            Ok(res)
        }
    }
37
}
#[cfg(test)]
mod test {
    // @@ begin test lint list maintained by maint/add_warning @@
    #![allow(clippy::bool_assert_comparison)]
    #![allow(clippy::clone_on_copy)]
    #![allow(clippy::dbg_macro)]
    #![allow(clippy::mixed_attributes_style)]
    #![allow(clippy::print_stderr)]
    #![allow(clippy::print_stdout)]
    #![allow(clippy::single_char_pattern)]
    #![allow(clippy::unwrap_used)]
    #![allow(clippy::unchecked_time_subtraction)]
    #![allow(clippy::useless_vec)]
    #![allow(clippy::needless_pass_by_value)]
    #![allow(clippy::string_slice)] // See arti#2571
    //! <!-- @@ end test lint list maintained by maint/add_warning @@ -->
    use std::{
        collections::HashSet,
        io::Read,
        iter,
        sync::{Arc, Once},
    };
    use flate2::read::{DeflateDecoder, GzDecoder};
    use rusqlite::Connection;
    use strum::IntoEnumIterator;
    use tempfile::tempdir;
    use tor_basic_utils::test_rng::testing_rng;
    use tor_netdoc::doc::netstatus::{md, plain};
    use crate::testdata2::{self, current_consensus_ns};
    use super::*;
    type Plain = plain::NetworkStatusUnverified;
    type Md = md::NetworkStatusUnverified;
    #[test]
    fn open_test() {
        let db_dir = tempdir().unwrap();
        let db_path = db_dir.path().join("db");
        open(&db_path).unwrap();
        let conn = Connection::open(&db_path).unwrap();
        // Check if the version was initialized properly.
        let version = conn
            .query_one(
                "SELECT version FROM arti_dirserver_schema_version WHERE rowid = 1",
                params![],
                |row| row.get::<_, String>(0),
            )
            .unwrap();
        assert_eq!(version, "1");
        // Set the version to something unknown.
        conn.execute(
            "UPDATE arti_dirserver_schema_version SET version = 42",
            params![],
        )
        .unwrap();
        drop(conn);
        assert_eq!(
            open(&db_path).unwrap_err().to_string(),
            "incompatible schema version: 42"
        );
    }
    #[test]
    fn read_tx_test() {
        let db_dir = tempdir().unwrap();
        let db_path = db_dir.path().join("db");
        let pool = open(&db_path).unwrap();
        // Do a write transaction despite forbidden.
        read_tx(&pool, |tx| {
            tx.execute_batch("DELETE FROM arti_dirserver_schema_version")
                .unwrap();
            let e = tx
                .query_one(
                    sql!("SELECT version FROM arti_dirserver_schema_version"),
                    params![],
                    |row| row.get::<_, String>(0),
                )
                .unwrap_err();
            assert_eq!(e, rusqlite::Error::QueryReturnedNoRows);
        })
        .unwrap();
        // Normal check.
        let version: String = read_tx(&pool, |tx| {
            tx.query_one(
                sql!("SELECT version FROM arti_dirserver_schema_version"),
                params![],
                |row| row.get(0),
            )
            .unwrap()
        })
        .unwrap();
        assert_eq!(version, "1");
    }
    #[test]
    fn rw_tx_test() {
        let db_dir = tempdir().unwrap();
        let db_path = db_dir.path().join("db");
        let pool = open(&db_path).unwrap();
        // Do a write transaction.
        rw_tx(&pool, |tx| {
            tx.execute_batch("DELETE FROM arti_dirserver_schema_version")
                .unwrap();
        })
        .unwrap();
        // Check that it was deleted.
        read_tx(&pool, |tx| {
            let e = tx
                .query_one(
                    sql!("SELECT version FROM arti_dirserver_schema_version"),
                    params![],
                    |row| row.get::<_, String>(0),
                )
                .unwrap_err();
            assert_eq!(e, rusqlite::Error::QueryReturnedNoRows);
        })
        .unwrap();
    }
    /// Tests whether our SQLite busy error handling works in normal situations.
    ///
    /// A normal situations means a situation where a lock is never held for
    /// more than 1000ms.  In our case, we will work with two threads.
    /// t1 will acquire an exclusive lock and inform t2 about it.  t2 waits
    /// until t1 has acquired this lock and then immediately informs t1, that
    /// it will now wait for a lock too.  Now, t1 will immediately terminate,
    /// thereby releasing the lock and leading t2 to eventually acquire it.
    #[test]
    fn rw_tx_busy_timeout_working() {
        let db_dir = tempdir().unwrap();
        let db_path = db_dir.path().join("db");
        let pool = open(db_path).unwrap();
        // t2 will wait on this before it starts doing stuff.
        let t1_acquired_lock = Arc::new(Once::new());
        // t1 will wait on this in order to terminate properly.
        let t2_is_waiting = Arc::new(Once::new());
        let t1 = std::thread::spawn({
            let pool = pool.clone();
            let t1_acquired_lock = t1_acquired_lock.clone();
            let t2_is_waiting = t2_is_waiting.clone();
            move || {
                rw_tx(&pool, move |_tx| {
                    // Inform t2 we have write lock.
                    t1_acquired_lock.call_once(|| ());
                    println!("t1 acquired write lock");
                    // Wait for t2 to start waiting.
                    t2_is_waiting.wait();
                })
                .unwrap();
                println!("t2 released write lock");
            }
        });
        println!("t2 waits for t1 to acquire write lock");
        t1_acquired_lock.wait();
        t2_is_waiting.call_once(|| ());
        rw_tx(&pool, |_| ()).unwrap();
        println!("t2 acquired and released write lock");
        t1.join().unwrap();
    }
    /// Tests whether our SQLite busy error handlings fails as expected.
    ///
    /// We configure SQLite to fail after 1000ms.  This test works with two
    /// threads.  t1 will acquire an exclusive lock on the database and will
    /// inform t2 about it, which itself will wait until t1 has acquired the
    /// lock.  t2 will then immediately try to also obtain an exclusive lock,
    /// which should fail after about 1000ms.  After the failure, t2 informs
    /// t1 that it has failed, causing t1 to terminate.
    #[test]
    fn rw_tx_busy_timeout_busy() {
        let db_dir = tempdir().unwrap();
        let db_path = db_dir.path().join("db");
        let pool = open(db_path).unwrap();
        // t2 will wait on this before it starts doing stuff.
        let t1_acquired_lock = Arc::new(Once::new());
        // t1 will wait on this in order to terminate properly.
        let t2_gave_up = Arc::new(Once::new());
        let t1 = std::thread::spawn({
            let pool = pool.clone();
            let t1_acquired_lock = t1_acquired_lock.clone();
            let t2_gave_up = t2_gave_up.clone();
            move || {
                rw_tx(&pool, move |_tx| {
                    // Inform t2 we have the write lock.
                    t1_acquired_lock.call_once(|| ());
                    println!("t1 acquired write lock");
                    // Wait for t2 to give up before we release (how mean from us).
                    t2_gave_up.wait();
                })
                .unwrap();
                println!("t1 released write lock");
            }
        });
        println!("t2 waits for t1 to acquire write lock");
        t1_acquired_lock.wait();
        let e = rw_tx(&pool, |_| ()).unwrap_err();
        assert_eq!(
            e.to_string(),
            "low-level rusqlite error: database is locked"
        );
        println!("t2 gave up on acquiring write lock");
        t2_gave_up.call_once(|| ());
        t1.join().unwrap();
    }
    #[test]
    fn store_insert_test() {
        let db_dir = tempdir().unwrap();
        let db_path = db_dir.path().join("db");
        open(&db_path).unwrap();
        let mut conn = Connection::open(&db_path).unwrap();
        let tx = conn.transaction().unwrap();
        let docid = store_insert(&tx, "foobar".as_bytes(), ContentEncoding::iter()).unwrap();
        assert_eq!(
            docid,
            "C3AB8FF13720E8AD9047DD39466B3C8974E592C2FA383D4A3960714CAEF0C4F2"
        );
        let res = tx
            .query_one(
                sql!(
                    "
                    SELECT content
                    FROM store
                    WHERE docid = 'C3AB8FF13720E8AD9047DD39466B3C8974E592C2FA383D4A3960714CAEF0C4F2'
                    "
                ),
                params![],
                |row| row.get::<_, Vec<u8>>(0),
            )
            .unwrap();
        assert_eq!(res, "foobar".as_bytes());
        let mut stmt = tx.prepare_cached(sql!(
            "
            SELECT algorithm
            FROM compressed_document
            WHERE identity_docid = 'C3AB8FF13720E8AD9047DD39466B3C8974E592C2FA383D4A3960714CAEF0C4F2'
            "
        )).unwrap();
        let algorithms = stmt
            .query_map(params![], |row| row.get::<_, String>(0))
            .unwrap();
        let algorithms = algorithms.map(|x| x.unwrap()).collect::<HashSet<_>>();
        assert_eq!(
            algorithms,
            HashSet::from([
                "deflate".to_string(),
                "gzip".to_string(),
                "x-zstd".to_string(),
                "x-tor-lzma".to_string()
            ])
        );
        // Now insert the same thing a second time again and see whether the
        // ON CONFLICT magic works.
        let docid_second = store_insert(&tx, "foobar".as_bytes(), ContentEncoding::iter()).unwrap();
        assert_eq!(docid, docid_second);
        // Remove a few compressed entries and get them again.
        let n = tx
            .execute(
                sql!(
                    "
                    DELETE FROM
                    compressed_document
                    WHERE algorithm IN ('deflate', 'x-zstd')
                    "
                ),
                params![],
            )
            .unwrap();
        assert_eq!(n, 2);
        let docid_third = store_insert(&tx, "foobar".as_bytes(), ContentEncoding::iter()).unwrap();
        assert_eq!(docid, docid_third);
        let algorithms = stmt
            .query_map(params![], |row| row.get::<_, String>(0))
            .unwrap();
        let algorithms = algorithms.map(|x| x.unwrap()).collect::<HashSet<_>>();
        assert_eq!(
            algorithms,
            HashSet::from([
                "deflate".to_string(),
                "gzip".to_string(),
                "x-zstd".to_string(),
                "x-tor-lzma".to_string()
            ])
        );
    }
    #[test]
    fn compress_test() {
        /// Asserts that `res` contains `encoding`.
        fn contains(encoding: ContentEncoding, res: &[(ContentEncoding, Vec<u8>)]) {
            assert!(res.iter().any(|x| x.0 == encoding));
        }
        const INPUT: &[u8] = "foobar".as_bytes();
        // Check whether everything was encoded.
        let res = ContentEncoding::iter()
            .map(|encoding| (encoding, compress(INPUT, encoding).unwrap()))
            .collect::<Vec<_>>();
        assert_eq!(res.len(), 5);
        contains(ContentEncoding::Identity, &res);
        contains(ContentEncoding::Deflate, &res);
        contains(ContentEncoding::Gzip, &res);
        contains(ContentEncoding::XTorLzma, &res);
        contains(ContentEncoding::XZstd, &res);
        // Check if we can decode it.
        for (encoding, compressed) in res {
            let mut decompressed = Vec::new();
            match encoding {
                ContentEncoding::Identity => decompressed = compressed,
                ContentEncoding::Deflate => {
                    DeflateDecoder::new(Cursor::new(compressed))
                        .read_to_end(&mut decompressed)
                        .unwrap();
                }
                ContentEncoding::Gzip => {
                    GzDecoder::new(Cursor::new(compressed))
                        .read_to_end(&mut decompressed)
                        .unwrap();
                }
                ContentEncoding::XTorLzma => {
                    lzma_rs::lzma_decompress(&mut Cursor::new(compressed), &mut decompressed)
                        .unwrap();
                }
                ContentEncoding::XZstd => {
                    decompressed = zstd::decode_all(Cursor::new(compressed)).unwrap();
                }
            }
            assert_eq!(decompressed, INPUT);
        }
    }
    /// Verify that the insertion of a consensus works.
    ///
    /// For this, we do not use the test database but rather a fresh one and
    /// call ConsensusMeta::insert().  After, we verify that the consensus
    /// can be queried as well as that the missing descriptors are properly
    /// calculated.
    #[test]
    fn consensus_insert() {
        let pool = open("").unwrap();
        let mut conn = pool.get().unwrap();
        let tx = conn.transaction().unwrap();
        let (body, sigs, data) = current_consensus_ns();
        let meta = ConsensusMeta::<Plain>::insert(
            &tx,
            iter::once(ContentEncoding::Identity),
            (&body, &sigs),
            data,
        )
        .unwrap();
        assert_eq!(
            meta,
            ConsensusMeta::<Plain> {
                docid: Sha256::digest(data.as_bytes()),
                sha3_256: testdata2::consensus_sha3(data),
                valid_after: body.preamble.lifetime.valid_after.0.into(),
                fresh_until: body.preamble.lifetime.fresh_until.0.into(),
                valid_until: body.preamble.lifetime.valid_until.0.into(),
                flavor: Default::default()
            }
        );
        let meta2 = ConsensusMeta::<Plain>::query(&tx).unwrap();
        assert_eq!(meta2, vec![meta]);
        let missing_descs = meta.missing_routers(&tx, iter::empty(), None).unwrap();
        let missing_descs2 = body.routers.iter().map(|r| Sha1(*r.doc_digest())).collect();
        assert_eq!(missing_descs, missing_descs2);
    }
    /// Tests whether the timeout computation lies within the proper interval.
    ///
    /// Because this involves randomness, it performs the test several thousand
    /// times.  This should be okay performance wise, as it takes about ~250ms
    /// with a debug build on my machine.
    #[test]
    fn sync_timeout() {
        // We repeat the tests a few thousand times to go over many random values.
        let docid = Sha256::digest(testdata2::current_consensus_ns().2.as_bytes());
        let lifetime = testdata2::current_consensus_ns().0.preamble.lifetime;
        let sha3_256 = testdata2::consensus_sha3(testdata2::current_consensus_ns().2);
        let cons = ConsensusMeta::<Plain> {
            docid,
            sha3_256,
            valid_after: lifetime.valid_after.0.into(),
            fresh_until: lifetime.fresh_until.0.into(),
            valid_until: lifetime.valid_until.0.into(),
            flavor: Default::default(),
        };
        for _ in 0..10000 {
            let when = cons.ttl(&mut testing_rng());
            assert!(when >= lifetime.fresh_until.0.into());
            // Computes the half between fresh_until and valid_until.
            assert!(
                when <= (lifetime.fresh_until.0
                    + (lifetime
                        .valid_until
                        .0
                        .duration_since(lifetime.fresh_until.0)
                        .unwrap()
                        / 2))
                    .into()
            );
        }
    }
    /// Tests whether the missing router descriptor queue is computed properly.
    ///
    /// For this, we remove existing router descriptors from the database and
    /// see whether they are determined as missing properly.
    #[test]
    fn missing_descriptors() {
        let pool = testdata2::test_db();
        let meta = read_tx(&pool, ConsensusMeta::<Plain>::query)
            .unwrap()
            .unwrap()[0];
        // Ensure that the returned consensus matches the one from testdata2.
        assert_eq!(
            meta.docid,
            DocumentId::digest(testdata2::current_consensus_ns().2.as_bytes())
        );
        // Delete a single router descriptor, so we can determine a missing one
        // using it.
        let removed_descriptor = *testdata2::current_consensus_ns().0.routers[0].doc_digest();
        let removed_descriptor = Sha1::from(removed_descriptor);
        pool.get()
            .unwrap()
            .execute(
                sql!(
                    "
                    DELETE FROM descriptor
                    WHERE sha1 = ?1
                    "
                ),
                params![removed_descriptor],
            )
            .unwrap();
        // Only one should be returned.
        let missing_routers = read_tx(&pool, |tx| meta.missing_routers(tx, iter::empty(), None))
            .unwrap()
            .unwrap();
        assert_eq!(missing_routers, HashSet::from([removed_descriptor]));
        // None should be returned if we exclude the descriptor we just removed.
        let missing_routers = read_tx(&pool, |tx| {
            meta.missing_routers(tx, iter::once(removed_descriptor), None)
        })
        .unwrap()
        .unwrap();
        assert!(missing_routers.is_empty());
        // If we delete all router descriptors we have, we should get all.
        rw_tx(&pool, |tx| {
            tx.execute(sql!("DELETE FROM descriptor"), params![])
        })
        .unwrap()
        .unwrap();
        // Now all should be returned; we verify this by checking that the
        // result is present in all_descriptors, which is a superset.
        let missing_routers = read_tx(&pool, |tx| meta.missing_routers(tx, iter::empty(), None))
            .unwrap()
            .unwrap();
        // This is a superset of missing_routers because it includes router
        // descriptors that are not a part of the current consensus.
        let all_descriptors = testdata2::current_router_descs()
            .iter()
            .map(|x| Sha1::from(x.1.hashes.sha1.unwrap()))
            .collect::<HashSet<_>>();
        assert!(
            missing_routers
                .iter()
                .all(|sha1| all_descriptors.contains(sha1))
        );
        // If we exclude one random descriptor, only that one should be missing.
        let to_exclude = missing_routers.iter().copied().next().unwrap();
        let missing_routers2 = read_tx(&pool, |tx| {
            meta.missing_routers(tx, iter::once(to_exclude), None)
        })
        .unwrap()
        .unwrap();
        assert_eq!(
            missing_routers
                .difference(&missing_routers2)
                .collect::<HashSet<_>>(),
            [&to_exclude].into()
        );
        // If we exclude all descriptors, it should be empty too.
        let missing_routers = read_tx(&pool, |tx| {
            meta.missing_routers(tx, all_descriptors.iter().copied(), None)
        })
        .unwrap()
        .unwrap();
        assert!(missing_routers.is_empty());
    }
    /// Tests whether or not the missing descriptors are actually random.
    #[test]
    fn missing_descriptors_are_random() {
        let pool = testdata2::test_db();
        rw_tx(&pool, |tx| {
            tx.execute(sql!("DELETE FROM descriptor"), ()).unwrap();
            let meta = ConsensusMeta::<Plain>::query(tx).unwrap()[0];
            // Ensure there are more than 1 missing descriptors now.
            let n = meta.missing_routers(tx, iter::empty(), None).unwrap().len();
            assert!(n > 1);
            let mut prev = HashSet::new();
            let mut randomness_works = false;
            for _ in 0..100 {
                let cur = meta.missing_routers(tx, iter::empty(), Some(1)).unwrap();
                if prev != cur {
                    randomness_works = true;
                    break;
                }
                prev = cur;
            }
            assert!(randomness_works);
        })
        .unwrap();
    }
    /// Tests whether the RANDOM() calls are executed before the ordering so
    /// that we have stability.
    #[test]
    fn missing_routers_monotonically_increasing() {
        let pool = testdata2::test_db();
        rw_tx(&pool, |tx| {
            let meta = ConsensusMeta::<Plain>::query(tx).unwrap();
            tx.execute(sql!("DELETE FROM descriptor"), ()).unwrap();
            let mut stmt = tx
                .prepare_cached(ConsensusMeta::<Plain>::MISSING_ROUTERS_QUERY)
                .unwrap();
            let empty = Rc::new(Vec::<Value>::new());
            let res = stmt
                .query_map(params![meta[0].docid, empty, -1], |row| {
                    row.get::<_, i64>(1)
                })
                .unwrap()
                .collect::<Result<Vec<_>, _>>()
                .unwrap();
            // Ensure there is more than 1 missing descriptor because we can
            // only test ordering if n >= 2.
            assert!(res.len() > 1);
            // Verify that the calls are monotonically increasing.
            assert!(res.is_sorted());
        })
        .unwrap();
    }
    /// Tests whether the missing extra-info documents are computed properly.
    // TODO DIRMIRROR: Expand on this once we have proper extra-info support.
    #[test]
    fn missing_extra_infos() {
        let pool = testdata2::test_db();
        let meta = read_tx(&pool, ConsensusMeta::<Plain>::query)
            .unwrap()
            .unwrap()[0];
        // Ensure that the returned consensus matches the one from testdata2.
        assert_eq!(
            meta.docid,
            DocumentId::digest(testdata2::current_consensus_ns().2.as_bytes())
        );
        // We should have no missing extra-infos.
        let missing_extras = read_tx(&pool, |tx| meta.missing_extras(tx, iter::empty(), None))
            .unwrap()
            .unwrap();
        assert!(missing_extras.is_empty());
        // TODO DIRMIRROR: Once we have support for extra-info's, add a test
        // for this here.  Right now, testing this is pretty useless as we
        // cannot add it nicely to the testdata2 module if there isn't even
        // an ExtraInfo struct from tor-netdoc.
    }
    /// Tests whether the missing micro descriptor queue is computed properly.
    ///
    /// For this, we remove existing micro descriptors from the database and
    /// see whether they are determined as missing properly.
    #[test]
    fn missing_micro_descriptors() {
        let pool = testdata2::test_db();
        let meta = read_tx(&pool, ConsensusMeta::<Md>::query).unwrap().unwrap()[0];
        // Ensure that the returned consensus matches the one from testdata2.
        assert_eq!(
            meta.docid,
            DocumentId::digest(testdata2::current_consensus_md().2.as_bytes())
        );
        // Delete a single router descriptor, so we can determine a missing one
        // using it.
        let removed_descriptor = *testdata2::current_consensus_md().0.routers[0].doc_digest();
        let removed_descriptor = Sha256::from(removed_descriptor);
        pool.get()
            .unwrap()
            .execute(
                sql!(
                    "
                    DELETE FROM descriptor
                    WHERE sha2 = ?1
                    "
                ),
                params![removed_descriptor],
            )
            .unwrap();
        // Only one should be returned.
        let missing_micros = read_tx(&pool, |tx| meta.missing_micros(tx, iter::empty(), None))
            .unwrap()
            .unwrap();
        assert_eq!(missing_micros, HashSet::from([removed_descriptor]));
        // If we delete all micro descriptors we have, we should get all.
        rw_tx(&pool, |tx| {
            tx.execute(sql!("DELETE FROM descriptor"), params![])
        })
        .unwrap()
        .unwrap();
        // Now all should be returned; we verify this by checking that the
        // result is present in all_descriptors, which is a superset.
        let missing_micros = read_tx(&pool, |tx| meta.missing_micros(tx, iter::empty(), None))
            .unwrap()
            .unwrap();
        // This is a superset of missing_micros because it includes micro
        // descriptors that are not a part of the current consensus.
        let all_descriptors = testdata2::current_micro_descs()
            .iter()
            .map(|x| Sha256::digest(x.1.as_bytes()))
            .collect::<HashSet<_>>();
        assert!(
            missing_micros
                .iter()
                .all(|sha2| all_descriptors.contains(sha2))
        );
    }
}