1
//! Directory Mirror Operation.
2
//!
3
//! # Specifications
4
//!
5
//! * [Directory cache operation](https://spec.torproject.org/dir-spec/directory-cache-operation.html).
6
//!
7
//! # Rationale
8
//!
9
//! This module implements the "core operation" of a directory mirror.
10
//! "Core operation" primarily refers to the logic involved in downloading
11
//! network documents from an upstream authority and inserting them into the
12
//! database.  This module notably **DOES NOT** provide any public (in the HTTP
13
//! sense) endpoints for querying documents.  This is purposely behind a different
14
//! module, so that the directory authority implementation can also make use of it.
15
//! You can think of this module as the one implementing the things unique
16
//! to directory mirrors.
17

            
18
use std::{net::SocketAddr, time::Duration};
19

            
20
use r2d2::Pool;
21
use r2d2_sqlite::SqliteConnectionManager;
22
use rand::Rng;
23
use rusqlite::{named_params, params, OptionalExtension, Transaction};
24
use strum::IntoEnumIterator;
25
use tor_basic_utils::RngExt;
26
use tor_dirclient::request::AuthCertRequest;
27
use tor_dircommon::{
28
    authority::AuthorityContacts,
29
    config::{DirTolerance, DownloadScheduleConfig},
30
};
31
use tor_error::internal;
32
use tor_llcrypto::pk::rsa::RsaIdentity;
33
use tor_netdoc::{
34
    doc::{
35
        authcert::{AuthCert, AuthCertKeyIds, AuthCertSigned},
36
        netstatus::ConsensusFlavor,
37
    },
38
    parse2::{self, NetdocSigned, ParseInput},
39
};
40
use tracing::warn;
41

            
42
use crate::{
43
    database::{self, sql, ContentEncoding, Timestamp},
44
    err::{DatabaseError, FatalError, NetdocRequestError},
45
    mirror::operation::download::DownloadManager,
46
};
47

            
48
mod download;
49

            
50
/// Obtains the most recent valid consensus from the database.
51
///
52
/// This function queries the database using a [`Transaction`] in order to have
53
/// a consistent view upon it.  It will return an [`Option`] containing various
54
/// consensus related timestamps plus the raw consensus itself (more on this
55
/// below).  In order to obtain a *valid* consensus, a [`Timestamp`] plus a
56
/// [`DirTolerance`] is supplied, which will be used for querying the datbaase.
57
///
58
/// # The [`Ok`] Return Value
59
///
60
/// In the [`Some`] case, the return value is composed of the following:
61
/// 1. The `valid-after` timestamp represented by a [`Timestamp`].
62
/// 2. The `fresh-until` timestamp represented by a [`Timestamp`].
63
/// 3. The `valid-until` timestamp represented by a [`Timestamp`].
64
/// 4. The raw consensus reprented by a [`String`].
65
///
66
/// The [`None`] case implies that no valid recent consensus has been found,
67
/// that is, no consensus at all or no consensus whose `valid-before` or
68
/// `valid-after` lies within the range composed by `now` and `tolerance`.
69
18
fn get_recent_consensus(
70
18
    tx: &Transaction,
71
18
    flavor: ConsensusFlavor,
72
18
    tolerance: &DirTolerance,
73
18
    now: Timestamp,
74
18
) -> Result<Option<(Timestamp, Timestamp, Timestamp, String)>, DatabaseError> {
75
    // Select the most recent flavored consensus document from the database.
76
    //
77
    // The `valid_after` and `valid_until` cells must be a member of the range:
78
    // `[valid_after - pre_valid_tolerance; valid_after + post_valid_tolerance]`
79
    // (inclusively).
80
    //
81
    // The query parameters being:
82
    // ?1: The consensus flavor as a String.
83
    // ?2: `now` as a Unix timestamp.
84
18
    let mut meta_stmt = tx.prepare_cached(sql!(
85
18
        "
86
18
        SELECT c.valid_after, c.fresh_until, c.valid_until, s.content
87
18
        FROM
88
18
          consensus AS c
89
18
          INNER JOIN store AS s ON s.docid = c.docid
90
18
        WHERE
91
18
          flavor = ?1
92
18
          AND ?2 >= valid_after - ?3
93
18
          AND ?2 <= valid_until + ?4
94
18
        ORDER BY valid_after DESC
95
18
        LIMIT 1
96
18
        "
97
18
    ))?;
98

            
99
    // Actually execute the query; a None is totally valid and considered as
100
    // no consensus being present in the current database.
101
18
    let res = meta_stmt
102
18
        .query_one(
103
18
            params![
104
18
                flavor.name(),
105
                now,
106
18
                tolerance
107
18
                    .pre_valid_tolerance()
108
18
                    .as_secs()
109
18
                    .try_into()
110
18
                    .unwrap_or(i64::MAX),
111
18
                tolerance
112
18
                    .post_valid_tolerance()
113
18
                    .as_secs()
114
18
                    .try_into()
115
18
                    .unwrap_or(i64::MAX)
116
            ],
117
10
            |row| {
118
                Ok((
119
10
                    row.get::<_, Timestamp>(0)?,
120
10
                    row.get::<_, Timestamp>(1)?,
121
10
                    row.get::<_, Timestamp>(2)?,
122
10
                    row.get::<_, Vec<u8>>(3)?,
123
                ))
124
10
            },
125
        )
126
18
        .optional()?;
127

            
128
18
    let (valid_after, fresh_until, valid_until, consensus) = match res {
129
10
        Some(res) => res,
130
8
        None => return Ok(None),
131
    };
132

            
133
10
    let consensus =
134
10
        String::from_utf8(consensus).map_err(|e| internal!("utf-8 contraint violated? {e}"))?;
135

            
136
10
    Ok(Some((valid_after, fresh_until, valid_until, consensus)))
137
18
}
138

            
139
/// Obtain the most recently published and valid certificate for each authority.
140
///
141
/// Returns the found [`AuthCert`] items as well as the missing [`AuthCertKeyIds`]
142
/// not present within the database.
143
///
144
/// # Performance
145
///
146
/// This function has a performance between `O(n * log n)` and `O(n^2)` because
147
/// it performs `signatories.len()` database queries, which each database query
148
/// potentially taking something between `O(log n)` to `O(n)` to execute.
149
/// However, given that this respective value is oftentimes fairly small, it
150
/// should not be much of a big concern.  However, interfacing code shall ensure
151
/// that it is not **too big** either, because a MITM may add lots of garbage
152
/// signatures, just to make this larger, as it is usually called within the
153
/// context of obtaining the certificates for a given consensus.
154
///
155
/// Because the database has the invariance that all entires inside are
156
/// valid, we do not bother about validating the signatures there again, hence
157
/// why the return type is not [`AuthCertSigned`].
158
6
fn get_recent_authority_certificates(
159
6
    tx: &Transaction,
160
6
    signatories: &[AuthCertKeyIds],
161
6
    tolerance: &DirTolerance,
162
6
    now: Timestamp,
163
6
) -> Result<(Vec<AuthCert>, Vec<AuthCertKeyIds>), DatabaseError> {
164
    // For every key pair in `signatories`, get the most recent valid cert.
165
    //
166
    // This query selects the most recent timestamp valid certificate from the
167
    // database for a single given key pair.  It means that this query has to be
168
    // executed as many times as there are entires in `signatories`.
169
    //
170
    // Unfortunately, there is no neater way to do this, because the alternative
171
    // would involve using a nested set which SQLite does not support, even with
172
    // the carray extension.  An alternative might be to precompute that string
173
    // and then insert it here using `format!` but that feels hacky, error- and
174
    // injection-prone.
175
    //
176
    // Parameters:
177
    // :id_rsa: The RSA identity key fingerprint in uppercase hexadecimal.
178
    // :sk_rsa: The RSA signing key fingerprint in uppercase hexadecimal.
179
    // :now: The current system timestamp.
180
    // :pre_tolerance: The tolerance for not-yet-valid certificates.
181
    // :post_tolerance: The tolerance for expired certificates.
182
6
    let mut stmt = tx.prepare_cached(sql!(
183
6
        "
184
6
        SELECT s.content
185
6
        FROM
186
6
          authority_key_certificate AS a
187
6
          INNER JOIN store AS s ON s.docid = a.docid
188
6
        WHERE
189
6
          (:id_rsa, :sk_rsa) = (a.kp_auth_id_rsa_sha1, a.kp_auth_sign_rsa_sha1)
190
6
          AND :now >= a.dir_key_published - :pre_tolerance
191
6
          AND :now <= a.dir_key_expires + :post_tolerance
192
6
        ORDER BY dir_key_published DESC
193
6
        LIMIT 1
194
6
        "
195
6
    ))?;
196

            
197
    // Keep track of the found (and parsed) certificates and the missing ones.
198
6
    let mut found = Vec::new();
199
6
    let mut missing = Vec::new();
200

            
201
    // Iterate over every key pair and query it, adding it to found if it exists
202
    // and was parsed successfully or to missing if it does not exist within the
203
    // database.
204
14
    for kp in signatories {
205
        // Query the certificate from the database.
206
8
        let raw_cert = stmt
207
8
            .query_one(
208
8
                named_params! {
209
8
                    ":id_rsa": kp.id_fingerprint.as_hex_upper(),
210
8
                    ":sk_rsa": kp.sk_fingerprint.as_hex_upper(),
211
8
                    ":now": now,
212
8
                    ":pre_tolerance": tolerance.pre_valid_tolerance().as_secs().try_into().unwrap_or(i64::MAX),
213
8
                    ":post_tolerance": tolerance.post_valid_tolerance().as_secs().try_into().unwrap_or(i64::MAX),
214
                },
215
4
                |row| row.get::<_, Vec<u8>>(0),
216
            )
217
8
            .optional()?;
218

            
219
        // Unwrap the Some (or None).
220
8
        let raw_cert = match raw_cert {
221
4
            Some(c) => {
222
4
                String::from_utf8(c).map_err(|e| internal!("utf-8 constraint violation? {e}"))?
223
            }
224
            None => {
225
4
                missing.push(*kp);
226
4
                continue;
227
            }
228
        };
229

            
230
        // This match statement is a bit tricky, but important.
231
        //
232
        // In the case that some newer version of arti may not be able to parse
233
        // an older certificate, such as due to missing a field or an older
234
        // version having inserted it because not knowing about it, we must not
235
        // fail.  Instead, we mark the certificate as missing, because it is
236
        // not usable for us.
237
4
        let cert = parse2::parse_netdoc::<AuthCertSigned>(&ParseInput::new(&raw_cert, ""));
238
4
        match cert {
239
2
            Ok(cert) => {
240
2
                // Mark the cert as found.
241
2
                // We assume all certificates in the database to be
242
2
                // cryptographically valid, hence why we use unwrap_unverified.
243
2
                found.push(cert.unwrap_unverified().0);
244
2
            }
245
2
            Err(e) => {
246
2
                warn!("invalid authcert found in database? {e}");
247
2
                missing.push(*kp);
248
2
                continue;
249
            }
250
        }
251
    }
252

            
253
6
    Ok((found, missing))
254
6
}
255

            
256
/// Downloads (missing) directory authority certificates from an authority.
257
///
258
/// The key pairs (identity and signing keys) are specified in `missing`.
259
/// This function will then use the [`DownloadManager`] to download
260
/// the missing certificates from a directory authority.
261
2
async fn download_authority_certificates<'a, 'b, R: Rng>(
262
2
    missing: &[AuthCertKeyIds],
263
2
    downloader: &DownloadManager<'a, 'b>,
264
2
    preferred: Option<&'a Vec<SocketAddr>>,
265
2
    rng: &mut R,
266
2
) -> Result<(&'a Vec<SocketAddr>, String), NetdocRequestError> {
267
2
    let mut requ = AuthCertRequest::new();
268
2
    missing.iter().for_each(|kp| requ.push(*kp));
269

            
270
2
    let (preferred, resp) = downloader
271
2
        .download(&requ, preferred, rng)
272
2
        .await
273
2
        .map_err(NetdocRequestError::Download)?;
274
2
    let resp = String::from_utf8(resp)?;
275

            
276
2
    Ok((preferred, resp))
277
2
}
278

            
279
/// Parses multiple raw directory authority certificates.
280
///
281
/// Returns the parsed [`AuthCertSigned`] alongside their raw plain-text
282
/// representation.
283
2
fn parse_authority_certificates<'a>(
284
2
    certs: &'a str,
285
2
) -> Result<Vec<(AuthCertSigned, &'a str)>, parse2::ParseError> {
286
2
    parse2::parse_netdoc_multiple_with_offsets::<AuthCertSigned>(&ParseInput::new(certs, ""))?
287
2
        .into_iter()
288
        // Creating the slice is fine, parse2 guarantees it is in-bounds.
289
3
        .map(|(cert, start, end)| Ok((cert, &certs[start..end])))
290
2
        .collect()
291
2
}
292

            
293
/// Verifies multiple raw directory authority certificates.
294
///
295
/// Returns the verified [`AuthCertSigned`] values as [`AuthCert`] values.
296
/// The [`str`] slice will remain unmodified, meaning that it will still include
297
/// the signature parts in plain-text.
298
/// This function is mostly used in conjunction with
299
/// [`parse_authority_certificates()`] in order to ensure its outputs were
300
/// correct.
301
2
fn verify_authority_certificates<'a>(
302
2
    certs: Vec<(AuthCertSigned, &'a str)>,
303
2
    v3idents: &[RsaIdentity],
304
2
    tolerance: &DirTolerance,
305
2
    now: Timestamp,
306
2
) -> Result<Vec<(AuthCert, &'a str)>, parse2::VerifyFailed> {
307
2
    certs
308
2
        .into_iter()
309
3
        .map(|(cert, raw)| {
310
2
            cert.verify_self_signed(
311
2
                v3idents,
312
2
                tolerance.pre_valid_tolerance(),
313
2
                tolerance.post_valid_tolerance(),
314
2
                now.into(),
315
            )
316
2
            .map(|cert| (cert, raw))
317
2
        })
318
2
        .collect()
319
2
}
320

            
321
/// Inserts the verified certificates into the database.
322
///
323
/// This function is mostly used in conjunction with
324
/// [`verify_authority_certificates()`] in order to make the data persistent
325
/// to disk.
326
2
fn insert_authority_certificates(
327
2
    tx: &Transaction,
328
2
    certs: &[(AuthCert, &str)],
329
2
) -> Result<(), DatabaseError> {
330
    // Inserts an authority certificate into the meta table.
331
    //
332
    // Parameters:
333
    // :docid - The docid as found in the store table.
334
    // :id_rsa - The identity key fingerprint.
335
    // :sign_rsa - The signing key fingerprint.
336
    // :published - The published timestamp.
337
    // :expires - The expires timestamp.
338
2
    let mut stmt = tx.prepare_cached(sql!(
339
2
        "
340
2
        INSERT INTO authority_key_certificate
341
2
          (docid, kp_auth_id_rsa_sha1, kp_auth_sign_rsa_sha1, dir_key_published, dir_key_expires)
342
2
        VALUES
343
2
          (:docid, :id_rsa, :sign_rsa, :published, :expires)
344
2
        "
345
2
    ))?;
346

            
347
    // Compress and insert all certificates into the store within the context of
348
    // our (still pending) transaction.  Keep track of the uncompressed docid
349
    // too.
350
2
    let certs = certs
351
2
        .iter()
352
3
        .map(|(cert, raw)| {
353
            // For now, we encode the authcerts in all encodings.
354
            // TODO: This is probably not a good idea, but it will also not be
355
            // the end of the world if we change this later -- at worst, clients
356
            // will simply get it in a different encoding they prefer less, but
357
            // that should not be super critical.
358
2
            let docid = database::store_insert(tx, raw.as_bytes(), ContentEncoding::iter())?;
359
2
            Ok::<_, DatabaseError>((docid, cert))
360
2
        })
361
2
        .collect::<Result<Vec<_>, _>>()?;
362

            
363
    // Insert every certificate, after it has been inserted into the store, into
364
    // the authority certificates meta table.
365
4
    for (docid, cert) in certs {
366
2
        stmt.execute(named_params! {
367
2
            ":docid": docid,
368
2
            ":id_rsa": cert.fingerprint.as_hex_upper(),
369
2
            ":sign_rsa": cert.dir_signing_key.to_rsa_identity().as_hex_upper(),
370
2
            ":published": Timestamp::from(cert.dir_key_published.0),
371
2
            ":expires": Timestamp::from(cert.dir_key_expires.0),
372
2
        })?;
373
    }
374

            
375
2
    Ok(())
376
2
}
377

            
378
/// Calculates the [`Duration`] to wait before querying the authorities again.
379
///
380
/// This function accepts a `fresh-until` and `valid-until`, both hopefully
381
/// obtained through the [`get_recent_consensus()`] function, and caculates a
382
/// [`Duration`] relative to `now`, describing the time to wait before querying
383
/// the authorities again.
384
///
385
/// If [`get_recent_consensus()`] returned [`None`], it is safe to skip a call
386
/// to this function and use [`Duration::ZERO`] instead.
387
///
388
/// # Specifications
389
///
390
/// * <https://spec.torproject.org/dir-spec/directory-cache-operation.html#download-ns-from-auth>
391
///
392
/// TODO DIRMIRROR: Consider not naming this timeout but something like
393
/// "download interval" or "poll interval".
394
20000
fn calculate_sync_timeout<R: Rng>(
395
20000
    fresh_until: Timestamp,
396
20000
    valid_until: Timestamp,
397
20000
    now: Timestamp,
398
20000
    rng: &mut R,
399
20000
) -> Duration {
400
20000
    assert!(fresh_until < valid_until);
401

            
402
20000
    let offset = rng
403
20000
        .gen_range_checked(0..=((valid_until - fresh_until).as_secs() / 2))
404
20000
        .expect("invalid range???");
405

            
406
    // fresh_until + offset - now
407
20000
    fresh_until + Duration::from_secs(offset) - now
408
20000
}
409

            
410
/// Runs forever in the current task, performing the core operation of a directory mirror.
411
///
412
/// This function runs forever in the current task, continously downloading
413
/// network documents from authorities and inserting them into the database,
414
/// while also performing garbage collection.
415
///
416
/// A core principle of this function is to be safe against power-losses, sudden
417
/// abortions, and such.  This means that re-starting this function will resume
418
/// seaminglessly from where it stopped.
419
///
420
/// # Algorithm
421
///
422
/// 1. Call [`get_recent_consensus()`] to obtain the most recent and non-expired
423
///    consensus from the database.
424
///     1. If the call returns [`Some`], goto (2).
425
///     2. If the call returns [`None`], goto (3).
426
/// 2. Spawn a task backed by [`tokio::time::timeout()`] whose purpose it is to
427
///    determine all missing network documents referred to by the current
428
///    consensus and scheduling downloads from the authorities in order to
429
///    obtain and insert them into the database.
430
///    TODO DIRMIRROR: Impose a timeout for each download attempt.
431
/// 3. Download a new consensus from the directory authorities and insert it into
432
///    the database.
433
/// 4. Perform a cycle of garbage collection.
434
/// 5. Goto (1).
435
///
436
/// # Specifications
437
///
438
/// * <https://spec.torproject.org/dir-spec/directory-cache-operation.html#download-desc-from-auth>
439
/// * <https://spec.torproject.org/dir-spec/client-operation.html#retrying-failed-downloads>
440
pub(super) async fn serve<R: Rng, F: Fn() -> Timestamp>(
441
    pool: Pool<SqliteConnectionManager>,
442
    flavor: ConsensusFlavor,
443
    _authorities: AuthorityContacts,
444
    _schedule: DownloadScheduleConfig,
445
    tolerance: DirTolerance,
446
    rng: &mut R,
447
    now_fn: F,
448
) -> Result<(), FatalError> {
449
    loop {
450
        let now = now_fn();
451

            
452
        // (1) Call get_recent_consensus() to obtain the most recent and non-expired
453
        // consensus from the database.
454
        // TODO: Use `Result::flatten` once MSRV is 1.89.0.
455
        let res = database::read_tx(&pool, {
456
            let tolerance = tolerance.clone();
457
            move |tx| get_recent_consensus(tx, flavor, &tolerance, now)
458
        });
459
        let res = match res {
460
            Ok(Ok(res)) => res,
461
            Err(e) | Ok(Err(e)) => {
462
                return Err(FatalError::ConsensusSelection(e));
463
            }
464
        };
465

            
466
        // (1.1) If the call returns Some, goto (2).
467
        if let Some((_valid_after, fresh_until, valid_until, _consensus)) = res {
468
            // (2) Run a closure backed by tokio::time::timeout() with a lifetime
469
            // returned by by calculate_sync_timeout, whose purpose it is to
470
            // determine all missing network documents reffered to by the current
471
            // consensus and scheduling downloads from the authorities in order
472
            // to obtain and insert them into the database.
473
            let sync_timeout = calculate_sync_timeout(fresh_until, valid_until, now, rng);
474
            tokio::time::timeout(sync_timeout, async {
475
                // TODO DIRMIRROR: Actually download descriptors.
476
                // Ensure a good timeout to protect against malicious
477
                // authorities transmitting data extra slow; a download should
478
                // probably not take `sync_timeout` but a few minutes instead.
479
                // This implies a nested timeout.  The `sync_timeout` for the
480
                // outer layer and a smaller timeout for each download, in order
481
                // to ensure that each download actually gets the same amount
482
                // of time to exist, instead of just the first ones having
483
                // the full-time, with subsequent ones having a smaller and
484
                // smaller time.
485
                let _ = std::future::pending::<()>().await;
486
            })
487
            .await
488
            .expect_err("std::future::pending returned?");
489
        }
490

            
491
        // (3) Download a new consensus from the directory authorities and insert
492
        // it into the database.
493
        //
494
        // At this stage we also have to ask ourselves what happens in the highly
495
        // unlikely but still possible case that a new consensus could not be
496
        // retrieved, due to connectivity-loss (actually likely) or the even
497
        // more unlikely case of the directory authorities all being down and/or
498
        // unable to compute a new consensus.
499
        //
500
        // The specification is fairly clean on that (see the link above).
501
        // Downloads are retried with a variation of the "decorrelated jitter"
502
        // algorithm; that is, determining a certain amount of time before trying
503
        // again from an authority we have not tried yet.
504
        //
505
        // TODO: Actually download from authorities.
506

            
507
        // (4) Perform a cycle of garbage collection.
508
        // TODO: Actually perform a cycle of garbage collection.
509
    }
510
}
511

            
512
#[cfg(test)]
513
mod test {
514
    // @@ begin test lint list maintained by maint/add_warning @@
515
    #![allow(clippy::bool_assert_comparison)]
516
    #![allow(clippy::clone_on_copy)]
517
    #![allow(clippy::dbg_macro)]
518
    #![allow(clippy::mixed_attributes_style)]
519
    #![allow(clippy::print_stderr)]
520
    #![allow(clippy::print_stdout)]
521
    #![allow(clippy::single_char_pattern)]
522
    #![allow(clippy::unwrap_used)]
523
    #![allow(clippy::unchecked_time_subtraction)]
524
    #![allow(clippy::useless_vec)]
525
    #![allow(clippy::needless_pass_by_value)]
526
    //! <!-- @@ end test lint list maintained by maint/add_warning @@ -->
527
    use std::time::SystemTime;
528

            
529
    use crate::database::{self, DocumentId};
530

            
531
    use super::*;
532
    use lazy_static::lazy_static;
533
    use tokio::{
534
        io::{AsyncReadExt, AsyncWriteExt},
535
        net::TcpListener,
536
    };
537
    use tor_basic_utils::test_rng::testing_rng;
538
    use tor_dircommon::{authority::AuthorityContactsBuilder, config::DirToleranceBuilder};
539
    use tor_llcrypto::pk::rsa::RsaIdentity;
540
    use tor_rtcompat::PreferredRuntime;
541

            
542
    lazy_static! {
543
    /// Wed Jan 01 2020 00:00:00 GMT+0000
544
    static ref VALID_AFTER: Timestamp =
545
        (SystemTime::UNIX_EPOCH + Duration::from_secs(1577836800)).into();
546

            
547
    /// Wed Jan 01 2020 01:00:00 GMT+0000
548
    static ref FRESH_UNTIL: Timestamp =
549
        *VALID_AFTER + Duration::from_secs(60 * 60);
550

            
551
    /// Wed Jan 01 2020 02:00:00 GMT+0000
552
    static ref FRESH_UNTIL_HALF: Timestamp =
553
        *FRESH_UNTIL + Duration::from_secs(60 * 60);
554

            
555
    /// Wed Jan 01 2020 03:00:00 GMT+0000
556
    static ref VALID_UNTIL: Timestamp =
557
        *FRESH_UNTIL + Duration::from_secs(60 * 60 * 2);
558
    }
559

            
560
    const CONSENSUS_CONTENT: &str = "Lorem ipsum dolor sit amet.";
561
    const CERT_CONTENT: &[u8] = include_bytes!("../../testdata/authcert-longclaw");
562

            
563
    lazy_static! {
564
        static ref CONSENSUS_DOCID: DocumentId = DocumentId::digest(CONSENSUS_CONTENT.as_bytes());
565
        static ref CERT_DOCID: DocumentId = DocumentId::digest(CERT_CONTENT);
566
    }
567

            
568
    fn create_dummy_db() -> Pool<SqliteConnectionManager> {
569
        let pool = database::open("").unwrap();
570
        database::rw_tx(&pool, |tx| {
571
            tx.execute(
572
                sql!("INSERT INTO store (docid, content) VALUES (?1, ?2)"),
573
                params![*CONSENSUS_DOCID, CONSENSUS_CONTENT.as_bytes()],
574
            )
575
            .unwrap();
576
            tx.execute(
577
                sql!("INSERT INTO store (docid, content) VALUES (?1, ?2)"),
578
                params![*CERT_DOCID, CERT_CONTENT],
579
            )
580
            .unwrap();
581

            
582
            tx.execute(
583
                sql!(
584
                    "
585
                    INSERT INTO consensus
586
                    (docid, unsigned_sha3_256, flavor, valid_after, fresh_until, valid_until)
587
                    VALUES
588
                    (?1, ?2, ?3, ?4, ?5, ?6)
589
                    "
590
                ),
591
                params![
592
                    *CONSENSUS_DOCID,
593
                    "0000000000000000000000000000000000000000000000000000000000000000", // not the correct hash
594
                    ConsensusFlavor::Plain.name(),
595
                    *VALID_AFTER,
596
                    *FRESH_UNTIL,
597
                    *VALID_UNTIL,
598
                ],
599
            )
600
            .unwrap();
601

            
602
            tx.execute(sql!(
603
                "
604
                INSERT INTO authority_key_certificate
605
                  (docid, kp_auth_id_rsa_sha1, kp_auth_sign_rsa_sha1, dir_key_published, dir_key_expires)
606
                VALUES
607
                  (:docid, :id_rsa, :sk_rsa, :published, :expires)
608
                "
609
                ),
610
                named_params! {
611
                ":docid": *CERT_DOCID,
612
                ":id_rsa": "49015F787433103580E3B66A1707A00E60F2D15B",
613
                ":sk_rsa": "C5D153A6F0DA7CC22277D229DCBBF929D0589FE0",
614
                ":published": 1764543578,
615
                ":expires": 1772492378,
616
            }).unwrap();
617
        })
618
        .unwrap();
619

            
620
        pool
621
    }
622

            
623
    #[test]
624
    fn recent_consensus() {
625
        let pool = create_dummy_db();
626
        let no_tolerance = DirToleranceBuilder::default()
627
            .pre_valid_tolerance(Duration::ZERO)
628
            .post_valid_tolerance(Duration::ZERO)
629
            .build()
630
            .unwrap();
631
        let liberal_tolerance = DirToleranceBuilder::default()
632
            .pre_valid_tolerance(Duration::from_secs(60 * 60)) // 1h before
633
            .post_valid_tolerance(Duration::from_secs(60 * 60)) // 1h after
634
            .build()
635
            .unwrap();
636

            
637
        database::read_tx(&pool, move |tx| {
638
            // Get None by being way before valid-after.
639
            assert!(get_recent_consensus(
640
                tx,
641
                ConsensusFlavor::Plain,
642
                &no_tolerance,
643
                SystemTime::UNIX_EPOCH.into(),
644
            )
645
            .unwrap()
646
            .is_none());
647

            
648
            // Get None by being way behind valid-until.
649
            assert!(get_recent_consensus(
650
                tx,
651
                ConsensusFlavor::Plain,
652
                &no_tolerance,
653
                *VALID_UNTIL + Duration::from_secs(60 * 60 * 24 * 365),
654
            )
655
            .unwrap()
656
            .is_none());
657

            
658
            // Get None by being minimally before valid-after.
659
            assert!(get_recent_consensus(
660
                tx,
661
                ConsensusFlavor::Plain,
662
                &no_tolerance,
663
                *VALID_AFTER - Duration::from_secs(1),
664
            )
665
            .unwrap()
666
            .is_none());
667

            
668
            // Get None by being minimally behind valid-until.
669
            assert!(get_recent_consensus(
670
                tx,
671
                ConsensusFlavor::Plain,
672
                &no_tolerance,
673
                *VALID_UNTIL + Duration::from_secs(1),
674
            )
675
            .unwrap()
676
            .is_none());
677

            
678
            // Get a valid consensus by being in the interval.
679
            let res1 =
680
                get_recent_consensus(tx, ConsensusFlavor::Plain, &no_tolerance, *VALID_AFTER)
681
                    .unwrap()
682
                    .unwrap();
683
            let res2 =
684
                get_recent_consensus(tx, ConsensusFlavor::Plain, &no_tolerance, *VALID_UNTIL)
685
                    .unwrap()
686
                    .unwrap();
687
            let res3 = get_recent_consensus(
688
                tx,
689
                ConsensusFlavor::Plain,
690
                &no_tolerance,
691
                *VALID_AFTER + Duration::from_secs(60 * 30),
692
            )
693
            .unwrap()
694
            .unwrap();
695
            assert_eq!(
696
                res1,
697
                (
698
                    *VALID_AFTER,
699
                    *FRESH_UNTIL,
700
                    *VALID_UNTIL,
701
                    CONSENSUS_CONTENT.to_string(),
702
                )
703
            );
704
            assert_eq!(res1, res2);
705
            assert_eq!(res2, res3);
706

            
707
            // Get a valid consensus using a liberal dir tolerance.
708
            let res1 = get_recent_consensus(
709
                tx,
710
                ConsensusFlavor::Plain,
711
                &liberal_tolerance,
712
                *VALID_AFTER - Duration::from_secs(60 * 30),
713
            )
714
            .unwrap()
715
            .unwrap();
716
            let res2 = get_recent_consensus(
717
                tx,
718
                ConsensusFlavor::Plain,
719
                &liberal_tolerance,
720
                *VALID_UNTIL + Duration::from_secs(60 * 30),
721
            )
722
            .unwrap()
723
            .unwrap();
724
            assert_eq!(
725
                res1,
726
                (
727
                    *VALID_AFTER,
728
                    *FRESH_UNTIL,
729
                    *VALID_UNTIL,
730
                    CONSENSUS_CONTENT.to_string(),
731
                )
732
            );
733
            assert_eq!(res1, res2);
734
        })
735
        .unwrap();
736
    }
737

            
738
    #[test]
739
    fn sync_timeout() {
740
        // We repeat the tests a few thousand times to go over many random values.
741
        for _ in 0..10000 {
742
            let now = (SystemTime::UNIX_EPOCH + Duration::from_secs(42)).into();
743

            
744
            let dur = calculate_sync_timeout(*FRESH_UNTIL, *VALID_UNTIL, now, &mut testing_rng());
745
            assert!(dur >= *FRESH_UNTIL - now);
746
            assert!(dur <= *FRESH_UNTIL_HALF - now);
747
        }
748
    }
749

            
750
    #[test]
751
    fn get_auth_cert() {
752
        let pool = create_dummy_db();
753

            
754
        // Empty.
755
        let (found, missing) = database::read_tx(&pool, |tx| {
756
            get_recent_authority_certificates(
757
                tx,
758
                &[],
759
                &DirTolerance::default(),
760
                (SystemTime::UNIX_EPOCH + Duration::from_secs(1765900013)).into(),
761
            )
762
        })
763
        .unwrap()
764
        .unwrap();
765
        assert!(found.is_empty());
766
        assert!(missing.is_empty());
767

            
768
        // Find one and two missing ones.
769
        let (found, missing) = database::read_tx(&pool, |tx| {
770
            get_recent_authority_certificates(
771
                tx,
772
                &[
773
                    // Found one.
774
                    AuthCertKeyIds {
775
                        id_fingerprint: RsaIdentity::from_hex(
776
                            "49015F787433103580E3B66A1707A00E60F2D15B",
777
                        )
778
                        .unwrap(),
779
                        sk_fingerprint: RsaIdentity::from_hex(
780
                            "C5D153A6F0DA7CC22277D229DCBBF929D0589FE0",
781
                        )
782
                        .unwrap(),
783
                    },
784
                    // Missing.
785
                    AuthCertKeyIds {
786
                        id_fingerprint: RsaIdentity::from_hex(
787
                            "0000000000000000000000000000000000000000",
788
                        )
789
                        .unwrap(),
790
                        sk_fingerprint: RsaIdentity::from_hex(
791
                            "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF",
792
                        )
793
                        .unwrap(),
794
                    },
795
                    // Missing.
796
                    AuthCertKeyIds {
797
                        id_fingerprint: RsaIdentity::from_hex(
798
                            "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF",
799
                        )
800
                        .unwrap(),
801
                        sk_fingerprint: RsaIdentity::from_hex(
802
                            "0000000000000000000000000000000000000000",
803
                        )
804
                        .unwrap(),
805
                    },
806
                ],
807
                &DirTolerance::default(),
808
                (SystemTime::UNIX_EPOCH + Duration::from_secs(1765900013)).into(),
809
            )
810
        })
811
        .unwrap()
812
        .unwrap();
813
        assert_eq!(found.len(), 1);
814
        assert_eq!(missing.len(), 2);
815

            
816
        // Now make one invalid and see that it will get set to missing.
817
        pool.get()
818
            .unwrap()
819
            .execute(
820
                sql!(
821
                    "
822
                    UPDATE store
823
                    SET content = X'61'
824
                    WHERE docid = (SELECT docid FROM authority_key_certificate)
825
                    "
826
                ),
827
                params![],
828
            )
829
            .unwrap();
830
        let (found, missing) = database::read_tx(&pool, |tx| {
831
            get_recent_authority_certificates(
832
                tx,
833
                &[
834
                    // Found one.
835
                    AuthCertKeyIds {
836
                        id_fingerprint: RsaIdentity::from_hex(
837
                            "49015F787433103580E3B66A1707A00E60F2D15B",
838
                        )
839
                        .unwrap(),
840
                        sk_fingerprint: RsaIdentity::from_hex(
841
                            "C5D153A6F0DA7CC22277D229DCBBF929D0589FE0",
842
                        )
843
                        .unwrap(),
844
                    },
845
                ],
846
                &DirTolerance::default(),
847
                (SystemTime::UNIX_EPOCH + Duration::from_secs(1765900013)).into(),
848
            )
849
        })
850
        .unwrap()
851
        .unwrap();
852
        assert!(found.is_empty());
853
        assert_eq!(
854
            missing[0],
855
            AuthCertKeyIds {
856
                id_fingerprint: RsaIdentity::from_hex("49015F787433103580E3B66A1707A00E60F2D15B",)
857
                    .unwrap(),
858
                sk_fingerprint: RsaIdentity::from_hex("C5D153A6F0DA7CC22277D229DCBBF929D0589FE0",)
859
                    .unwrap(),
860
            }
861
        );
862
    }
863

            
864
    /// Tests the combination of the following functions:
865
    ///
866
    /// * [`download_authority_certificates()`]
867
    /// * [`verify_authority_certificates()`]
868
    /// * [`insert_authority_certificates()`]
869
    #[tokio::test]
870
    async fn missing_certificates() {
871
        // Don't use the dummy db because we will download.
872
        let pool = database::open("").unwrap();
873

            
874
        // Create server.
875
        let listener = TcpListener::bind("[::]:0").await.unwrap();
876
        let sa = listener.local_addr().unwrap();
877
        tokio::spawn(async move {
878
            let raw_cert = include_str!("../../testdata/authcert-longclaw");
879
            let resp = format!(
880
                "HTTP/1.1 200 Ok\r\nContent-Length: {}\r\n\r\n{raw_cert}",
881
                raw_cert.len()
882
            )
883
            .as_bytes()
884
            .to_vec();
885

            
886
            let (mut stream, _) = listener.accept().await.unwrap();
887
            let mut buf = vec![0; 1024];
888
            let _ = stream.read(&mut buf).await.unwrap();
889
            stream.write_all(&resp).await.unwrap();
890
            stream.flush().await.unwrap();
891
        });
892

            
893
        let mut authorities = AuthorityContactsBuilder::default();
894
        authorities.set_v3idents(vec![RsaIdentity::from_hex(
895
            "49015F787433103580E3B66A1707A00E60F2D15B",
896
        )
897
        .unwrap()]);
898
        authorities.set_uploads(vec![]);
899
        authorities.set_downloads(vec![vec![sa]]);
900
        let authorities = authorities.build().unwrap();
901

            
902
        // Download certificate.
903
        let rt = PreferredRuntime::current().unwrap();
904
        let downloader = DownloadManager::new(authorities.downloads(), &rt);
905
        let (preferred, certs_raw) = download_authority_certificates(
906
            &[AuthCertKeyIds {
907
                id_fingerprint: RsaIdentity::from_hex("49015F787433103580E3B66A1707A00E60F2D15B")
908
                    .unwrap(),
909
                sk_fingerprint: RsaIdentity::from_hex("C5D153A6F0DA7CC22277D229DCBBF929D0589FE0")
910
                    .unwrap(),
911
            }],
912
            &downloader,
913
            None,
914
            &mut testing_rng(),
915
        )
916
        .await
917
        .unwrap();
918
        assert_eq!(preferred, &authorities.downloads()[0]);
919
        assert_eq!(certs_raw, include_str!("../../testdata/authcert-longclaw"));
920

            
921
        // Parse certificate.
922
        let certs = parse_authority_certificates(&certs_raw).unwrap();
923
        assert_eq!(certs[0].1, certs_raw);
924

            
925
        // Verify certificate.
926
        let certs = verify_authority_certificates(
927
            certs,
928
            authorities.v3idents(),
929
            &DirTolerance::default(),
930
            (SystemTime::UNIX_EPOCH + Duration::from_secs(1765900013)).into(),
931
        )
932
        .unwrap();
933
        assert_eq!(certs.len(), 1);
934
        assert_eq!(
935
            certs[0].0.fingerprint.0,
936
            RsaIdentity::from_hex("49015F787433103580E3B66A1707A00E60F2D15B").unwrap()
937
        );
938
        assert_eq!(
939
            certs[0].0.dir_signing_key.to_rsa_identity(),
940
            RsaIdentity::from_hex("C5D153A6F0DA7CC22277D229DCBBF929D0589FE0").unwrap()
941
        );
942

            
943
        // Insert the stuff into the database.
944
        database::rw_tx(&pool, |tx| insert_authority_certificates(tx, &certs))
945
            .unwrap()
946
            .unwrap();
947

            
948
        // Verify it is actually there.
949
        let (id_rsa, sign_rsa, published, expires, raw) = database::read_tx(&pool, |tx| {
950
            tx.query_one(
951
                sql!(
952
                    "
953
                    SELECT
954
                      a.kp_auth_id_rsa_sha1, a.kp_auth_sign_rsa_sha1, a.dir_key_published, a.dir_key_expires, s.content
955
                    FROM
956
                      authority_key_certificate AS a
957
                    INNER JOIN
958
                      store AS s ON a.docid = s.docid
959
                    "
960
                ),
961
                params![],
962
                |row| Ok((
963
                    row.get::<_, String>(0)?,
964
                    row.get::<_, String>(1)?,
965
                    row.get::<_, Timestamp>(2)?,
966
                    row.get::<_, Timestamp>(3)?,
967
                    row.get::<_, Vec<u8>>(4)?,
968
                )),
969
            )
970
        })
971
        .unwrap().unwrap();
972

            
973
        assert_eq!(id_rsa, certs[0].0.fingerprint.as_hex_upper());
974
        assert_eq!(
975
            sign_rsa,
976
            certs[0].0.dir_signing_key.to_rsa_identity().as_hex_upper()
977
        );
978
        assert_eq!(published, certs[0].0.dir_key_published.0.into());
979
        assert_eq!(expires, certs[0].0.dir_key_expires.0.into());
980
        assert_eq!(raw, include_bytes!("../../testdata/authcert-longclaw"));
981

            
982
        // Now (just to be sure) verify that compressed stuff also exists.
983
        let count = database::read_tx(&pool, |tx| {
984
            tx.query_one(
985
                sql!(
986
                    "
987
                    SELECT COUNT(*)
988
                    FROM compressed_document
989
                    "
990
                ),
991
                params![],
992
                |row| row.get::<_, i64>(0),
993
            )
994
        })
995
        .unwrap()
996
        .unwrap();
997
        assert_eq!(count, 4);
998
    }
999
}