//! Net document storage backed by sqlite3.

//!

//! We store most objects in sqlite tables, except for very large ones,

//! which we store as "blob" files in a separate directory.

use super::ExpirationConfig;

use crate::docmeta::{AuthCertMeta, ConsensusMeta};

use crate::err::ReadOnlyStorageError;

use crate::storage::{InputString, Store};

use crate::{Error, Result};

use fs_mistrust::CheckedDir;

use tor_basic_utils::PathExt as _;

use tor_error::{internal, into_internal, warn_report};

use tor_netdoc::doc::authcert::AuthCertKeyIds;

use tor_netdoc::doc::microdesc::MdDigest;

use tor_netdoc::doc::netstatus::{ConsensusFlavor, Lifetime};

#[cfg(feature = "routerdesc")]

use tor_netdoc::doc::routerdesc::RdDigest;

use web_time_compat::SystemTimeExt;

#[cfg(feature = "bridge-client")]

pub(crate) use {crate::storage::CachedBridgeDescriptor, tor_guardmgr::bridge::BridgeConfig};

use std::collections::{HashMap, HashSet};

use std::fs::OpenOptions;

use std::path::{Path, PathBuf};

use std::result::Result as StdResult;

use std::sync::Arc;

use std::time::SystemTime;

use fslock_guard::LockFileGuard;

use rusqlite::{OpenFlags, OptionalExtension, Transaction, params};

use time::OffsetDateTime;

use tracing::{trace, warn};

/// Possible status of a lockfile.

///

/// (Sqlite does its own locking, but we would like to cover the blobs directory

/// as well)

enum LockFile {

    /// We are not even trying to lock, but permitting write operations

    /// regardless.

    ///

    /// This is the implementation we use for ephemeral testing databases.

    /// Don't use it in production!

    NotLocking,

    /// We aren't locked.

    ///

    /// The provided path is the path to the lockfile that we will try to open if

    /// we

    Unlocked(PathBuf),

    /// We have the lock.

    ///

    Locked(

        // We never need to read this field; we only need to hold it so that the

        // lock file isn't closed.

        #[allow(unused)] LockFileGuard,

    ),

}

/// Local directory cache using a Sqlite3 connection.

pub(crate) struct SqliteStore {

    /// Connection to the sqlite3 database.

    conn: rusqlite::Connection,

    /// Location for the sqlite3 database; used to reopen it.

    sql_path: Option<PathBuf>,

    /// Location to store blob files.

    blob_dir: CheckedDir,

    /// Lockfile to prevent concurrent write attempts from different

    /// processes.

    ///

    /// If this is LockFile::NotLocking we aren't using a lockfile.  Watch out!

    ///

    /// (sqlite supports that with connection locking, but we want to

    /// be a little more coarse-grained here)

    lockfile: LockFile,

}

/// # Some notes on blob consistency, and the lack thereof.

///

/// We store large documents (currently, consensuses) in separate files,

/// called "blobs",

/// outside of the sqlite database.

/// We do this for performance reasons: for large objects,

/// mmap is far more efficient than sqlite in RAM and CPU.

///

/// In the sqlite database, we keep track of our blobs

/// using the ExtDocs table.

/// This scheme makes it possible for the blobs and the table

/// get out of sync.

///

/// In summary:

///   - _Vanished_ blobs (ones present only in ExtDocs) are possible;

///     we try to tolerate them.

///   - _Orphaned_ blobs (ones present only on the disk) are possible;

///     we try to tolerate them.

///   - _Corrupted_ blobs (ones with the wrong contents) are possible

///     but (we hope) unlikely;

///     we do not currently try to tolerate them.

///

/// In more detail:

///

/// Here are the practices we use when _writing_ blobs:

///

/// - We always create a blob before updating the ExtDocs table,

///   and remove an entry from the ExtDocs before deleting the blob.

/// - If we decide to roll back the transaction that adds the row to ExtDocs,

///   we delete the blob after doing so.

/// - We use [`CheckedDir::write_and_replace`] to store blobs,

///   so a half-formed blob shouldn't be common.

///   (We assume that "close" and "rename" are serialized by the OS,

///   so that _if_ the rename happens, the file is completely written.)

/// - Blob filenames include a digest of the file contents,

///   so collisions are unlikely.

///

/// Here are the practices we use when _deleting_ blobs:

/// - First, we drop the row from the ExtDocs table.

///   Only then do we delete the file.

///

/// These practices can result in _orphaned_ blobs

/// (ones with no row in the ExtDoc table),

/// or in _half-written_ blobs files with tempfile names

/// (which also have no row in the ExtDoc table).

/// This happens if we crash at the wrong moment.

/// Such blobs can be safely removed;

/// we do so in [`SqliteStore::remove_unreferenced_blobs`].

///

/// Despite our efforts, _vanished_ blobs

/// (entries in the ExtDoc table with no corresponding file)

/// are also possible.  They could happen for these reasons:

/// - The filesystem might not serialize or sync things in a way that's

///   consistent with the DB.

/// - An automatic process might remove random cache files.

/// - The user might run around deleting things to free space.

///

/// We try to tolerate vanished blobs.

///

/// _Corrupted_ blobs are also possible.  They can happen on FS corruption,

/// or on somebody messing around with the cache directory manually.

/// We do not attempt to tolerate corrupted blobs.

///

/// ## On trade-offs

///

/// TODO: The practices described above are more likely

/// to create _orphaned_ blobs than _vanished_ blobs.

/// We initially made this trade-off decision on the mistaken theory

/// that we could avoid vanished blobs entirely.

/// We _may_ want to revisit this choice,

/// on the rationale that we can respond to vanished blobs as soon as we notice they're gone,

/// whereas we can only handle orphaned blobs with a periodic cleanup.

/// On the other hand, since we need to handle both cases,

/// it may not matter very much in practice.

#[allow(unused)]

mod blob_consistency {}

/// Specific error returned when a blob will not be read.

///

/// This error is an internal type: it's never returned to the user.

#[derive(Debug)]

enum AbsentBlob {

    /// We did not find a blob file on the disk.

    VanishedFile,

    /// We did not even find a blob to read in ExtDocs.

    NothingToRead,

}

impl SqliteStore {

    /// Construct or open a new SqliteStore at some location on disk.

    /// The provided location must be a directory, or a possible

    /// location for a directory: the directory will be created if

    /// necessary.

    ///

    /// If readonly is true, the result will be a read-only store.

    /// Otherwise, when readonly is false, the result may be

    /// read-only or read-write, depending on whether we can acquire

    /// the lock.

    ///

    /// # Limitations:

    ///

    /// The file locking that we use to ensure that only one dirmgr is

    /// writing to a given storage directory at a time is currently

    /// _per process_. Therefore, you might get unexpected results if

    /// two SqliteStores are created in the same process with the

    /// path.

        } else {

        };

        // Check permissions on the sqlite and lock files; don't require them to

        // exist.

            {

            }

        }

                None => {

                    // We couldn't get the lock.

                }

            }

        } else {

        };

        } else {

        };

    /// Construct a new SqliteStore from a database connection and a location

    /// for blob files.

    ///

    /// Used for testing with a memory-backed database.

    ///

    /// Note: `blob_dir` must not be used for anything other than storing the blobs associated with

    /// this database, since we will freely remove unreferenced files from this directory.

    #[cfg(test)]

    /// Construct a new SqliteStore from a database connection and a location

    /// for blob files.

    ///

    /// The `readonly` argument specifies whether the database connection should be read-only.

        // sqlite (as of Jun 2024) does not enforce foreign keys automatically unless you set this

        // pragma on the connection.

    /// Check whether this database has a schema format we can read, and

    /// install or upgrade the schema if necessary.

        // Update the schema from current_vsn to the latest (does not commit)

            }

            } else {

                // The other process should have created the database!

            }

            } else {

            }

        // rolls back the transaction, but nothing was done.

    /// Read a blob from disk, mapping it if possible.

    ///

    /// Return `Ok(Err(.))` if the file for the blob was not found on disk;

    /// returns an error in other cases.

    ///

    /// (See [`blob_consistency`] for information on why the blob might be absent.)

            Err(fs_mistrust::Error::NotFound(_)) => {

                    "{:?} was listed in the database, but its corresponding file had been deleted",

                    path

                );

            }

        };

                action: "loading",

    /// Write a file to disk as a blob, and record it in the ExtDocs table.

    ///

    /// Return a SavedBlobHandle that describes where the blob is, and which

    /// can be used either to commit the blob or delete it.

    ///

    /// See [`blob_consistency`] for more information on guarantees.

    /// As `latest_consensus`, but do not retry.

                )

        };

            // TODO blobs: If the cache is inconsistent (because this blob is _vanished_), and the cache has not yet

            // been cleaned, this may fail to find the latest consensus that we actually have.

        } else {

        }

    /// Save a blob to disk and commit it.

    #[cfg(test)]

    /// Return the valid-after time for the latest non non-pending consensus,

    #[cfg(test)]

    // We should revise the tests to use latest_consensus_meta instead.

    /// Remove the blob with name `fname`, but do not give an error on failure.

    ///

    /// See [`blob_consistency`]: we should call this only having first ensured

    /// that the blob is removed from the ExtDocs table.

    /// Delete any blob files that are old enough, and not mentioned in the ExtDocs table.

    ///

    /// There shouldn't typically be any, but we don't want to let our cache grow infinitely

    /// if we have a bug.

        // Now, look for any unreferenced blobs that are a bit old.

                action: "getting metadata",

            {

                // this file is sufficiently recent that we should not remove it, just to be cautious.

                    // This filename wasn't utf-8.  We will never create one of these.

                        "Removing bizarre file '{}' from blob store.",

                    );

                }

            };

        }

    /// Remove any entry in the ExtDocs table for which a blob file is vanished.

    ///

    /// This method is `O(n)` in the size of the ExtDocs table and the size of the directory.

    /// It doesn't take self, to avoid problems with the borrow checker.

                // The blob is present; great!

        }

}

impl Store for SqliteStore {

        }

            // This is an ephemeral database with no disk representation.

        };

            LockFile::NotLocking => {

                // This should be unreachable.

            }

        };

        // We aren't locked. Try to fix that.

            // Somebody else has the lock.

        };

        // Open a fresh RW sql connection. If it fails, we'll unlock the guard

        // and remain in our old state.

        // This works around a false positive; see

        //   https://github.com/rust-lang/rust-clippy/issues/8114

        #[allow(clippy::let_and_return)]

        };

        // In theory bad system clocks might generate table rows with times far in the future.

        // However, for data which is cached here which comes from the network consensus,

        // we rely on the fact that no consensus from the future exists, so this can't happen.

        // Bridge descriptors come from bridges and bridges might send crazy times,

        // so we need to discard any that look like they are from the future,

        // since otherwise wrong far-future timestamps might live in our DB indefinitely.

        #[cfg(feature = "bridge-client")]

        // Find all consensus blobs that are no longer referenced,

        // and delete their entries from extdocs.

            // TODO: This query can be O(n); but that won't matter for clients.

            // For relays, we may want to add an index to speed it up, if we use this code there too.

            }

        };

        // Now that the transaction has been committed, these blobs are

        // unreferenced in the ExtDocs table, and we can remove them from disk.

                        e,

                        "Couldn't remove orphaned blob file {}",

                    );

        }

    // Note: We cannot, and do not, call this function when a transaction already exists.

        }

        // We use unchecked_transaction() here because this API takes a non-mutable `SqliteStore`.

        // `unchecked_transaction()` will give an error if it is used

        // when a transaction already exists.

        // That's fine: We don't call this function from inside this module,

        // when a transaction might exist,

        // and we can't call multiple SqliteStore functions at once: it isn't sync.

        // Here we enforce that:

        static_assertions::assert_not_impl_any!(SqliteStore: Sync);

        // If we decide that this is unacceptable,

        // then since sqlite doesn't really support concurrent use of a connection,

        // we _could_ change the Store::latest_consensus API take &mut self,

        // or we could add a mutex,

        // or we could just not use a transaction object.

            Err(AbsentBlob::VanishedFile) => {

            }

        }

        } else {

        }

    #[cfg(test)]

        {

        } else {

        }

        /// How long to keep a consensus around after it has expired

        const CONSENSUS_LIFETIME: time::Duration = time::Duration::days(4);

        // After a few days have passed, a consensus is no good for

        // anything at all, not even diffs.

        // TODO: We should probably remove the blob as well, but for now

        // this is enough.

        // TODO(nickm): Do I need to get a transaction here for performance?

        }

        }

        // TODO(nickm): Should I speed this up with a transaction, or

        // does it not matter for queries?

        }

        }

        }

    #[cfg(feature = "routerdesc")]

        // TODO(nickm): Should I speed this up with a transaction, or

        // does it not matter for queries?

        }

    #[cfg(feature = "routerdesc")]

        }

    #[cfg(feature = "bridge-client")]

    #[cfg(feature = "bridge-client")]

            // Hopefully whoever *does* have the lock will update the cache.

            // Otherwise it will contain a stale entry forever

            // (which we'll ignore, but waste effort on).

            bridge_line,

            entry.document,

        ];

    #[cfg(feature = "bridge-client")]

            // This is called when we find corrupted or stale cache entries,

            // to stop us wasting time on them next time.

            // Hopefully whoever *does* have the lock will do this.

        };

}

/// Functionality related to uncommitted blobs.

mod blob_handle {

    use std::path::{Path, PathBuf};

    use crate::Result;

    use rusqlite::Transaction;

    use tor_basic_utils::PathExt as _;

    use tor_error::warn_report;

    /// Handle to a blob that we have saved to disk but

    /// not yet committed to

    /// the database, and the database transaction where we added a reference to it.

    ///

    /// Used to either commit the blob (by calling [`SavedBlobHandle::commit`]),

    /// or roll it back (by dropping the [`SavedBlobHandle`] without committing it.)

    #[must_use]

    pub(super) struct SavedBlobHandle<'a> {

        /// Transaction we're using to add the blob to the ExtDocs table.

        ///

        /// Note that struct fields are dropped in declaration order,

        /// so when we drop an uncommitted SavedBlobHandle,

        /// we roll back the transaction before we delete the file.

        /// (In practice, either order would be fine.)

        tx: Transaction<'a>,

        /// Filename for the file, with respect to the blob directory.

        fname: String,

        /// Declared digest string for this blob. Of the format

        /// "digesttype-hexstr".

        digeststr: String,

        /// An 'unlinker' for the blob file.

        unlinker: Unlinker,

    }

    impl<'a> SavedBlobHandle<'a> {

        /// Construct a SavedBlobHandle from its parts.

        /// Return a reference to the underlying database transaction.

        /// Return the digest string of the saved blob.

        /// Other tables use this as a foreign key into ExtDocs.digest

        /// Return the filename of this blob within the blob directory.

        #[allow(unused)] // used for testing.

        /// Commit the relevant database transaction.

            // The blob has been written to disk, so it is safe to

            // commit the transaction.

            // If the commit returns an error, self.unlinker will remove the blob.

            // (This could result in a vanished blob if the commit reports an error,

            // but the transaction is still visible in the database.)

            // If we reach this point, we don't want to remove the file.

    }

    /// Handle to a file which we might have to delete.

    ///

    /// When this handle is dropped, the file gets deleted, unless you have

    /// first called [`Unlinker::forget`].

    pub(super) struct Unlinker {

        /// The location of the file to remove, or None if we shouldn't

        /// remove it.

        p: Option<PathBuf>,

    }

    impl Unlinker {

        /// Make a new Unlinker for a given filename.

        /// Forget about this unlinker, so that the corresponding file won't

        /// get dropped.

    }

    impl Drop for Unlinker {

                        e,

                        "Couldn't remove rolled-back blob file {}",

                    );

    }

}

/// Convert a hexadecimal sha3-256 digest from the database into an array.

/// Convert a hexadecimal sha3-256 "digest string" as used in the

/// digest column from the database into an array.

    } else {

    }

/// Create a ConsensusMeta from a `Row` returned by one of

/// `FIND_LATEST_CONSENSUS_META` or `FIND_CONSENSUS_AND_META_BY_DIGEST`.

    ))

/// Return `SystemTime::get()` as an OffsetDateTime in UTC.

/// Set up the tables for the arti cache schema in a sqlite database.

const INSTALL_V0_SCHEMA: &str = "

  -- Helps us version the schema.  The schema here corresponds to a

  -- version number called 'version', and it should be readable by

  -- anybody who is compliant with versions of at least 'readable_by'.

  CREATE TABLE TorSchemaMeta (

     name TEXT NOT NULL PRIMARY KEY,

     version INTEGER NOT NULL,

     readable_by INTEGER NOT NULL

  );

  INSERT INTO TorSchemaMeta (name, version, readable_by) VALUES ( 'TorDirStorage', 0, 0 );

  -- Keeps track of external blobs on disk.

  CREATE TABLE ExtDocs (

    -- Records a digest of the file contents, in the form '<digest_type>-hexstr'

    digest TEXT PRIMARY KEY NOT NULL,

    -- When was this file created?

    created DATE NOT NULL,

    -- After what time will this file definitely be useless?

    expires DATE NOT NULL,

    -- What is the type of this file? Currently supported are 'con_<flavor>'.

    --   (Before tor-dirmgr ~0.28.0, we would erroneously record 'con_flavor' as 'sha3-256';

    --   Nothing depended on this yet, but will be used in the future

    --   as we add more large-document types.)

    type TEXT NOT NULL,

    -- Filename for this file within our blob directory.

    filename TEXT NOT NULL

  );

  -- All the microdescriptors we know about.

  CREATE TABLE Microdescs (

    sha256_digest TEXT PRIMARY KEY NOT NULL,

    last_listed DATE NOT NULL,

    contents BLOB NOT NULL

  );

  -- All the authority certificates we know.

  CREATE TABLE Authcerts (

    id_digest TEXT NOT NULL,

    sk_digest TEXT NOT NULL,

    published DATE NOT NULL,

    expires DATE NOT NULL,

    contents BLOB NOT NULL,

    PRIMARY KEY (id_digest, sk_digest)

  );

  -- All the consensuses we're storing.

  CREATE TABLE Consensuses (

    valid_after DATE NOT NULL,

    fresh_until DATE NOT NULL,

    valid_until DATE NOT NULL,

    flavor TEXT NOT NULL,

    pending BOOLEAN NOT NULL,

    sha3_of_signed_part TEXT NOT NULL,

    digest TEXT NOT NULL,

    FOREIGN KEY (digest) REFERENCES ExtDocs (digest) ON DELETE CASCADE

  );

  CREATE INDEX Consensuses_vu on CONSENSUSES(valid_until);

";

/// Update the database schema, from each version to the next

const UPDATE_SCHEMA: &[&str] = &["

  -- Update the database schema from version 0 to version 1.

  CREATE TABLE RouterDescs (

    sha1_digest TEXT PRIMARY KEY NOT NULL,

    published DATE NOT NULL,

    contents BLOB NOT NULL

  );

","

  -- Update the database schema from version 1 to version 2.

  -- We create this table even if the bridge-client feature is disabled, but then don't touch it at all.

  CREATE TABLE BridgeDescs (

    bridge_line TEXT PRIMARY KEY NOT NULL,

    fetched DATE NOT NULL,

    until DATE NOT NULL,

    contents BLOB NOT NULL

  );

","

 -- Update the database schema from version 2 to version 3.

 -- Table to hold our latest ProtocolStatuses object, to tell us if we're obsolete.

 -- We hold this independently from our consensus,

 -- since we want to read it very early in our startup process,

 -- even if the consensus is expired.

 CREATE TABLE ProtocolStatus (

    -- Enforce that there is only one row in this table.

    -- (This is a bit kludgy, but I am assured that it is a common practice.)

    zero INTEGER PRIMARY KEY NOT NULL,

    -- valid-after date of the consensus from which we got this status

    date DATE NOT NULL,

    -- ProtoStatuses object, encoded as json

    statuses TEXT NOT NULL

 );

"];

/// Update the database schema version tracking, from each version to the next

const UPDATE_SCHEMA_VERSION: &str = "

  UPDATE TorSchemaMeta SET version=? WHERE version<?;

";

/// Version number used for this version of the arti cache schema.

const SCHEMA_VERSION: u32 = UPDATE_SCHEMA.len() as u32;

/// Query: find the latest-expiring microdesc consensus with a given

/// pending status.

const FIND_CONSENSUS_P: &str = "

  SELECT valid_after, valid_until, filename

  FROM Consensuses

  INNER JOIN ExtDocs ON ExtDocs.digest = Consensuses.digest

  WHERE pending = ? AND flavor = ?

  ORDER BY valid_until DESC

  LIMIT 1;

";

/// Query: find the latest-expiring microdesc consensus, regardless of

/// pending status.

const FIND_CONSENSUS: &str = "

  SELECT valid_after, valid_until, filename

  FROM Consensuses

  INNER JOIN ExtDocs ON ExtDocs.digest = Consensuses.digest

  WHERE flavor = ?

  ORDER BY valid_until DESC

  LIMIT 1;

";

/// Query: Find the valid-after time for the latest-expiring

/// non-pending consensus of a given flavor.

const FIND_LATEST_CONSENSUS_META: &str = "

  SELECT valid_after, fresh_until, valid_until, sha3_of_signed_part, digest

  FROM Consensuses

  WHERE pending = 0 AND flavor = ?

  ORDER BY valid_until DESC

  LIMIT 1;

";

/// Look up a consensus by its digest-of-signed-part string.

const FIND_CONSENSUS_AND_META_BY_DIGEST_OF_SIGNED: &str = "

  SELECT valid_after, fresh_until, valid_until, sha3_of_signed_part, Consensuses.digest, filename

  FROM Consensuses

  INNER JOIN ExtDocs on ExtDocs.digest = Consensuses.digest

  WHERE Consensuses.sha3_of_signed_part = ?

  LIMIT 1;

";

/// Query: Update the consensus whose digest field is 'digest' to call it

/// no longer pending.

const MARK_CONSENSUS_NON_PENDING: &str = "

  UPDATE Consensuses