upki: add binary index to improve performance

Author: djc

Cargo.lock

@@ -13,7 +13,7 @@ aws-lc-rs = "1.15.2"
 base64 = "0.22.1"
 chrono = { version = "0.4.42", features = ["alloc"], default-features = false }
 clap = { version = "4.5", features = ["derive"] }
-clubcard-crlite = "0.3.2"
+clubcard-crlite = "0.3.3"
 criterion = "0.8"
 directories = "6"
 eyre = "0.6"
@@ -62,3 +62,6 @@ unreachable_pub = "warn"
 unused_extern_crates = "warn"
 unused_import_braces = "warn"
 unused_qualifications = "warn"
+
+[patch.crates-io]
+clubcard-crlite = { git = "https://github.com/mozilla/clubcard-crlite", rev = "83d4afe61d511fe88e8bf22e8a71c3b582536e69" }

Cargo.toml

@@ -10,7 +10,7 @@ use codspeed_criterion_compat::{Criterion, criterion_group, criterion_main};
 use criterion::{Criterion, criterion_group, criterion_main};
 use revoke_test::RevocationTestSites;
 use rustls_pki_types::CertificateDer;
-use upki::revocation::{Manifest, RevocationCheckInput, RevocationStatus};
+use upki::revocation::{Index, Manifest, RevocationCheckInput, RevocationStatus};
 use upki::{Config, ConfigPath};
 
 fn revocation(c: &mut Criterion) {
@@ -47,10 +47,10 @@ fn revocation(c: &mut Criterion) {
         let revoked_certs = certificates_for_test_site(BENCHMARK_CASE);
 
         b.iter(|| {
-            let manifest = Manifest::from_config(&config).unwrap();
+            let mut index = Index::from_cache(&config).unwrap();
             let input = RevocationCheckInput::from_certificates(&revoked_certs).unwrap();
             assert_eq!(
-                manifest.check(&input, &config).unwrap(),
+                index.check(&input).unwrap(),
                 RevocationStatus::CertainlyRevoked
             );
         })

revoke-test/benches/bench.rs

@@ -14,7 +14,8 @@ use rustls::{
     ExtendedKeyPurpose, RootCertStore, SignatureScheme, SupportedCipherSuite,
 };
 use upki::revocation::{
-    CertSerial, CtTimestamp, IssuerSpkiHash, Manifest, RevocationCheckInput, RevocationStatus,
+    CertSerial, CtTimestamp, Index, IssuerSpkiHash, Manifest, RevocationCheckInput,
+    RevocationStatus,
 };
 use upki::{self, Config, ConfigPath};
 use webpki::{EndEntityCert, ExtendedKeyUsage, InvalidNameContext, VerifiedPath};
@@ -128,9 +129,7 @@ impl ServerVerifier {
             sct_timestamps,
         };
 
-        match Manifest::from_config(&self.config)
-            .and_then(|manifest| manifest.check(&input, &self.config))
-        {
+        match Index::from_cache(&self.config).and_then(|mut index| index.check(&input)) {
             Ok(rs) => Ok(rs),
             Err(e) => Err(rustls::Error::General(e.to_string())),
         }

rustls-upki/src/lib.rs

@@ -8,7 +8,7 @@ use std::path::Path;
 use std::slice;
 
 use rustls_pki_types::CertificateDer;
-use upki::revocation::{self, Manifest, RevocationCheckInput, RevocationStatus};
+use upki::revocation::{self, Index, RevocationCheckInput, RevocationStatus};
 use upki::{Config, Error};
 
 /// Check the revocation status of a certificate.
@@ -47,12 +47,12 @@ pub unsafe extern "C" fn upki_check_revocation(
             Err(err) => return Error::Revocation(err).into(),
         };
 
-        let manifest = match Manifest::from_config(config) {
-            Ok(manifest) => manifest,
+        let mut index = match Index::from_cache(config) {
+            Ok(index) => index,
             Err(err) => return Error::Revocation(err).into(),
         };
 
-        match manifest.check(&input, config) {
+        match index.check(&input) {
             Ok(status) => match status {
                 RevocationStatus::NotCoveredByRevocationData => {
                     upki_result::UPKI_REVOCATION_NOT_COVERED

upki-ffi/src/lib.rs

@@ -8,7 +8,7 @@ use clap::{Parser, Subcommand};
 use eyre::{Context, Report};
 use rustls_pki_types::CertificateDer;
 use rustls_pki_types::pem::PemObject;
-use upki::revocation::{Manifest, RevocationCheckInput, fetch};
+use upki::revocation::{Index, Manifest, RevocationCheckInput, fetch};
 use upki::{Config, ConfigPath};
 
 #[tokio::main(flavor = "current_thread")]
@@ -51,8 +51,8 @@ async fn main() -> Result<ExitCode, Report> {
             }
 
             let input = RevocationCheckInput::from_certificates(&certs)?;
-            Manifest::from_config(&config)?
-                .check(&input, &config)?
+            Index::from_cache(&config)?
+                .check(&input)?
                 .to_cli()
         }
     })

upki/src/main.rs

@@ -20,7 +20,8 @@ use std::process::ExitCode;
 use aws_lc_rs::digest;
 use tracing::{debug, info};
 
-use super::{Error, Filter, Manifest};
+use super::index::INDEX_BIN;
+use super::{Error, Filter, Index, Manifest};
 use crate::Config;
 
 /// Update the local revocation cache by fetching updates over the network.
@@ -157,6 +158,11 @@ impl Plan {
             steps.push(PlanStep::download(filter, remote_url, local));
         }
 
+        steps.push(PlanStep::SaveIndex {
+            manifest: manifest.clone(),
+            local_dir: local.to_owned(),
+        });
+
         steps.push(PlanStep::SaveManifest {
             manifest: manifest.clone(),
             local_dir: local.to_owned(),
@@ -197,6 +203,12 @@ enum PlanStep {
     /// Delete the given single local file.
     Delete(PathBuf),
 
+    /// Build and save the index from filter universe metadata.
+    SaveIndex {
+        manifest: Manifest,
+        local_dir: PathBuf,
+    },
+
     /// Save the manifest structure
     SaveManifest {
         manifest: Manifest,
@@ -264,6 +276,46 @@ impl PlanStep {
                     path: target,
                 })?;
             }
+            Self::SaveIndex {
+                manifest,
+                local_dir,
+            } => {
+                debug!("building index");
+                let Some(buf) = Index::write(&manifest, &local_dir) else {
+                    return Ok(());
+                };
+
+                #[cfg(target_family = "unix")]
+                let temp = tempfile::Builder::new()
+                    .permissions(Permissions::from_mode(0o644))
+                    .suffix(".new")
+                    .tempfile_in(&local_dir);
+                #[cfg(not(target_family = "unix"))]
+                let temp = tempfile::Builder::new()
+                    .suffix(".new")
+                    .tempfile_in(&local_dir);
+
+                let mut local_temp = temp.map_err(|error| Error::FileWrite {
+                    error,
+                    path: local_dir.clone(),
+                })?;
+
+                local_temp
+                    .as_file_mut()
+                    .write_all(&buf)
+                    .map_err(|error| Error::FileWrite {
+                        error,
+                        path: local_temp.path().to_owned(),
+                    })?;
+
+                let path = local_dir.join(INDEX_BIN);
+                local_temp
+                    .persist(&path)
+                    .map_err(|error| Error::FileWrite {
+                        error: error.error,
+                        path,
+                    })?;
+            }
             Self::SaveManifest {
                 manifest,
                 local_dir,
@@ -305,6 +357,9 @@ impl fmt::Display for PlanStep {
                 filter.size
             ),
             Self::Delete(path) => write!(f, "delete stale file {path:?}"),
+            Self::SaveIndex { local_dir, .. } => {
+                write!(f, "build index from filters into {local_dir:?}")
+            }
             Self::SaveManifest { local_dir, .. } => {
                 write!(f, "save new manifest into {local_dir:?}")
             }