tests: restore cert reloading assertions and fix TLS test helpers

Replace TODO stubs with working implementations:

- peer_certificate_der(): raw tokio-rustls handshake to inspect peer
  certificates. reqwest's TlsInfo::peer_certificate() only works with
  the native-tls backend, returning None with rustls — so we drop down
  to tokio_rustls::TlsConnector directly where
  ServerConnection::peer_certificates() always works.

- cert_file_to_der(): parse PEM cert files to DER for comparison.

- make_http_tls(): now honors TestTlsConfig (builds hyper-rustls
  connector from the client config that trusts the test CA).

- make_ws_tls(): uses rustls::StreamOwned for synchronous TLS
  WebSocket connections.

Cert reloading test assertions in both environmentd and balancerd are
now fully restored — no remaining TODO stubs.

Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>

Author: jasonhernandez

src/balancerd/tests/server.rs

@@ -253,40 +253,28 @@ async fn test_balancer() {
             .send()
             .await
             .unwrap();
-        // TODO(SEC-219): peer certificate inspection not available with rustls postgres connector
-        // let tlsinfo = resp.extensions().get::<reqwest::tls::TlsInfo>().unwrap();
-        // let resp_x509 = X509::from_der(tlsinfo.peer_certificate().unwrap()).unwrap();
-        // let server_x509 = X509::from_pem(&std::fs::read(&server_cert).unwrap()).unwrap();
-        // assert_eq!(resp_x509, server_x509);
         assert_contains!(resp.text().await.unwrap(), "12234");
+        let server_cert_der = test_util::cert_file_to_der(&server_cert);
+        let tls_cfg = TestTlsConfig::with_ca(&ca.ca_cert_path());
+        let peer_der = test_util::peer_certificate_der(balancer_https_listen, &tls_cfg).await;
+        assert_eq!(peer_der, server_cert_der);
 
         // Generate new certs. Install only the key, reload, and make sure the old cert is still in
         // use.
         let (next_cert, next_key) = ca
             .request_cert("next", vec![IpAddr::V4(Ipv4Addr::LOCALHOST)])
             .unwrap();
-        // TODO(SEC-219): peer certificate inspection not available with rustls postgres connector
-        // let next_x509 = X509::from_pem(&std::fs::read(&next_cert).unwrap()).unwrap();
-        // assert_ne!(next_x509, server_x509);
+        let next_cert_der = test_util::cert_file_to_der(&next_cert);
+        assert_ne!(next_cert_der, server_cert_der);
         std::fs::copy(next_key, &server_key).unwrap();
         let (tx, rx) = oneshot::channel();
         reload_tx.try_send(Some(tx)).unwrap();
         let res = rx.await.unwrap();
         assert_err!(res);
 
         // We should still be on the old cert because now the cert and key mismatch.
-        let resp = client
-            .post(&https_url)
-            .header("Content-Type", "application/json")
-            .basic_auth(frontegg_user, Some(&frontegg_password))
-            .body(body)
-            .send()
-            .await
-            .unwrap();
-        // TODO(SEC-219): peer certificate inspection not available with rustls postgres connector
-        // let tlsinfo = resp.extensions().get::<reqwest::tls::TlsInfo>().unwrap();
-        // let resp_x509 = X509::from_der(tlsinfo.peer_certificate().unwrap()).unwrap();
-        // assert_eq!(resp_x509, server_x509);
+        let peer_der = test_util::peer_certificate_der(balancer_https_listen, &tls_cfg).await;
+        assert_eq!(peer_der, server_cert_der);
 
         // Now move the cert too. Reloading should succeed and the response should have the new
         // cert.
@@ -295,18 +283,8 @@ async fn test_balancer() {
         reload_tx.try_send(Some(tx)).unwrap();
         let res = rx.await.unwrap();
         assert_ok!(res);
-        let resp = client
-            .post(&https_url)
-            .header("Content-Type", "application/json")
-            .basic_auth(frontegg_user, Some(&frontegg_password))
-            .body(body)
-            .send()
-            .await
-            .unwrap();
-        // TODO(SEC-219): peer certificate inspection not available with rustls postgres connector
-        // let tlsinfo = resp.extensions().get::<reqwest::tls::TlsInfo>().unwrap();
-        // let resp_x509 = X509::from_der(tlsinfo.peer_certificate().unwrap()).unwrap();
-        // assert_eq!(resp_x509, next_x509);
+        let peer_der = test_util::peer_certificate_der(balancer_https_listen, &tls_cfg).await;
+        assert_eq!(peer_der, next_cert_der);
 
         if !is_multi_tenant_resolver {
             continue;

src/environmentd/src/test_util.rs

@@ -1693,6 +1693,39 @@ pub fn make_pg_tls(config: TestTlsConfig) -> MakeRustlsConnect {
     MakeRustlsConnect::new((*config.build_rustls_client_config()).clone())
 }
 
+/// Performs a TLS handshake to `addr` and returns the peer's leaf certificate
+/// in DER encoding.
+///
+/// We use a raw `tokio_rustls` connection instead of reqwest because
+/// `reqwest::tls::TlsInfo::peer_certificate()` only returns the peer cert
+/// when reqwest is built with the `native-tls` backend. With the `rustls-tls`
+/// backend it always returns `None` — a known reqwest limitation. By dropping
+/// down to `tokio_rustls` directly we can call
+/// `ServerConnection::peer_certificates()` which always works.
+pub async fn peer_certificate_der(
+    addr: std::net::SocketAddr,
+    tls_config: &TestTlsConfig,
+) -> Vec<u8> {
+    use tokio_rustls::TlsConnector;
+
+    let connector = TlsConnector::from(tls_config.build_rustls_client_config());
+    let server_name = rustls::pki_types::ServerName::IpAddress(addr.ip().into());
+    let tcp = tokio::net::TcpStream::connect(addr).await.unwrap();
+    let tls = connector.connect(server_name, tcp).await.unwrap();
+    let (_, session) = tls.get_ref();
+    let certs = session.peer_certificates().expect("peer certificates");
+    certs[0].as_ref().to_vec()
+}
+
+/// Reads a PEM certificate file and returns the first certificate as DER bytes.
+pub fn cert_file_to_der(path: &Path) -> Vec<u8> {
+    let pem = fs::read(path).unwrap();
+    let certs: Vec<CertificateDer> = rustls_pemfile::certs(&mut &*pem)
+        .collect::<Result<_, _>>()
+        .unwrap();
+    certs[0].as_ref().to_vec()
+}
+
 /// Configuration for test TLS connections, replacing the old
 /// `FnOnce(&mut SslConnectorBuilder)` closure pattern.
 #[derive(Clone, Debug)]

src/environmentd/tests/auth.rs

@@ -69,25 +69,25 @@ use uuid::Uuid;
 // without increasing test time.
 const EXPIRES_IN_SECS: u64 = 20;
 
-// TODO(SEC-219): migrate make_http_tls to rustls (hyper-rustls).
-// For now, HTTP test paths using this function will panic at runtime.
-fn make_http_tls(_config: &TestTlsConfig) -> hyper_rustls::HttpsConnector<HttpConnector> {
+fn make_http_tls(config: &TestTlsConfig) -> hyper_rustls::HttpsConnector<HttpConnector> {
+    let tls_config: rustls::ClientConfig = (*config.build_rustls_client_config()).clone();
     let mut http = HttpConnector::new();
     http.enforce_http(false);
     hyper_rustls::HttpsConnectorBuilder::new()
-        .with_native_roots()
-        .unwrap()
+        .with_tls_config(tls_config)
         .https_or_http()
         .enable_http2()
         .wrap_connector(http)
 }
 
-// TODO(SEC-219): migrate make_ws_tls to rustls.
-// For now, WebSocket test paths using this function will connect via plain TCP.
-fn make_ws_tls(uri: &Uri, _config: &TestTlsConfig) -> impl Read + Write {
-    // Plain TCP — TLS WS tests will fail at the protocol level until
-    // this is migrated to tokio-rustls or rustls-connector.
-    TcpStream::connect(format!("{}:{}", uri.host().unwrap(), uri.port().unwrap())).unwrap()
+fn make_ws_tls(uri: &Uri, config: &TestTlsConfig) -> impl Read + Write {
+    let tls_config = config.build_rustls_client_config();
+    let server_name = rustls::pki_types::ServerName::try_from(uri.host().unwrap().to_string())
+        .expect("valid server name");
+    let conn = rustls::ClientConnection::new(tls_config, server_name).unwrap();
+    let tcp =
+        TcpStream::connect(format!("{}:{}", uri.host().unwrap(), uri.port().unwrap())).unwrap();
+    rustls::StreamOwned::new(conn, tcp)
 }
 
 // Use two error types because some tests need to retry certain errors because

src/environmentd/tests/server.rs

@@ -4737,35 +4737,29 @@ async fn test_cert_reloading() {
         .send()
         .await
         .unwrap();
-    // TODO(SEC-219): certificate comparison needs rustls-native peer cert API.
-    // Previously we compared X509::from_der(tlsinfo.peer_certificate()) against the
-    // server cert file. With rustls, peer_certificate() is not available on reqwest TlsInfo.
-    let _tlsinfo = resp.extensions().get::<reqwest::tls::TlsInfo>().unwrap();
     assert_contains!(resp.text().await.unwrap(), "12234");
+    let server_cert_der = test_util::cert_file_to_der(&server_cert);
+    let tls_cfg = TestTlsConfig::with_ca(&ca.ca_cert_path());
+    let peer_der = test_util::peer_certificate_der(envd_server.http_local_addr(), &tls_cfg).await;
+    assert_eq!(peer_der, server_cert_der);
     check_pgwire(&conn_str, &ca.ca_cert_path()).await;
 
     // Generate new certs. Install only the key, reload, and make sure the old cert is still in
     // use.
     let (next_cert, next_key) = ca
         .request_cert("next", vec![IpAddr::V4(Ipv4Addr::LOCALHOST)])
         .unwrap();
+    let next_cert_der = test_util::cert_file_to_der(&next_cert);
+    assert_ne!(next_cert_der, server_cert_der);
     std::fs::copy(next_key, &server_key).unwrap();
     let (tx, rx) = oneshot::channel();
     reload_tx.try_send(Some(tx)).unwrap();
     let res = rx.await.unwrap();
     assert_err!(res);
 
     // We should still be on the old cert because now the cert and key mismatch.
-    let resp = client
-        .post(&https_url)
-        .header("Content-Type", "application/json")
-        .basic_auth(frontegg_user, Some(&frontegg_password))
-        .body(body)
-        .send()
-        .await
-        .unwrap();
-    // TODO(SEC-219): certificate comparison needs rustls-native peer cert API.
-    let _tlsinfo = resp.extensions().get::<reqwest::tls::TlsInfo>().unwrap();
+    let peer_der = test_util::peer_certificate_der(envd_server.http_local_addr(), &tls_cfg).await;
+    assert_eq!(peer_der, server_cert_der);
     check_pgwire(&conn_str, &ca.ca_cert_path()).await;
 
     // Now move the cert too. Reloading should succeed and the response should have the new
@@ -4775,16 +4769,8 @@ async fn test_cert_reloading() {
     reload_tx.try_send(Some(tx)).unwrap();
     let res = rx.await.unwrap();
     assert_ok!(res);
-    let resp = client
-        .post(&https_url)
-        .header("Content-Type", "application/json")
-        .basic_auth(frontegg_user, Some(&frontegg_password))
-        .body(body)
-        .send()
-        .await
-        .unwrap();
-    // TODO(SEC-219): certificate comparison needs rustls-native peer cert API.
-    let _tlsinfo = resp.extensions().get::<reqwest::tls::TlsInfo>().unwrap();
+    let peer_der = test_util::peer_certificate_der(envd_server.http_local_addr(), &tls_cfg).await;
+    assert_eq!(peer_der, next_cert_der);
     check_pgwire(&conn_str, &ca.ca_cert_path()).await;
 }