From 549d7636266ea83d05c381820f671c344b64ffdd Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Fri, 15 Aug 2025 12:42:23 -0500 Subject: [PATCH 1/6] When using a `pkcs11:`-style URL, load via the engine API With this, the following works: ``` xrd.tls /home/foo/.config/certificates/tls.crt \ pkcs11:token=test-token;object=priv_key;type=public?pin-value=1234 ``` (cherry picked from commit 682f449f9469cc93e2a23be24a0208caea8c789c) --- src/Xrd/XrdConfig.cc | 2 +- src/XrdTls/XrdTlsContext.cc | 25 +++++++++++++++++++++++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/Xrd/XrdConfig.cc b/src/Xrd/XrdConfig.cc index 9f936409a11..9e0226769bb 100644 --- a/src/Xrd/XrdConfig.cc +++ b/src/Xrd/XrdConfig.cc @@ -2464,7 +2464,7 @@ int XrdConfig::xtls(XrdSysError *eDest, XrdOucStream &Config) if (!(val = Config.GetWord())) return 0; - if (*val == '/') + if (*val == '/' || !strncmp(val, "pkcs11:", 7)) {tlsKey = strdup(val); if (!(val = Config.GetWord())) return 0; } diff --git a/src/XrdTls/XrdTlsContext.cc b/src/XrdTls/XrdTlsContext.cc index 61237433818..8f7ddec26e5 100644 --- a/src/XrdTls/XrdTlsContext.cc +++ b/src/XrdTls/XrdTlsContext.cc @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -509,7 +510,7 @@ bool VerPaths(const char *cert, const char *pkey, // If a private key is present than make sure it's a file and only the // owner has access to it. // - if (pkey && (emsg = XrdOucUtils::ValPath(pkey, pkey_mode, false))) + if (pkey && pkey[0] == '/' && (emsg = XrdOucUtils::ValPath(pkey, pkey_mode, false))) {eMsg = "Unable to use key file "; eMsg += pkey; eMsg += "; "; eMsg += emsg; return false; @@ -763,7 +764,27 @@ XrdTlsContext::XrdTlsContext(const char *cert, const char *key, // Load the private key // - if (SSL_CTX_use_PrivateKey_file(pImpl->ctx, key, SSL_FILETYPE_PEM) != 1 ) + if (key[0] == 'p') { + + ENGINE *e = ENGINE_by_id("pkcs11"); + if (e) { + if(!ENGINE_init(e)) { + ENGINE_free(e); + FATAL_SSL("Unable to initialize pkcs11 engine"); + } + } else { + FATAL_SSL("Unable to create pkcs11 engine"); + } + auto priv_key = ENGINE_load_private_key(e, key, nullptr, nullptr); + + if (!priv_key) { + FATAL_SSL("Failed to load private key through engine"); + } + if (SSL_CTX_use_PrivateKey(pImpl->ctx, priv_key) != 1) + FATAL_SSL("Failed to have SSL context use private key"); + EVP_PKEY_free(priv_key); + + } else if (SSL_CTX_use_PrivateKey_file(pImpl->ctx, key, SSL_FILETYPE_PEM) != 1 ) FATAL_SSL("Unable to create TLS context; invalid private key."); // Make sure the key and certificate file match. From 30900fb0a1906aed9246b6e32b5d72ca60212c08 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Fri, 5 Dec 2025 19:57:25 -0600 Subject: [PATCH 2/6] Full pkcs11 integration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - EnsureOpenSSLConfigLoaded() is a helper function ensuring the OpenSSL global configuration file (OPENSSL_CONF) is loaded exactly once per process - Export PKCS11_MODULE_PATH (e.g. /usr/lib64/pkcs11/p11-kit-client.so) from the Pelican launcher and feed that directly to pkcs11 engine, guaranteeing it loads the RPC client module - Abort early if we can’t ingest the config, because without it the pkcs11 engine will never know which module to use - Use `ENGINE_free(e)` to clean up ENGINE objects --- src/XrdTls/XrdTlsContext.cc | 38 +++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/XrdTls/XrdTlsContext.cc b/src/XrdTls/XrdTlsContext.cc index 8f7ddec26e5..bbc1d6b8e14 100644 --- a/src/XrdTls/XrdTlsContext.cc +++ b/src/XrdTls/XrdTlsContext.cc @@ -17,10 +17,13 @@ //------------------------------------------------------------------------------ #include +#include +#include #include #include #include #include +#include #include #include #include @@ -47,6 +50,32 @@ namespace XrdTlsGlobal { extern XrdSysTrace SysTrace; }; + +namespace +{ +#ifndef OPENSSL_NO_CONF +bool EnsureOpenSSLConfigLoaded() +{ + static std::once_flag configFlag; + static bool loadOK = true; + + std::call_once(configFlag, []() { +#if OPENSSL_VERSION_NUMBER >= 0x10100000L + if (!OPENSSL_init_ssl(OPENSSL_INIT_LOAD_CONFIG, nullptr)) + loadOK = false; +#else + OPENSSL_config(nullptr); +#endif + }); + return loadOK; +} +#else +bool EnsureOpenSSLConfigLoaded() +{ + return false; +} +#endif +} /******************************************************************************/ /* X r d T l s C o n t e x t I m p l */ @@ -765,9 +794,18 @@ XrdTlsContext::XrdTlsContext(const char *cert, const char *key, // Load the private key // if (key[0] == 'p') { + if (!EnsureOpenSSLConfigLoaded()) + FATAL_SSL("Unable to load OpenSSL configuration; cannot initialize pkcs11 engine."); ENGINE *e = ENGINE_by_id("pkcs11"); if (e) { + const char* modulePath = getenv("PKCS11_MODULE_PATH"); + if (modulePath && modulePath[0]) { + if (!ENGINE_ctrl_cmd_string(e, "MODULE_PATH", modulePath, 0)) { + ENGINE_free(e); + FATAL_SSL("Unable to configure pkcs11 engine MODULE_PATH"); + } + } if(!ENGINE_init(e)) { ENGINE_free(e); FATAL_SSL("Unable to initialize pkcs11 engine"); From c87c13b6961be642dd6264b123d768930a385d64 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Wed, 31 Dec 2025 00:01:23 +0000 Subject: [PATCH 3/6] Fix compilation errors when building against OpenSSL configured with no-engine or no-deprecated options. Changes: - Include opensslconf.h first to get feature detection macros before attempting to include engine.h (which may not exist) - Check OPENSSL_NO_ENGINE, OPENSSL_NO_DEPRECATED_3_0, and OPENSSL_NO_DEPRECATED to determine ENGINE API availability - Define XRDTLS_HAVE_ENGINE macro to guard all ENGINE-related code - Provide clear runtime error when PKCS11 is requested but ENGINE API is unavailable This fixes CI build failures where OpenSSL 3.0 was compiled with deprecated APIs removed, causing: - "openssl/engine.h: No such file or directory" when header missing - "ENGINE_by_id was not declared in this scope" when APIs unavailable PKCS11 support now requires OpenSSL built with ENGINE support enabled. --- src/XrdTls/XrdTlsContext.cc | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/XrdTls/XrdTlsContext.cc b/src/XrdTls/XrdTlsContext.cc index bbc1d6b8e14..12be7d3eb35 100644 --- a/src/XrdTls/XrdTlsContext.cc +++ b/src/XrdTls/XrdTlsContext.cc @@ -19,13 +19,25 @@ #include #include #include + +// Include OpenSSL config headers first to get feature detection macros +#include +#include + +// ENGINE API availability check: +// - OPENSSL_NO_ENGINE: OpenSSL compiled with no-engine (engine.h may not exist) +// - OPENSSL_NO_DEPRECATED_3_0: OpenSSL 3.0+ compiled with no-deprecated (ENGINE is deprecated) +// - OPENSSL_NO_DEPRECATED: All deprecated APIs disabled +#if !defined(OPENSSL_NO_ENGINE) && !defined(OPENSSL_NO_DEPRECATED_3_0) && !defined(OPENSSL_NO_DEPRECATED) + #define XRDTLS_HAVE_ENGINE 1 + #include +#endif + #include #include -#include #include #include #include -#include #include #include "XrdOuc/XrdOucUtils.hh" @@ -794,6 +806,7 @@ XrdTlsContext::XrdTlsContext(const char *cert, const char *key, // Load the private key // if (key[0] == 'p') { +#ifdef XRDTLS_HAVE_ENGINE if (!EnsureOpenSSLConfigLoaded()) FATAL_SSL("Unable to load OpenSSL configuration; cannot initialize pkcs11 engine."); @@ -821,6 +834,9 @@ XrdTlsContext::XrdTlsContext(const char *cert, const char *key, if (SSL_CTX_use_PrivateKey(pImpl->ctx, priv_key) != 1) FATAL_SSL("Failed to have SSL context use private key"); EVP_PKEY_free(priv_key); +#else + FATAL_SSL("PKCS11 support not available; OpenSSL built without ENGINE API support."); +#endif } else if (SSL_CTX_use_PrivateKey_file(pImpl->ctx, key, SSL_FILETYPE_PEM) != 1 ) FATAL_SSL("Unable to create TLS context; invalid private key."); From a377eb7d3490e1aad3b2be5b2122956fe7748cd2 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Mon, 5 Jan 2026 16:23:37 -0600 Subject: [PATCH 4/6] Use Provider API instead of ENGINE API for OpenSSL 3.0+ --- src/XrdTls/XrdTlsContext.cc | 57 +++++++++++++++++++++++++++++++------ 1 file changed, 49 insertions(+), 8 deletions(-) diff --git a/src/XrdTls/XrdTlsContext.cc b/src/XrdTls/XrdTlsContext.cc index 12be7d3eb35..d897bf8a9d5 100644 --- a/src/XrdTls/XrdTlsContext.cc +++ b/src/XrdTls/XrdTlsContext.cc @@ -24,11 +24,20 @@ #include #include -// ENGINE API availability check: -// - OPENSSL_NO_ENGINE: OpenSSL compiled with no-engine (engine.h may not exist) -// - OPENSSL_NO_DEPRECATED_3_0: OpenSSL 3.0+ compiled with no-deprecated (ENGINE is deprecated) -// - OPENSSL_NO_DEPRECATED: All deprecated APIs disabled -#if !defined(OPENSSL_NO_ENGINE) && !defined(OPENSSL_NO_DEPRECATED_3_0) && !defined(OPENSSL_NO_DEPRECATED) +// PKCS#11 support strategy: +// - OpenSSL 3.x+ (EL9+, AlmaLinux 10): Prefer OSSL_STORE API with pkcs11-provider (modern) +// Falls back to ENGINE if pkcs11-provider is not installed but engine_pkcs11 is +// - OpenSSL 1.1.x (EL8): Use ENGINE API with libp11/engine_pkcs11 (legacy compatibility) + +// OpenSSL 3.0+ Provider/OSSL_STORE support (preferred for modern systems) +#if OPENSSL_VERSION_NUMBER >= 0x30000000L + #define XRDTLS_HAVE_OSSL_STORE 1 + #include +#endif + +// ENGINE API support: +// - OpenSSL 1.1.x: Primary method for PKCS#11 +#if !defined(OPENSSL_NO_ENGINE) #define XRDTLS_HAVE_ENGINE 1 #include #endif @@ -806,10 +815,42 @@ XrdTlsContext::XrdTlsContext(const char *cert, const char *key, // Load the private key // if (key[0] == 'p') { -#ifdef XRDTLS_HAVE_ENGINE if (!EnsureOpenSSLConfigLoaded()) - FATAL_SSL("Unable to load OpenSSL configuration; cannot initialize pkcs11 engine."); + FATAL_SSL("Unable to load OpenSSL configuration; cannot initialize pkcs11."); + +#ifdef XRDTLS_HAVE_OSSL_STORE + // OpenSSL 3.x+: Use OSSL_STORE API with pkcs11-provider (modern, preferred) + // Requires pkcs11-provider to be installed: https://github.com/latchset/pkcs11-provider + OSSL_STORE_CTX *store_ctx = OSSL_STORE_open(key, nullptr, nullptr, nullptr, nullptr); + if (!store_ctx) + FATAL_SSL("Failed to open PKCS11 URI. Ensure pkcs11-provider is installed and configured."); + + EVP_PKEY *priv_key = nullptr; + while (!OSSL_STORE_eof(store_ctx)) { + OSSL_STORE_INFO *info = OSSL_STORE_load(store_ctx); + if (info) { + int type = OSSL_STORE_INFO_get_type(info); + if (type == OSSL_STORE_INFO_PKEY) { + priv_key = OSSL_STORE_INFO_get1_PKEY(info); + OSSL_STORE_INFO_free(info); + break; + } + OSSL_STORE_INFO_free(info); + } + } + OSSL_STORE_close(store_ctx); + + if (!priv_key) + FATAL_SSL("Failed to load private key from PKCS11 URI. Check pkcs11-provider configuration."); + + if (SSL_CTX_use_PrivateKey(pImpl->ctx, priv_key) != 1) { + EVP_PKEY_free(priv_key); + FATAL_SSL("Failed to have SSL context use private key"); + } + EVP_PKEY_free(priv_key); +#elif defined(XRDTLS_HAVE_ENGINE) + // OpenSSL 1.1.x (EL8): Use ENGINE API with libp11/engine_pkcs11 ENGINE *e = ENGINE_by_id("pkcs11"); if (e) { const char* modulePath = getenv("PKCS11_MODULE_PATH"); @@ -835,7 +876,7 @@ XrdTlsContext::XrdTlsContext(const char *cert, const char *key, FATAL_SSL("Failed to have SSL context use private key"); EVP_PKEY_free(priv_key); #else - FATAL_SSL("PKCS11 support not available; OpenSSL built without ENGINE API support."); + FATAL_SSL("PKCS11 support not available."); #endif } else if (SSL_CTX_use_PrivateKey_file(pImpl->ctx, key, SSL_FILETYPE_PEM) != 1 ) From 286c1e56926b75b2ec83a05b1f4a007cf27d3a1f Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Thu, 8 Jan 2026 03:10:47 +0000 Subject: [PATCH 5/6] Amend the comment for private key file path check - Note that if it's a PKCS#11 URI, it will be validated later. --- src/XrdTls/XrdTlsContext.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/XrdTls/XrdTlsContext.cc b/src/XrdTls/XrdTlsContext.cc index d897bf8a9d5..76dc9d3cd73 100644 --- a/src/XrdTls/XrdTlsContext.cc +++ b/src/XrdTls/XrdTlsContext.cc @@ -557,8 +557,8 @@ bool VerPaths(const char *cert, const char *pkey, return false; } -// If a private key is present than make sure it's a file and only the -// owner has access to it. +// If a private key is provided as a filesystem path, verify it's a file and only the +// owner has access to it. If it's a PKCS#11 URI, it will be validated later. // if (pkey && pkey[0] == '/' && (emsg = XrdOucUtils::ValPath(pkey, pkey_mode, false))) {eMsg = "Unable to use key file "; From c88349d9758caf5b9032952a270e4b7c1ea97a81 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Thu, 8 Jan 2026 14:41:45 +0000 Subject: [PATCH 6/6] XrdTls: Isolate PKCS#11 operations in dedicated OSSL_LIB_CTX Load PKCS#11 provider in an isolated OpenSSL context to prevent interference with other TLS operations (e.g., SciTokens library). - Add InitIsolatedPKCS11Context() to create isolated OSSL_LIB_CTX - Load OPENSSL_CONF into isolated context for proper configuration - Use OSSL_STORE_open_ex() with isolated context for PKCS#11 URIs - Add cleanup for isolated context in destructor This fixes "bad record MAC" errors that occurred when the global PKCS#11 provider affected unrelated HTTPS operations (when SciTokens tried to fetch JWKS over HTTPS). --- src/XrdTls/XrdTlsContext.cc | 89 ++++++++++++++++++++++++++++++++++--- 1 file changed, 82 insertions(+), 7 deletions(-) diff --git a/src/XrdTls/XrdTlsContext.cc b/src/XrdTls/XrdTlsContext.cc index 76dc9d3cd73..6ee2dc05aef 100644 --- a/src/XrdTls/XrdTlsContext.cc +++ b/src/XrdTls/XrdTlsContext.cc @@ -33,6 +33,7 @@ #if OPENSSL_VERSION_NUMBER >= 0x30000000L #define XRDTLS_HAVE_OSSL_STORE 1 #include + #include #endif // ENGINE API support: @@ -96,6 +97,53 @@ bool EnsureOpenSSLConfigLoaded() return false; } #endif + +#if OPENSSL_VERSION_NUMBER >= 0x30000000L +// Initialize an isolated OSSL_LIB_CTX for PKCS11 operations +// This prevents PKCS11 from affecting other parts of the process (e.g., SciTokens library) +bool InitIsolatedPKCS11Context(OSSL_LIB_CTX **libCtx, OSSL_PROVIDER **pkcs11Prov, OSSL_PROVIDER **defaultProv) +{ + // Create an isolated library context + *libCtx = OSSL_LIB_CTX_new(); + if (!*libCtx) { + return false; + } + + // Load the OpenSSL configuration into the isolated context + // This allows the pkcs11-provider to be properly configured with module paths, etc. + const char *opensslConf = getenv("OPENSSL_CONF"); + if (opensslConf && opensslConf[0]) { + // Load the config file into the isolated context + if (!OSSL_LIB_CTX_load_config(*libCtx, opensslConf)) { + // Config loading failed, cleanup and return error + OSSL_LIB_CTX_free(*libCtx); + *libCtx = nullptr; + return false; + } + } else { + // No OPENSSL_CONF set, try loading providers manually + // Load the default provider in the isolated context (needed for basic crypto) + *defaultProv = OSSL_PROVIDER_load(*libCtx, "default"); + if (!*defaultProv) { + OSSL_LIB_CTX_free(*libCtx); + *libCtx = nullptr; + return false; + } + + // Load the pkcs11 provider in the isolated context + *pkcs11Prov = OSSL_PROVIDER_load(*libCtx, "pkcs11"); + if (!*pkcs11Prov) { + OSSL_PROVIDER_unload(*defaultProv); + OSSL_LIB_CTX_free(*libCtx); + *libCtx = nullptr; + *defaultProv = nullptr; + return false; + } + } + + return true; +} +#endif } /******************************************************************************/ @@ -107,10 +155,18 @@ struct XrdTlsContextImpl XrdTlsContextImpl(XrdTlsContext *p) : ctx(0), ctxnew(0), owner(p), flsCVar(0), flushT(0), - crlRunning(false), flsRunning(false) {} + crlRunning(false), flsRunning(false) +#if OPENSSL_VERSION_NUMBER >= 0x30000000L + , pkcs11LibCtx(0), pkcs11Provider(0) +#endif + {} ~XrdTlsContextImpl() {if (ctx) SSL_CTX_free(ctx); if (ctxnew) delete ctxnew; if (flsCVar) delete flsCVar; +#if OPENSSL_VERSION_NUMBER >= 0x30000000L + if (pkcs11Provider) OSSL_PROVIDER_unload(pkcs11Provider); + if (pkcs11LibCtx) OSSL_LIB_CTX_free(pkcs11LibCtx); +#endif } SSL_CTX *ctx; @@ -125,6 +181,10 @@ struct XrdTlsContextImpl time_t lastCertModTime = 0; int sessionCacheOpts = -1; std::string sessionCacheId; +#if OPENSSL_VERSION_NUMBER >= 0x30000000L + OSSL_LIB_CTX *pkcs11LibCtx; // Isolated context for PKCS11 + OSSL_PROVIDER *pkcs11Provider; // PKCS11 provider handle +#endif }; /******************************************************************************/ @@ -819,11 +879,25 @@ XrdTlsContext::XrdTlsContext(const char *cert, const char *key, FATAL_SSL("Unable to load OpenSSL configuration; cannot initialize pkcs11."); #ifdef XRDTLS_HAVE_OSSL_STORE - // OpenSSL 3.x+: Use OSSL_STORE API with pkcs11-provider (modern, preferred) - // Requires pkcs11-provider to be installed: https://github.com/latchset/pkcs11-provider - OSSL_STORE_CTX *store_ctx = OSSL_STORE_open(key, nullptr, nullptr, nullptr, nullptr); - if (!store_ctx) - FATAL_SSL("Failed to open PKCS11 URI. Ensure pkcs11-provider is installed and configured."); + // OpenSSL 3.x+: Use OSSL_STORE API with pkcs11-provider in an ISOLATED context + // This prevents PKCS11 from affecting other TLS operations (e.g., SciTokens library) + + OSSL_PROVIDER *defaultProv = nullptr; + + // Initialize isolated PKCS11 context + // This loads the OPENSSL_CONF into an isolated context so PKCS11 doesn't affect + // other libraries in the process (like SciTokens) + if (!InitIsolatedPKCS11Context(&pImpl->pkcs11LibCtx, &pImpl->pkcs11Provider, &defaultProv)) { + FATAL_SSL("Failed to initialize isolated PKCS11 context. Check OPENSSL_CONF and pkcs11-provider installation."); + } + + // Open PKCS11 URI using the isolated context + OSSL_STORE_CTX *store_ctx = OSSL_STORE_open_ex(key, pImpl->pkcs11LibCtx, nullptr, + nullptr, nullptr, nullptr, nullptr, nullptr); + if (!store_ctx) { + if (defaultProv) OSSL_PROVIDER_unload(defaultProv); + FATAL_SSL("Failed to open PKCS11 URI in isolated context."); + } EVP_PKEY *priv_key = nullptr; while (!OSSL_STORE_eof(store_ctx)) { @@ -839,9 +913,10 @@ XrdTlsContext::XrdTlsContext(const char *cert, const char *key, } } OSSL_STORE_close(store_ctx); + if (defaultProv) OSSL_PROVIDER_unload(defaultProv); if (!priv_key) - FATAL_SSL("Failed to load private key from PKCS11 URI. Check pkcs11-provider configuration."); + FATAL_SSL("Failed to load private key from PKCS11 URI in isolated context."); if (SSL_CTX_use_PrivateKey(pImpl->ctx, priv_key) != 1) { EVP_PKEY_free(priv_key);