Merge pull request #3061 from h2zh/drop-privs-cont

Get ready to enable DropPrivileges by default

Author: jhiemstrawisc

cache/advertise.go

@@ -281,7 +281,10 @@ func getTickerRate(tok string) time.Duration {
 	return validateTickerRate(tickerRate, tokenLifetime)
 }
 
-func LaunchFedTokManager(ctx context.Context, egrp *errgroup.Group, cache server_structs.XRootDServer) {
+// LaunchFedTokManager starts the federation token refresh loop. When Server.DropPrivileges
+// is true, the token file is chown'ed to the xrootd user and group by xrdhttp-pelican plugin
+// (via xrootd.FileCopyToXrootdDir(false, 9, file)); pass nil to skip (e.g. in tests).
+func LaunchFedTokManager(ctx context.Context, egrp *errgroup.Group, cache server_structs.XRootDServer, copyToXrootdDir server_utils.FedTokCopyToXrootdFunc) {
 	// Do our initial token fetch+set, then turn things over to the ticker
 	tok, err := server_utils.CreateFedTok(ctx, cache)
 	if err != nil {
@@ -294,8 +297,14 @@ func LaunchFedTokManager(ctx context.Context, egrp *errgroup.Group, cache server
 	// gives us a bit of buffer in the event the Director is down for a short period of time.
 	tickerRate := getTickerRate(tok)
 
+	// Set up the federation token directories in the cache
+	err = server_utils.SetupFedTokDirs(cache)
+	if err != nil {
+		log.Errorf("Failed to setup federation token directory: %v", err)
+	}
+
 	// Set the token in the cache
-	err = server_utils.SetFedTok(ctx, cache, tok)
+	err = server_utils.SetFedTok(ctx, cache, tok, copyToXrootdDir)
 	if err != nil {
 		log.Errorf("Failed to set the federation token: %v", err)
 	}
@@ -326,7 +335,7 @@ func LaunchFedTokManager(ctx context.Context, egrp *errgroup.Group, cache server
 				}
 
 				// Set the token in the cache
-				err = server_utils.SetFedTok(ctx, cache, tok)
+				err = server_utils.SetFedTok(ctx, cache, tok, copyToXrootdDir)
 				if err != nil {
 					log.Errorf("Failed to write the federation token: %v", err)
 				}

config/config.go

@@ -1341,8 +1341,8 @@ func SetServerDefaults(v *viper.Viper) error {
 
 	// Set fed token locations for cache/origin. Note that fed tokens aren't yet used by the
 	// Origin (2026-02-05), but they may be soon for things like third party copy.
-	v.SetDefault(param.Origin_FedTokenLocation.GetName(), filepath.Join(configDir, "origin-fed-token"))
-	v.SetDefault(param.Cache_FedTokenLocation.GetName(), filepath.Join(configDir, "cache-fed-token"))
+	v.SetDefault(param.Origin_FedTokenLocation.GetName(), filepath.Join(configDir, "fed-token", "origin-fed-token"))
+	v.SetDefault(param.Cache_FedTokenLocation.GetName(), filepath.Join(configDir, "fed-token", "cache-fed-token"))
 
 	runtimeDir, _, err := ensureRuntimeDir(v)
 	if err != nil {
@@ -1572,7 +1572,7 @@ func SetServerDefaults(v *viper.Viper) error {
 	// stash a copy of its value now.
 	v.SetDefault(param.Origin_TokenAudience.GetName(), v.GetString(param.Origin_Url.GetName()))
 
-	// Set defaults for Director, Registry, and Broker URLs only if the Discovery URL is not set.
+	// Set defaults for Director and Registry URLs only if the Discovery URL is not set.
 	// This is necessary because, in Viper, there is currently no way to check if a value is coming
 	// from the default or was explicitly set by the user. Therefore, if the DiscoveryURL is present,
 	// when populating the Director, Registry, and Broker URLs, the discoverFederationImpl function
@@ -1586,7 +1586,6 @@ func SetServerDefaults(v *viper.Viper) error {
 	// https://github.com/spf13/viper/issues/1814
 	if !v.IsSet(param.Federation_DiscoveryUrl.GetName()) {
 		v.SetDefault("Federation.RegistryUrl", v.GetString(param.Server_ExternalWebUrl.GetName()))
-		v.SetDefault("Federation.BrokerURL", v.GetString(param.Server_ExternalWebUrl.GetName()))
 		v.SetDefault("Federation_DirectorUrl", v.GetString(param.Server_ExternalWebUrl.GetName()))
 	}
 
@@ -1673,26 +1672,19 @@ func InitServer(ctx context.Context, currentServers server_structs.ServerType) e
 		// Set up the directories for the server to run as a non-root user;
 		// for the most part, we need to recursively chown and chmod the directory
 		// so either root or pelican can access it.
-		pelicanLocations := []string{
+		pelicanLocationsNoRecursive := []string{
 			param.Server_DbLocation.GetString(),
 		}
-		if currentServers.IsEnabled(server_structs.RegistryType) {
-			pelicanLocations = append(pelicanLocations, param.Registry_DbLocation.GetString())
-		}
-		if currentServers.IsEnabled(server_structs.OriginType) {
-			pelicanLocations = append(pelicanLocations, param.Origin_DbLocation.GetString())
-		}
-		if currentServers.IsEnabled(server_structs.DirectorType) {
-			pelicanLocations = append(pelicanLocations, param.Director_DbLocation.GetString(), param.Director_GeoIPLocation.GetString())
-		}
-		if err = setFileAndDirPerms(pelicanLocations, 0750, 0640, puser.Uid, 0, true); err != nil {
-			return errors.Wrap(err, "failure when setting up the file permissions for pelican")
-		}
-
-		pelicanLocationsNoRecursive := []string{}
 		if (currentServers.IsEnabled(server_structs.OriginType) || currentServers.IsEnabled(server_structs.CacheType)) && param.Shoveler_Enable.GetBool() {
 			pelicanLocationsNoRecursive = append(pelicanLocationsNoRecursive, param.Shoveler_AMQPTokenLocation.GetString())
 		}
+		if currentServers.IsEnabled(server_structs.CacheType) {
+			tokLoc := param.Cache_FedTokenLocation.GetString()
+			tokDir := filepath.Dir(tokLoc)
+			dir := filepath.Dir(tokDir)
+			tempTokDir := filepath.Join(dir, "fed-token-temp")
+			pelicanLocationsNoRecursive = append(pelicanLocationsNoRecursive, tempTokDir)
+		}
 		if err = setFileAndDirPerms(pelicanLocationsNoRecursive, 0750, 0640, puser.Uid, 0, false); err != nil {
 			return errors.Wrap(err, "failure when setting up the file permissions for pelican")
 		}
@@ -1718,9 +1710,9 @@ func InitServer(ctx context.Context, currentServers server_structs.ServerType) e
 		if (currentServers.IsEnabled(server_structs.OriginType) || currentServers.IsEnabled(server_structs.CacheType)) && param.Shoveler_Enable.GetBool() {
 			pelicanDirs = append(pelicanDirs, param.Shoveler_QueueDirectory.GetString())
 		}
-		if currentServers.IsEnabled(server_structs.OriginType) {
-			pelicanDirs = append(pelicanDirs, param.Origin_GlobusConfigLocation.GetString())
-		}
+		// Note: Origin_GlobusConfigLocation is intentionally NOT added here.
+		// It's under Origin_RunLocation (e.g. /run/pelican/xrootd/origin/) which should be owned by xrootd, not pelican.
+		// InitGlobusBackend() handles creating and chowning the Globus directories properly.
 		if err = setDirPerms(pelicanDirs, 0750, 0640, puser.Uid, puser.Gid, true); err != nil {
 			return errors.Wrap(err, "failure when setting up the directory permissions for pelican")
 		}
@@ -1988,6 +1980,14 @@ func InitServer(ctx context.Context, currentServers server_structs.ServerType) e
 		return err
 	}
 
+	// When drop privileges is enabled, ensure the pelican user can read TLS credentials.
+	// XRootD does not need direct access to these files as Pelican copies them to a runtime location.
+	if param.Server_DropPrivileges.GetBool() {
+		if err = CheckTLSCredsForDropPrivileges(); err != nil {
+			return err
+		}
+	}
+
 	// The certificate was either generated or has been provided by now. Verify that any configured
 	// hostnames are valid w.r.t the given certificate.
 	//

config/init_server_creds.go

@@ -27,6 +27,7 @@ import (
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"encoding/pem"
+	goerrors "errors"
 	"fmt"
 	"io/fs"
 	"math/big"
@@ -121,6 +122,51 @@ func createDirForKeys(dir string) error {
 	return nil
 }
 
+// CheckTLSCredsForDropPrivileges ensures proper permissions on TLS key and certificate files
+// when Server.DropPrivileges is enabled, so the pelican user can read them.
+func CheckTLSCredsForDropPrivileges() error {
+	if !param.Server_DropPrivileges.GetBool() {
+		return nil
+	}
+
+	puser, err := GetPelicanUser()
+	if err != nil {
+		return errors.Wrap(err, "failed to get pelican user for TLS credentials check")
+	}
+
+	tlsCert := param.Server_TLSCertificateChain.GetString()
+	tlsKey := param.Server_TLSKey.GetString()
+
+	// Check both TLS certificate and key files; collect all errors so we report
+	// every file with permission or existence problems instead of exiting on the first.
+	var errs []error
+	filesToCheck := []string{tlsCert, tlsKey}
+	for _, filePath := range filesToCheck {
+		if filePath == "" {
+			continue
+		}
+
+		fileInfo, err := os.Stat(filePath)
+		if err != nil {
+			if os.IsNotExist(err) {
+				// File doesn't exist - this is an error at this point since GenerateCert
+				// should have been called first
+				return errors.Errorf("file %s does not exist when checking TLS credentials for drop privileges", filePath)
+			}
+			return errors.Wrapf(err, "failed to stat TLS file %s", filePath)
+		}
+
+		// Check file readability using platform-specific logic
+		if err := checkFileReadableByUser(filePath, fileInfo, puser); err != nil {
+			errs = append(errs, err)
+		}
+	}
+	if len(errs) > 0 {
+		return goerrors.Join(errs...)
+	}
+	return nil
+}
+
 // Return a pointer to an ECDSA private key or RSA private key read from keyLocation.
 //
 // This can be used to load ECDSA or RSA private key for various purposes,

config/init_server_creds_unix.go

@@ -0,0 +1,111 @@
+//go:build !windows
+
+/***************************************************************
+ *
+ * Copyright (C) 2026, Pelican Project, Morgridge Institute for Research
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.  You may
+ * obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ***************************************************************/
+
+package config
+
+import (
+	"fmt"
+	"os"
+	"os/user"
+	"syscall"
+
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+)
+
+// checkFileReadableByUser checks if the given user can read the file on Unix systems.
+// It examines the file's uid/gid/mode to determine readability.
+//
+// Note: This is a best-effort check performed at startup. It is subject to a TOCTOU
+// (time-of-check-time-of-use) race: file permissions could change between this check and
+// when the file is actually read. This is acceptable because the check is intended as an
+// early diagnostic to catch common misconfiguration, not as a security gate.
+func checkFileReadableByUser(filePath string, fileInfo os.FileInfo, puser User) error {
+	stat, ok := fileInfo.Sys().(*syscall.Stat_t)
+	if !ok {
+		// Fallback for any platform where Stat_t isn't available
+		log.Warnf("Cannot verify ownership of %s on this platform; ensure the pelican user (%s) can read it", filePath, puser.Username)
+		return nil
+	}
+
+	fileUid := int(stat.Uid)
+	fileGid := int(stat.Gid)
+	fileMode := fileInfo.Mode().Perm()
+
+	// Check if the pelican user can read the file:
+	// 1. If pelican user owns the file and has read permission (owner read bit)
+	// 2. If pelican user's primary or supplementary group matches file's group and group has read permission
+	// 3. If others have read permission
+	canRead := false
+
+	if fileUid == puser.Uid {
+		// Pelican user owns the file, check owner read permission
+		if fileMode&0400 != 0 {
+			canRead = true
+		}
+	} else if fileMode&0040 != 0 && userInGroup(puser, fileGid) {
+		// File has group read permission and the pelican user is in the file's group
+		// (either via primary GID or supplementary groups)
+		canRead = true
+	} else {
+		// Check others read permission
+		if fileMode&0004 != 0 {
+			canRead = true
+		}
+	}
+
+	if !canRead {
+		return errors.Errorf("TLS file %s is not readable by the pelican user (%s, uid=%d, gid=%d). "+
+			"File has owner uid=%d, gid=%d, mode=%04o. "+
+			"Please ensure the pelican user can read this file when Server.DropPrivileges is enabled",
+			filePath, puser.Username, puser.Uid, puser.Gid, fileUid, fileGid, fileMode)
+	}
+
+	return nil
+}
+
+// userInGroup returns true if the user's primary GID matches fileGid, or if
+// fileGid appears in the user's supplementary group list.
+func userInGroup(puser User, fileGid int) bool {
+	if fileGid == puser.Gid {
+		return true
+	}
+
+	// Look up supplementary groups for the user
+	u, err := user.Lookup(puser.Username)
+	if err != nil {
+		log.Debugf("Could not look up supplementary groups for user %s: %v", puser.Username, err)
+		return false
+	}
+	groupIds, err := u.GroupIds()
+	if err != nil {
+		log.Debugf("Could not retrieve supplementary groups for user %s: %v", puser.Username, err)
+		return false
+	}
+
+	fileGidStr := fmt.Sprintf("%d", fileGid)
+	for _, gidStr := range groupIds {
+		if gidStr == fileGidStr {
+			return true
+		}
+	}
+
+	return false
+}

config/init_server_creds_windows.go

@@ -0,0 +1,35 @@
+//go:build windows
+
+/***************************************************************
+ *
+ * Copyright (C) 2026, Pelican Project, Morgridge Institute for Research
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.  You may
+ * obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ***************************************************************/
+
+package config
+
+import (
+	"os"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// checkFileReadableByUser on Windows logs a warning since Unix-style permission
+// checking doesn't apply. Windows uses ACLs which have a different permission model.
+// The Server.DropPrivileges feature is primarily designed for Unix systems.
+func checkFileReadableByUser(filePath string, fileInfo os.FileInfo, puser User) error {
+	log.Warnf("Cannot verify ownership of %s on Windows; ensure the pelican user (%s) can read it", filePath, puser.Username)
+	return nil
+}

e2e_fed_tests/cache_test.go

@@ -65,7 +65,7 @@ func TestCacheFedTokMaint(t *testing.T) {
 
 	// Give this "cache" instance a unique location so it doesn't compete with the fed test cache token
 	require.NoError(t, param.Set(param.Cache_FedTokenLocation.GetName(), filepath.Join(t.TempDir(), t.Name()+"_fedtok")))
-	cache.LaunchFedTokManager(ctx, egrp, &cacheServer)
+	cache.LaunchFedTokManager(ctx, egrp, &cacheServer, nil)
 	tokFile := cacheServer.GetFedTokLocation()
 
 	ticker := time.NewTicker(1 * time.Second)

launchers/cache_serve.go

@@ -25,6 +25,7 @@ import (
 	_ "embed"
 	"net"
 	"net/url"
+	"os"
 	"strconv"
 	"time"
 
@@ -112,7 +113,11 @@ func CacheServe(ctx context.Context, engine *gin.Engine, egrp *errgroup.Group, m
 	// Director tests or federation tokens.
 	if !param.Cache_EnableSiteLocalMode.GetBool() {
 		cache.LaunchDirectorTestFileCleanup(ctx)
-		cache.LaunchFedTokManager(ctx, egrp, cacheServer)
+		cache.LaunchFedTokManager(ctx, egrp, cacheServer, func(f *os.File) error {
+			// In drop-privileges mode, the token file is chown'ed to the xrootd user
+			// and group by xrdhttp-pelican plugin by passing command "9" to the plugin.
+			return xrootd.FileCopyToXrootdDir(false, 9, f)
+		})
 	}
 
 	concLimit := param.Cache_Concurrency.GetInt()