Adjust fed token dirs and their perms

- Setup the permanent and temporary directory for the federation token file
and sets proper perms.
- The permanent directory is always owned by xrootd user.
- In normal mode, the temporary directory is owned by the root. While in drop privileges mode, it is owned by the pelican user.

Author: h2zh

cache/advertise.go

@@ -297,6 +297,12 @@ func LaunchFedTokManager(ctx context.Context, egrp *errgroup.Group, cache server
 	// gives us a bit of buffer in the event the Director is down for a short period of time.
 	tickerRate := getTickerRate(tok)
 
+	// Set up the federation token directories in the cache
+	err = server_utils.SetupFedTokDirs(cache)
+	if err != nil {
+		log.Errorf("Failed to setup federation token directory: %v", err)
+	}
+
 	// Set the token in the cache
 	err = server_utils.SetFedTok(ctx, cache, tok, copyToXrootdDir)
 	if err != nil {

config/config.go

@@ -1661,6 +1661,13 @@ func InitServer(ctx context.Context, currentServers server_structs.ServerType) e
 		if (currentServers.IsEnabled(server_structs.OriginType) || currentServers.IsEnabled(server_structs.CacheType)) && param.Shoveler_Enable.GetBool() {
 			pelicanLocationsNoRecursive = append(pelicanLocationsNoRecursive, param.Shoveler_AMQPTokenLocation.GetString())
 		}
+		if currentServers.IsEnabled(server_structs.CacheType) {
+			tokLoc := param.Cache_FedTokenLocation.GetString()
+			tokDir := filepath.Dir(tokLoc)
+			dir := filepath.Dir(tokDir)
+			tempTokDir := filepath.Join(dir, "fed-token-temp")
+			pelicanLocationsNoRecursive = append(pelicanLocationsNoRecursive, tempTokDir)
+		}
 		if err = setFileAndDirPerms(pelicanLocationsNoRecursive, 0750, 0640, puser.Uid, 0, false); err != nil {
 			return errors.Wrap(err, "failure when setting up the file permissions for pelican")
 		}
@@ -1918,10 +1925,10 @@ func InitServer(ctx context.Context, currentServers server_structs.ServerType) e
 	// Origin (2025-02-04), but they may be soon for things like third party copy.
 	configDir := viper.GetString("ConfigDir")
 	if currentServers.IsEnabled(server_structs.OriginType) {
-		viper.SetDefault(param.Origin_FedTokenLocation.GetName(), filepath.Join(configDir, "origin-fed-token"))
+		viper.SetDefault(param.Origin_FedTokenLocation.GetName(), filepath.Join(configDir, "fed-token", "origin-fed-token"))
 	}
 	if currentServers.IsEnabled(server_structs.CacheType) {
-		viper.SetDefault(param.Cache_FedTokenLocation.GetName(), filepath.Join(configDir, "cache-fed-token"))
+		viper.SetDefault(param.Cache_FedTokenLocation.GetName(), filepath.Join(configDir, "fed-token", "cache-fed-token"))
 		os.Setenv("XRD_PELICANCACHETOKENLOCATION", param.Cache_FedTokenLocation.GetString())
 	}
 

server_utils/server_utils.go

@@ -531,6 +531,53 @@ func CreateFedTok(ctx context.Context, server server_structs.XRootDServer) (tok
 	return
 }
 
+// getFedTokenTempDir returns the temporary directory for fed token writes
+// This func is placed here to avoid circular dependencies.
+func getFedTokenTempDir(server server_structs.XRootDServer) string {
+	tokLoc := server.GetFedTokLocation()
+	tokDir := filepath.Dir(tokLoc)
+	dir := filepath.Dir(tokDir)
+	return filepath.Join(dir, "fed-token-temp")
+}
+
+// SetupFedTokDirs creates the permanent and temporary directory for the federation token file
+// and sets proper perms. It intends to be called once at server startup. The permanent directory
+// is always owned by xrootd user. In normal mode, the temporary directory is owned by the root.
+// While in drop privileges mode, it is owned by the pelican user.
+func SetupFedTokDirs(server server_structs.XRootDServer) error {
+	// Permanent directory
+	xrootdUid, err := config.GetDaemonUID()
+	if err != nil {
+		return errors.Wrap(err, "failed to get xrootd user")
+	}
+	xrootdGid, err := config.GetDaemonGID()
+	if err != nil {
+		return errors.Wrap(err, "failed to get xrootd group")
+	}
+	permanentDir := filepath.Dir(server.GetFedTokLocation())
+	err = config.MkdirAll(permanentDir, 0700, xrootdUid, xrootdGid)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create directory %s", permanentDir)
+	}
+
+	// Temporary directory
+	puser, err := config.GetPelicanUser()
+	if err != nil {
+		return errors.Wrap(err, "failed to get pelican user")
+	}
+
+	tempDir := getFedTokenTempDir(server)
+	if tempDir == "" {
+		return errors.New("the temporary directory for federation token writes is an empty string")
+	}
+
+	err = config.MkdirAll(tempDir, 0750, puser.Uid, puser.Gid)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create directory %s", tempDir)
+	}
+	return nil
+}
+
 // FedTokCopyToXrootdFunc is an optional callback used when Server.DropPrivileges is true.
 // It is defined here so that server_utils does not import the xrootd package; the caller
 // (e.g. launchers/cache_serve) wires in xrootd.FileCopyToXrootdDir to avoid import cycles.
@@ -543,14 +590,9 @@ func SetFedTok(ctx context.Context, server server_structs.XRootDServer, tok stri
 		return errors.New("token location is empty")
 	}
 
-	dir := filepath.Dir(tokLoc)
-	if err := os.MkdirAll(dir, 0755); err != nil {
-		return errors.Wrap(err, "failed to create fed token directories")
-	}
-
 	// Create a temporary file for storing the token. Later we'll do an atomic rename
 	filenamePattern := fmt.Sprintf(".fedtoken.%d.*", time.Now().UnixNano())
-	tmpFile, err := os.CreateTemp(dir, filenamePattern)
+	tmpFile, err := os.CreateTemp(getFedTokenTempDir(server), filenamePattern)
 	if err != nil {
 		return errors.Wrap(err, "failed to create temporary token file")
 	}
@@ -561,49 +603,41 @@ func SetFedTok(ctx context.Context, server server_structs.XRootDServer, tok stri
 		os.Remove(tmpName)
 	}()
 
-	// Change ownership to xrootd user
-	uid, err := config.GetDaemonUID()
-	if err != nil {
-		return errors.Wrap(err, "failed to get daemon UID")
-	}
-	gid, err := config.GetDaemonGID()
-	if err != nil {
-		return errors.Wrap(err, "failed to get daemon GID")
-	}
-
-	if err := os.Chown(tmpName, uid, gid); err != nil {
-		return errors.Wrapf(err, "failed to change token file ownership of %s to %d:%d", tmpName, uid, gid)
-	}
-
 	if _, err := tmpFile.WriteString(tok); err != nil {
 		return errors.Wrap(err, "failed to write token to temporary file")
 	}
 
-	if param.Server_DropPrivileges.GetBool() && copyToXrootdDir != nil {
+	if !param.Server_DropPrivileges.GetBool() {
+		// Change ownership to xrootd user
+		uid, err := config.GetDaemonUID()
+		if err != nil {
+			return errors.Wrap(err, "failed to get daemon UID")
+		}
+		gid, err := config.GetDaemonGID()
+		if err != nil {
+			return errors.Wrap(err, "failed to get daemon GID")
+		}
+
+		if err := os.Chown(tmpName, uid, gid); err != nil {
+			return errors.Wrapf(err, "failed to change token file ownership of %s to %d:%d", tmpName, uid, gid)
+		}
+
+		if err := tmpFile.Sync(); err != nil {
+			return errors.Wrap(err, "failed to sync token file")
+		}
+
+		if err := os.Rename(tmpName, tokLoc); err != nil {
+			return errors.Wrap(err, "failed to move token file to final location")
+		}
+	} else if param.Server_DropPrivileges.GetBool() && copyToXrootdDir != nil {
 		// After writing the token content to the file, the file pointer remains at the end.
 		// Seek back to the beginning so the copy operation reads from the start.
 		if _, err := tmpFile.Seek(0, io.SeekStart); err != nil {
 			return errors.Wrap(err, "failed to seek to beginning of the file")
 		}
 		if err := copyToXrootdDir(tmpFile); err != nil {
-			return errors.Wrapf(err, "failed to copy token file to xrootd directory")
-		}
-		if err := tmpFile.Close(); err != nil {
-			return errors.Wrap(err, "failed to close temporary token file")
+			return errors.Wrapf(err, "failed to rename the federation token file via the xrdhttp-pelican plugin")
 		}
-		return nil
-	}
-
-	if err := tmpFile.Sync(); err != nil {
-		return errors.Wrap(err, "failed to sync token file")
-	}
-
-	if err := tmpFile.Close(); err != nil {
-		return errors.Wrap(err, "failed to close temporary token file")
-	}
-
-	if err := os.Rename(tmpName, tokLoc); err != nil {
-		return errors.Wrap(err, "failed to move token file to final location")
 	}
 
 	return nil

server_utils/server_utils_test.go

@@ -268,7 +268,8 @@ func TestSetFedTok(t *testing.T) {
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			if tc.setupDir {
-				require.NoError(t, os.MkdirAll(filepath.Dir(tc.server.tokenLoc), 0755))
+				err := SetupFedTokDirs(tc.server)
+				require.NoError(t, err)
 			}
 
 			err := SetFedTok(context.Background(), tc.server, tc.token, nil)