Use xrdhttp-pelican plugin to put the tester file into the selfTest dir when drop privs is enabled

- Modified the functions in self test package impacted by the drop privs mode
- Tell the plugin what are the test file transplant destinations by setting the environment variables
(hardcoded the destination paths)
- The self-test file is named "self-test-*.txt" (* is a random string decided by os.CreateTemp)
when it gets created in its birthplace, but it is renamed to "self-test-cache-server.txt" and
overwrite the previous file when it is transplanted to the selfTest dir. The original file in
the birthplace will be deleted after the transplant.

Author: h2zh

launchers/cache_serve.go

@@ -114,7 +114,7 @@ func CacheServe(ctx context.Context, engine *gin.Engine, egrp *errgroup.Group, m
 			return nil, err
 		}
 
-		self_monitor.PeriodicCacheSelfTest(ctx, egrp, false)
+		self_monitor.PeriodicSelfTest(ctx, egrp, false)
 	}
 
 	// Director and origin also registers this metadata URL; avoid registering twice.

launchers/origin_serve.go

@@ -110,7 +110,7 @@ func OriginServe(ctx context.Context, engine *gin.Engine, egrp *errgroup.Group,
 	}
 
 	if param.Origin_SelfTest.GetBool() {
-		self_monitor.PeriodicCacheSelfTest(ctx, egrp, true)
+		self_monitor.PeriodicSelfTest(ctx, egrp, true)
 	}
 
 	privileged := param.Origin_Multiuser.GetBool()

self_monitor/self_monitor.go

@@ -41,6 +41,7 @@ import (
 	"github.com/pelicanplatform/pelican/server_utils"
 	"github.com/pelicanplatform/pelican/token"
 	"github.com/pelicanplatform/pelican/token_scopes"
+	"github.com/pelicanplatform/pelican/xrootd"
 )
 
 const (
@@ -62,22 +63,13 @@ func InitSelfTestDir() error {
 	}
 
 	basePath := param.Cache_NamespaceLocation.GetString()
-	pelicanMonPath := filepath.Join(basePath, "/pelican")
-	monitoringPath := filepath.Join(pelicanMonPath, "/monitoring")
-	selfTestPath := filepath.Join(monitoringPath, "/selfTest")
-	err = os.MkdirAll(selfTestPath, 0700)
+	selfTestPath := filepath.Join(basePath, selfTestDir)
+	err = config.MkdirAll(selfTestPath, 0750, uid, gid)
 	if err != nil {
 		return errors.Wrap(err, "Fail to create directory for the self-test")
 	}
-	if err = os.Chown(pelicanMonPath, uid, gid); err != nil {
-		return errors.Wrapf(err, "Unable to change ownership of self-test /pelican directory %v to desired daemon gid %v", monitoringPath, gid)
-	}
-	if err = os.Chown(monitoringPath, uid, gid); err != nil {
-		return errors.Wrapf(err, "Unable to change ownership of self-test /pelican/monitoring directory %v to desired daemon gid %v", monitoringPath, gid)
-	}
-	if err = os.Chown(selfTestPath, uid, gid); err != nil {
-		return errors.Wrapf(err, "Unable to change ownership of self-test /pelican/monitoring directory %v to desired daemon gid %v", monitoringPath, gid)
-	}
+	log.Debugf("Created cache self-test directory at %s", selfTestPath)
+
 	return nil
 }
 
@@ -169,6 +161,94 @@ func generateTestFile() (string, error) {
 	return baseUrl.String(), nil
 }
 
+// generateTestFileViaPlugin creates a test file and its .cinfo file in a temp location (birthplace),
+// then copies them to the selfTestDir using the xrdhttp-pelican plugin. This function is used
+// when drop privileges is enabled, as the pelican server is running as an unprivileged user
+// and cannot directly create files in the selfTestDir.
+func generateTestFileViaPlugin() (string, error) {
+	// Create a temp directory own by pelican user to bypass privilege restrictions, named "birthplace"
+	selfTestBirthplace := filepath.Join(param.Monitoring_DataLocation.GetString(), "selfTest")
+	err := os.MkdirAll(selfTestBirthplace, 0750)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to create selftest directory")
+	}
+
+	// Create a test file and its cinfo in the birthplace
+	testFileBytes := []byte(selfTestBody)
+	fileSize := len(testFileBytes)
+	cinfo := cache.Cinfo{
+		Store: cache.Store{
+			FileSize:     int64(fileSize),
+			CreationTime: time.Now().Unix(),
+			Status:       2,
+		},
+	}
+	cinfoBytes, err := cinfo.Serialize()
+	if err != nil {
+		return "", err
+	}
+
+	file, err := os.CreateTemp(selfTestBirthplace, selfTestPrefix+"*.txt")
+	if err != nil {
+		return "", errors.Wrap(err, "failed to create test file")
+	}
+	cinfoFile, err := os.Create(file.Name() + ".cinfo")
+	if err != nil {
+		return "", errors.Wrap(err, "failed to create test file cinfo")
+	}
+	defer func() {
+		file.Close()
+		cinfoFile.Close()
+		// Delete the test file and its cinfo in the birthplace
+		if err := os.Remove(file.Name()); err != nil {
+			log.Warningf("Failed to remove test file %s: %v", file.Name(), err)
+		}
+		if err := os.Remove(cinfoFile.Name()); err != nil {
+			log.Warningf("Failed to remove test file cinfo %s: %v", cinfoFile.Name(), err)
+		}
+	}()
+
+	// Write test data and cinfo to the files
+	if _, err := file.Write(testFileBytes); err != nil {
+		return "", errors.Wrapf(err, "failed to write test content to self-test file %s", file.Name())
+	}
+	if _, err := cinfoFile.Write(cinfoBytes); err != nil {
+		return "", errors.Wrapf(err, "failed to write cinfo content to self-test cinfo %s", cinfoFile.Name())
+	}
+
+	// After writing the test content to the file, the file pointer remains at the end.
+	// Seek back to the beginning of the file so that the copy operation reads from the start.
+	if _, err := file.Seek(0, io.SeekStart); err != nil {
+		return "", errors.Wrap(err, "failed to seek to beginning of test file")
+	}
+	if _, err := cinfoFile.Seek(0, io.SeekStart); err != nil {
+		return "", errors.Wrap(err, "failed to seek to beginning of cinfo file")
+	}
+
+	// Transplant the test file and cinfo from birthplace to the selfTestDir via xrdhttp-pelican plugin
+	if err = xrootd.SelfTestFileCopy(4, file); err != nil {
+		return "", errors.Wrap(err, "failed to copy the test file to the self-test directory")
+	}
+	if err = xrootd.SelfTestFileCopy(5, cinfoFile); err != nil {
+		return "", errors.Wrap(err, "failed to copy the test cinfo file to the self-test directory")
+	}
+
+	// Construct and return the URL of the copied test file in the selfTestDir
+	cachePort := param.Cache_Port.GetInt()
+	baseUrlStr := fmt.Sprintf("https://%s:%d", param.Server_Hostname.GetString(), cachePort)
+	baseUrl, err := url.Parse(baseUrlStr)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to validate the base url for self-test download")
+	}
+	// This is for web URL path, do not use filepath pkg.
+	// The file name of the test file is always the same in the selfTestDir,
+	// no matter what's the file name in the birthplace.
+	extFilePath := path.Join(selfTestDir, selfTestPrefix+"cache-server.txt")
+	baseUrl.Path = extFilePath
+
+	return baseUrl.String(), nil
+}
+
 func generateFileTestScitoken() (string, error) {
 	issuerUrl := param.Server_ExternalWebUrl.GetString()
 	if issuerUrl == "" { // if both are empty, then error
@@ -227,6 +307,14 @@ func downloadTestFile(ctx context.Context, fileUrl string) error {
 }
 
 func deleteTestFile(fileUrlStr string) error {
+	// If drop privileges is enabled, the test file at selfTestDir will be overwritten by the next self-test,
+	// because the test file and cinfo at selfTestDir always have the same name.
+	// Also, the test file and cinfo in their birthplace were deleted right after generateTestFileViaPlugin,
+	// so this function can be skipped.
+	if param.Server_DropPrivileges.GetBool() {
+		return nil
+	}
+
 	basePath := param.Cache_NamespaceLocation.GetString()
 	fileUrl, err := url.Parse(fileUrlStr)
 	if err != nil {
@@ -250,18 +338,30 @@ func deleteTestFile(fileUrlStr string) error {
 }
 
 func runSelfTest(ctx context.Context) (bool, error) {
-	fileUrl, err := generateTestFile()
-	if err != nil {
-		return false, errors.Wrap(err, "self-test failed when generating the file")
+	var fileUrl string
+	var err error
+	if param.Server_DropPrivileges.GetBool() {
+		fileUrl, err = generateTestFileViaPlugin()
+		if err != nil {
+			return false, errors.Wrap(err, "self-test failed when generating the file")
+		}
+	} else {
+		fileUrl, err = generateTestFile()
+		if err != nil {
+			return false, errors.Wrap(err, "self-test failed when generating the file")
+		}
 	}
+
 	err = downloadTestFile(ctx, fileUrl)
 	if err != nil {
+		log.Warningf("Self-test download failed for file %s; err: %v", fileUrl, err)
 		errDel := deleteTestFile(fileUrl)
 		if errDel != nil {
 			return false, errors.Wrap(errDel, "self-test failed during delete")
 		}
 		return false, errors.Wrap(err, "self-test failed during download. File is cleaned up at "+fileUrl)
 	}
+
 	err = deleteTestFile(fileUrl)
 	if err != nil {
 		return false, errors.Wrap(err, "self-test failed during delete")
@@ -304,22 +404,23 @@ func doSelfMonitorOrigin(ctx context.Context) {
 // Start self-test monitoring of the origin/cache.  This will upload, download, and delete
 // a generated filename every 15 seconds to the local origin.  On failure, it will
 // set the xrootd component's status to critical.
-func PeriodicCacheSelfTest(ctx context.Context, ergp *errgroup.Group, isOrigin bool) {
+func PeriodicSelfTest(ctx context.Context, ergp *errgroup.Group, isOrigin bool) {
+	customInterval := param.Cache_SelfTestInterval.GetDuration()
 	doSelfMonitor := doSelfMonitorCache
 	if isOrigin {
 		doSelfMonitor = doSelfMonitorOrigin
+		customInterval = param.Origin_SelfTestInterval.GetDuration()
 	}
 
-	customInterval := param.Cache_SelfTestInterval.GetDuration()
 	if customInterval == 0 {
 		customInterval = 15 * time.Second
-		log.Error("Invalid config value: Cache.SelfTestInterval is 0. Fallback to 15s.")
+		log.Error("Invalid config value: both Origin.SelfTestInterval and Cache.SelfTestInterval are 0. Fallback to 15s.")
 	}
 	ticker := time.NewTicker(customInterval)
-	defer ticker.Stop()
 	firstRound := time.After(5 * time.Second)
 
 	ergp.Go(func() error {
+		defer ticker.Stop()
 		for {
 			select {
 			case <-firstRound:

xrootd/launch.go

@@ -147,6 +147,12 @@ func makeUnprivilegedXrootdLauncher(daemonName string, configPath string, isCach
 		result.ExtraEnv = append(result.ExtraEnv, "XRDHTTP_PELICAN_CA_FILE="+filepath.Join(xrootdRun, "ca-bundle.crt"))
 		result.ExtraEnv = append(result.ExtraEnv, "XRDHTTP_PELICAN_CERT_FILE="+filepath.Join(xrootdRun, "copied-tls-creds.crt"))
 		result.ExtraEnv = append(result.ExtraEnv, "XRDHTTP_PELICAN_INFO_FD="+strconv.Itoa(result.fds[1]))
+
+		basePath := param.Cache_NamespaceLocation.GetString() + "/pelican/monitoring/selfTest"
+		testFileLocation := basePath + "/self-test-cache-server.txt"
+		testFileCinfoLocation := basePath + "/self-test-cache-server.txt.cinfo"
+		result.ExtraEnv = append(result.ExtraEnv, "XRDHTTP_PELICAN_CACHE_SELF_TEST_FILE="+testFileLocation)
+		result.ExtraEnv = append(result.ExtraEnv, "XRDHTTP_PELICAN_CACHE_SELF_TEST_FILE_CINFO="+testFileCinfoLocation)
 	}
 	return
 }

xrootd/xrootd_config.go

@@ -614,10 +614,17 @@ func copyXrootdCertificates(server server_structs.XRootDServer) error {
 	return nil
 }
 
+func SelfTestFileCopy(cmd int, file *os.File) error {
+	log.Debug("Transplanting a self-test file")
+	if err := sendChildFD(false, cmd, file); err != nil {
+		return errors.Wrap(err, "Failed to copy the self-test file via FD")
+	}
+	return nil
+}
+
 // After privileges have been dropped, copy the server certificates
 // to the xrootd process.
 func dropPrivilegeCopy(server server_structs.XRootDServer) error {
-
 	certFile := param.Server_TLSCertificateChain.GetString()
 	certKey := param.Server_TLSKey.GetString()
 	if _, err := tls.LoadX509KeyPair(certFile, certKey); err != nil {