fix: resolve Git Anonymous Resolver excessive memory usage

Switch git resolver from go-git library to use git binary.

The go-git library does not resolve deltas efficiently, and as a result
is not recommended to be used to clone repositories with full-depth.
In one example RemoteResolutionRequest targeting a repository which
summed 145Mb, configuring the resolution timeout to 10 minutes and
giving the resolver to have 25Gb of memory, the resolver pod was OOM
killed after ~6 minutes. Additionally, since go-git's delta resolution
does not accept any contexts, the time required and memory used during
resolving a large repository will not be cutoff when the context is
canceled, impacting the resolver's performance for other concurrent
remote resolutions.

Since the git resolver can be provided a revision which is not tracked
at any ref in the repository, and because go-git only supports fetching
fully-qualified refs, go-git does not support fetching arbitrary
revisions. Therefore, in order to guarantee the requested revision is
fetched, if we continue to use the go-git library we must fetch all
revisions.

Switching to the git binary enables the git resolver to take advantage
of the git-fetch's support for fetching arbitrary revisions. Note that
if the revision is not at any ref head, fetching the revision does
depend on the git server enabling uploadpack.allowReachableSHA1InWant.

Resolves #8652

See also: https://git-scm.com/docs/protocol-capabilities#_allow_reachable_sha1_in_want

NOTE: This feature is enabled and supported in Github and Gitlab but
not Gitea: go-gitea/gitea#11958

Author: aThorp96

.ko.yaml

@@ -0,0 +1,2 @@
+baseImageOverrides:
+  github.com/tektoncd/pipeline/cmd/resolvers: cgr.dev/chainguard/git@sha256:566235a8ef752f37d285042ee05fc37dbb04293e50f116a231984080fb835693

config/resolvers/resolvers-deployment.yaml

@@ -106,6 +106,9 @@ spec:
           value: "https://artifacthub.io/"
         - name: TEKTON_HUB_API
           value: "https://api.hub.tekton.dev/"
+        volumeMounts:
+          - name: tmp-clone-volume
+            mountPath: "/tmp"
         securityContext:
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true
@@ -115,3 +118,7 @@ spec:
             - "ALL"
           seccompProfile:
             type: RuntimeDefault
+      volumes:
+        - name: tmp-clone-volume
+          emptyDir:
+            sizeLimit: 4Gi

docs/git-resolver.md

@@ -76,6 +76,15 @@ The differences between the two modes are:
 ### Git Clone with git clone
 
 Git clone with `git clone` is supported for anonymous and authenticated cloning.
+This mode shallow clones the git repo before fetching and checking out the
+provided revision.
+
+**Note**: if the revision is a commit SHA which is not pointed-at by a Branch
+or Tag ref, the revision might not be able to be fetched, depending on the
+git provider's [uploadpack.allowReachableSHA1InWant](https://git-scm.com/docs/protocol-capabilities#_allow_reachable_sha1_in_want)
+setting. This is not an issue for major git providers such as Github and
+Gitlab, but may be of note for smaller or self-hosted providers such as
+Gitea.
 
 #### Task Resolution
 

pkg/remoteresolution/resolver/git/resolver_test.go

@@ -429,7 +429,7 @@ func TestResolve(t *testing.T) {
 			pathInRepo: "foo/new",
 			url:        anonFakeRepoURL,
 		},
-		expectedErr: createError("revision error: reference not found"),
+		expectedErr: createError("git fetch error: fatal: couldn't find remote ref non-existent-revision: exit status 128"),
 	}, {
 		name: "api: successful task from params api information",
 		args: &params{

pkg/resolution/resolver/git/repository.go

@@ -0,0 +1,149 @@
+/*
+Copyright 2025 The Tekton Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package git
+
+import (
+	"context"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+type cmdExecutor = func(context.Context, string, ...string) *exec.Cmd
+
+type remote struct {
+	url         string
+	username    string
+	password    string
+	cmdExecutor cmdExecutor
+}
+
+func (r remote) clone(ctx context.Context) (*repository, func(), error) {
+	urlParts := strings.Split(r.url, "/")
+	repoName := urlParts[len(urlParts)-1]
+	tmpDir, err := os.MkdirTemp("", repoName+"-*")
+	if err != nil {
+		return nil, func() {}, err
+	}
+	cleanupFunc := func() {
+		os.RemoveAll(tmpDir)
+	}
+
+	repo := repository{
+		url:       r.url,
+		username:  r.username,
+		password:  r.password,
+		directory: tmpDir,
+		executor:  r.cmdExecutor,
+	}
+
+	_, err = repo.execGit(ctx, "clone", repo.url, tmpDir, "--depth=1", "--no-checkout")
+	if err != nil {
+		if strings.Contains(err.Error(), "could not read Username") {
+			err = errors.New("clone error: authentication required")
+		}
+		return nil, cleanupFunc, err
+	}
+	return &repo, cleanupFunc, nil
+}
+
+type repository struct {
+	url       string
+	username  string
+	password  string
+	directory string
+	executor  cmdExecutor
+}
+
+func (repo *repository) currentRevision(ctx context.Context) (string, error) {
+	revisionSha, err := repo.execGit(ctx, "rev-list", "-n1", "HEAD")
+	if err != nil {
+		return "", err
+	}
+	return strings.TrimSpace(string(revisionSha)), nil
+}
+
+func (repo *repository) checkout(ctx context.Context, revision string) error {
+	_, err := repo.execGit(ctx, "fetch", "origin", revision, "--depth=1")
+	if err != nil {
+		return err
+	}
+
+	_, err = repo.execGit(ctx, "checkout", "FETCH_HEAD")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (repo *repository) execGit(ctx context.Context, subCmd string, args ...string) ([]byte, error) {
+	if repo.executor == nil {
+		repo.executor = exec.CommandContext
+	}
+
+	args = append([]string{subCmd}, args...)
+
+	// We need to configure  which directory contains the cloned repository since `cd`ing
+	// into the repository directory is not concurrency-safe
+	configArgs := []string{"-C", repo.directory}
+	env := []string{"GIT_TERMINAL_PROMPT=false"}
+	if subCmd == "clone" {
+		// NOTE: Since this is only HTTP basic auth, authentication only supports http
+		// cloning, while unauthenticated cloning works for any other protocol supported
+		// by the git binary which doesn't require authentication.
+		if repo.username != "" && repo.password != "" {
+			token := base64.URLEncoding.EncodeToString([]byte(repo.username + ":" + repo.password))
+			env = append(
+				env,
+				"GIT_AUTH_HEADER=Authorization=Basic "+token,
+			)
+			configArgs = append(configArgs, "--config-env", "http.extraHeader=GIT_AUTH_HEADER")
+		}
+	}
+	cmd := repo.executor(ctx, "git", append(configArgs, args...)...)
+	cmd.Env = append(cmd.Env, env...)
+
+	out, err := cmd.Output()
+	if err != nil {
+		msg := string(out)
+		var exitErr *exec.ExitError
+		if errors.As(err, &exitErr) {
+			msg = string(exitErr.Stderr)
+		}
+		err = fmt.Errorf("git %s error: %s: %w", subCmd, strings.TrimSpace(msg), err)
+	}
+	return out, err
+}
+
+func (repo *repository) getFileContent(path string) ([]byte, error) {
+	if _, err := os.Stat(repo.directory); errors.Is(err, os.ErrNotExist) {
+		return nil, fmt.Errorf("repository clone no longer exists, used after cleaned? %w", err)
+	}
+	fileContents, err := os.ReadFile(filepath.Join(repo.directory, path))
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil, errors.New("file does not exist")
+		}
+		return nil, err
+	}
+	return fileContents, nil
+}

pkg/resolution/resolver/git/repository_test.go

@@ -0,0 +1,164 @@
+/*
+ Copyright 2025 The Tekton Authors
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+*/
+
+package git
+
+import (
+	"context"
+	"encoding/base64"
+	"os/exec"
+	"reflect"
+	"testing"
+)
+
+func TestClone(t *testing.T) {
+	type testCase struct {
+		url       string
+		username  string
+		password  string
+		expectErr string
+	}
+
+	testCases := map[string]testCase{
+		"normal usage":           {url: "https://github.com/tektoncd/pipeline"},
+		"normal usage with .git": {url: "https://github.com/tektoncd/pipeline.git"},
+		"private repository":     {url: "https://github.com/tektoncd/not-a-repository.git"},
+		"with crendentials":      {url: "https://github.com/tektoncd/not-a-repository.git", username: "fake", password: "fake"},
+	}
+
+	for name, test := range testCases {
+		t.Run(name, func(t *testing.T) {
+			executions := []*exec.Cmd{}
+
+			executor := func(ctx context.Context, name string, args ...string) *exec.Cmd {
+				args = append([]string{name}, args...)
+				// Run the command as `echo` args to avoid side effects
+				cmd := exec.CommandContext(ctx, "echo", args...)
+				executions = append(executions, cmd)
+				return cmd
+			}
+
+			mockCmdRemote := remote{url: test.url, username: test.username, password: test.password, cmdExecutor: executor}
+			repo, cleanup, err := mockCmdRemote.clone(context.Background())
+			defer cleanup()
+			if test.expectErr != "" {
+				if err.Error() != test.expectErr {
+					t.Fatalf("Expected error %q but got %q", test.expectErr, err)
+				}
+			} else {
+				if err != nil {
+					t.Fatalf("Error cloning repository %q: %v", test.url, err)
+				}
+			}
+
+			expectedEnv := []string{"GIT_TERMINAL_PROMPT=false"}
+			expectedCmd := []string{"git", "-C", repo.directory}
+			if test.username != "" {
+				token := base64.URLEncoding.EncodeToString([]byte(test.username + ":" + test.password))
+				expectedCmd = append(expectedCmd, "--config-env", "http.extraHeader=GIT_AUTH_HEADER")
+				expectedEnv = append(expectedEnv, "GIT_AUTH_HEADER=Authorization=Basic "+token)
+			}
+			expectedCmd = append(expectedCmd, "clone", test.url, repo.directory, "--depth=1", "--no-checkout")
+
+			if len(executions) != 1 {
+				t.Fatalf("Expected 1 command execution during cloning, got %d: %v", len(executions), executions)
+			}
+
+			cmd := executions[0]
+			// Remove the `echo` prefix
+			cmdParts := cmd.Args[1:]
+			if !reflect.DeepEqual(cmdParts, expectedCmd) {
+				t.Fatalf("Expected clone command to be %v but got %v", expectedCmd, cmdParts)
+			}
+			if !reflect.DeepEqual(cmd.Env, expectedEnv) {
+				t.Fatalf("Expected clone command env vars to be %v but got %v", expectedEnv, cmd.Env)
+			}
+		})
+	}
+}
+
+func TestCheckout(t *testing.T) {
+	repoPath, revisions := createTestRepo(
+		t,
+		[]commitForRepo{
+			{
+				Filename: "README.md",
+				Content:  "some content",
+				Branch:   "non-main",
+				Tag:      "1.0.0",
+			},
+			{
+				Filename: "otherfile.yaml",
+				Content:  "some data",
+				Branch:   "to-be-deleted",
+			},
+		},
+	)
+	gitCmd := getGitCmd(t, repoPath)
+	if err := gitCmd("checkout", "main").Run(); err != nil {
+		t.Fatalf("cloud not checkout main branch after repo initialization: %v", err)
+	}
+	if err := gitCmd("branch", "-D", "to-be-deleted").Run(); err != nil {
+		t.Fatalf("coun't delete branch to orphan commit: %v", err)
+	}
+
+	ctx := context.Background()
+
+	type testCase struct {
+		revision         string
+		expectedRevision string
+		expectErr        string
+	}
+	testCases := map[string]testCase{
+		"revision is branch":          {revision: "non-main", expectedRevision: revisions[0]},
+		"revision is tag":             {revision: "1.0.0", expectedRevision: revisions[0]},
+		"revision is sha":             {revision: revisions[0], expectedRevision: revisions[0]},
+		"revision is unreachable sha": {revision: revisions[1], expectedRevision: revisions[1]},
+		"non-existent revision":       {revision: "fake-revision", expectErr: "git fetch error: fatal: couldn't find remote ref fake-revision: exit status 128"},
+	}
+
+	for name, test := range testCases {
+		t.Run(name, func(t *testing.T) {
+			repo, cleanup, err := remote{url: repoPath}.clone(ctx)
+			defer cleanup()
+
+			if err != nil {
+				t.Fatalf("Error cloning repository %v", err)
+			}
+
+			err = repo.checkout(ctx, test.revision)
+			if test.expectErr != "" {
+				if err == nil {
+					t.Fatal("Expected error checking out revision but got none")
+				} else if err.Error() != test.expectErr {
+					t.Fatalf("Expected error %q but got %q", test.expectErr, err)
+				}
+				return
+			} else if err != nil {
+				t.Fatalf("Error checking out revision: %v", err)
+			}
+
+			revision, err := repo.currentRevision(ctx)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if revision != test.expectedRevision {
+				t.Fatalf("Expected revision to be %q but got %q", test.expectedRevision, revision)
+			}
+		})
+	}
+}