fix: scan files with lines exceeding bufio's default 64 KB token limit (#5022)

* fix: scan files with lines exceeding bufio's default 64 KB token limit

* added related test

Author: MuneebUllahKhan222

pkg/sources/git/git.go

@@ -750,7 +750,6 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
 
 		email := commit.Author
 		when := commit.Date.UTC().Format("2006-01-02 15:04:05 -0700")
-
 		if fullHash != lastCommitHash {
 			depth++
 			lastCommitHash = fullHash
@@ -901,6 +900,14 @@ func (s *Git) gitChunk(ctx context.Context, diff *gitparse.Diff, fileName, email
 	defer func() { _ = reader.Close() }()
 
 	originalChunk := bufio.NewScanner(reader)
+	// Default bufio max token size (64 KB) is too small for files with long lines
+	// (e.g. minified JS, base64 blobs). Raise the cap to 10 MB so those lines
+	// are still scanned; the oversize-line path below will chunk them correctly.
+	// The initial buffer starts at 4 KB (same as bufio's default) and grows only
+	// when a line actually exceeds the current size, keeping allocations cheap for
+	// the common case of small diffs.
+	const maxScanTokenSize = 10 * 1024 * 1024
+	originalChunk.Buffer(make([]byte, 4096), maxScanTokenSize)
 	newChunkBuffer := bytes.Buffer{}
 	lastOffset := 0
 	for offset := 0; originalChunk.Scan(); offset++ {
@@ -967,6 +974,9 @@ func (s *Git) gitChunk(ctx context.Context, diff *gitparse.Diff, fileName, email
 			ctx.Logger().Error(err, "error writing to chunk buffer", "filename", fileName, "commit", hash, "file", diff.PathB)
 		}
 	}
+	if err := originalChunk.Err(); err != nil {
+		ctx.Logger().Error(err, "error scanning chunk", "filename", fileName, "commit", hash, "file", diff.PathB)
+	}
 	// Send anything still in the new chunk buffer
 	if newChunkBuffer.Len() > 0 {
 		metadata := s.sourceMetadataFunc(SourceMetadataInfo{

pkg/sources/git/git_test.go

@@ -8,6 +8,7 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -1163,3 +1164,47 @@ func TestPrepareRepoWithNormalizationBare(t *testing.T) {
 		})
 	}
 }
+
+// TestGitChunk_LongLine verifies that files containing lines longer than
+// bufio's default 64 KB token limit are still scanned. Before the fix,
+// bufio.Scanner would silently stop on the first oversized line and produce
+// zero chunks for that file.
+func TestGitChunk_LongLine(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+
+	repoPath := setupTestRepo(t, "long-line-repo")
+
+	// Build a single line that is 100 KB — well above the old 64 KB cap.
+	longLine := strings.Repeat("a", 100*1024)
+	addTestFileAndCommit(t, repoPath, "long_line.txt", longLine)
+
+	conn, err := anypb.New(&sourcespb.Git{
+		Credential: &sourcespb.Git_Unauthenticated{
+			Unauthenticated: &credentialspb.Unauthenticated{},
+		},
+		Repositories: []string{"file://" + repoPath},
+	})
+	assert.NoError(t, err)
+
+	s := Source{}
+	assert.NoError(t, s.Init(ctx, "test long line", 0, 0, false, conn, 1))
+
+	chunksCh := make(chan *sources.Chunk, 64)
+	var count int
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for range chunksCh {
+			count++
+		}
+	}()
+
+	assert.NoError(t, s.Chunks(ctx, chunksCh))
+	close(chunksCh)
+	wg.Wait()
+	// ensure the goroutine has finished writing to count before we read it
+	// one chunk for the commit/file metadata, and at least one chunk for the file content
+	assert.Equal(t, 2, count, "expected two chunks from a file with a 100 KB line")
+}