Skip to content

Commit c5ebff4

Browse files
MuneebUllahKhan222MuneebUllahKhan222
authored
[INS-470] Add Tly detector to defaults.go, gate it behind feat flag and update its verification logic (#5006)
* refactored tly detector
1 parent ba02c92 commit c5ebff4

6 files changed

Lines changed: 83 additions & 26 deletions

File tree

‎main.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -534,6 +534,7 @@ func run(state overseer.State, logSync func() error) {
534534
feature.GitLabOAuthDetectorEnabled.Store(true)
535535
feature.EnigmaDetectorEnabled.Store(true)
536536
feature.DatadogApiKeyDetectorEnabled.Store(true)
537+
feature.TlyDetectorEnabled.Store(true)
537538

538539
conf := &config.Config{}
539540
if *configFilename != "" {

‎pkg/detectors/tly/tly.go‎

Lines changed: 61 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ package tly
22

33
import (
44
"context"
5+
"fmt"
6+
"io"
57
"net/http"
68
"strings"
79

@@ -31,38 +33,76 @@ func (s Scanner) Keywords() []string {
3133
}
3234

3335
// FromData will find and optionally verify TLy secrets in a given set of bytes.
34-
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
35-
dataStr := string(data)
36+
func (s Scanner) FromData(
37+
ctx context.Context,
38+
verify bool,
39+
data []byte,
40+
) (results []detectors.Result, err error) {
3641

37-
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
42+
dataStr := string(data)
3843

39-
for _, match := range matches {
40-
resMatch := strings.TrimSpace(match[1])
44+
uniqueKeys := make(map[string]struct{})
45+
for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) {
46+
uniqueKeys[strings.TrimSpace(match[1])] = struct{}{}
47+
}
4148

42-
s1 := detectors.Result{
49+
for key := range uniqueKeys {
50+
result := detectors.Result{
4351
DetectorType: detector_typepb.DetectorType_TLy,
44-
Raw: []byte(resMatch),
45-
SecretParts: map[string]string{"key": resMatch},
52+
Raw: []byte(key),
53+
SecretParts: map[string]string{
54+
"key": key,
55+
},
4656
}
4757

4858
if verify {
49-
req, err := http.NewRequestWithContext(ctx, "GET", "https://t.ly/api/v1/link/stats?api_token="+resMatch+"&short_url=https://t.ly/h9YS", nil)
50-
if err != nil {
51-
continue
52-
}
53-
res, err := client.Do(req)
54-
if err == nil {
55-
defer func() { _ = res.Body.Close() }()
56-
if res.StatusCode >= 200 && res.StatusCode < 300 {
57-
s1.Verified = true
58-
}
59-
}
59+
verified, verificationErr := verifyTLyKey(ctx, client, key)
60+
result.SetVerificationError(verificationErr, key)
61+
result.Verified = verified
6062
}
6163

62-
results = append(results, s1)
64+
results = append(results, result)
65+
}
66+
67+
return
68+
}
69+
70+
func verifyTLyKey(
71+
ctx context.Context,
72+
client *http.Client,
73+
key string,
74+
) (bool, error) {
75+
76+
req, err := http.NewRequestWithContext(
77+
ctx,
78+
http.MethodGet,
79+
"https://api.t.ly/api/v1/link/list",
80+
http.NoBody,
81+
)
82+
if err != nil {
83+
return false, err
6384
}
6485

65-
return results, nil
86+
req.Header.Set("Authorization", "Bearer "+key)
87+
req.Header.Set("Accept", "application/json")
88+
89+
res, err := client.Do(req)
90+
if err != nil {
91+
return false, err
92+
}
93+
defer func() {
94+
_, _ = io.Copy(io.Discard, res.Body)
95+
_ = res.Body.Close()
96+
}()
97+
98+
switch res.StatusCode {
99+
case http.StatusOK:
100+
return true, nil
101+
case http.StatusUnauthorized, http.StatusForbidden:
102+
return false, nil
103+
default:
104+
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
105+
}
66106
}
67107

68108
func (s Scanner) Type() detector_typepb.DetectorType {

‎pkg/detectors/tly/tly_integration_test.go‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ import (
99
"testing"
1010
"time"
1111

12-
"github.com/kylelemons/godebug/pretty"
13-
12+
"github.com/google/go-cmp/cmp"
13+
"github.com/google/go-cmp/cmp/cmpopts"
1414
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
1515
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
1616
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"
@@ -96,8 +96,19 @@ func TestTLy_FromChunk(t *testing.T) {
9696
}
9797
got[i].Raw = nil
9898
}
99-
if diff := pretty.Compare(got, tt.want); diff != "" {
100-
t.Errorf("TLy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
99+
100+
ignoreOpts := cmpopts.IgnoreFields(
101+
detectors.Result{},
102+
"ExtraData",
103+
"verificationError",
104+
"primarySecret",
105+
"SecretParts",
106+
"chunkOffset",
107+
"chunkOffsetSet",
108+
)
109+
110+
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
111+
t.Errorf("Tly.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
101112
}
102113
})
103114
}

‎pkg/engine/defaults/defaults.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -773,6 +773,7 @@ import (
773773
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/timecamp"
774774
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/timezoneapi"
775775
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tineswebhook"
776+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tly"
776777
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tmetric"
777778
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/todoist"
778779
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tokeet"
@@ -1671,6 +1672,7 @@ func buildDetectorList() []detectors.Detector {
16711672
&timecamp.Scanner{},
16721673
&timezoneapi.Scanner{},
16731674
&tineswebhook.Scanner{},
1675+
&tly.Scanner{},
16741676
&tmetric.Scanner{},
16751677
&todoist.Scanner{},
16761678
// &toggltrack.Scanner{},
@@ -1784,6 +1786,8 @@ func buildDetectorList() []detectors.Detector {
17841786
return !feature.EnigmaDetectorEnabled.Load()
17851787
case *datadogapikey.Scanner:
17861788
return !feature.DatadogApiKeyDetectorEnabled.Load()
1789+
case *tly.Scanner:
1790+
return !feature.TlyDetectorEnabled.Load()
17871791
default:
17881792
return false
17891793
}

‎pkg/engine/defaults/defaults_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,6 @@ var excludedFromDefaultList = map[detector_typepb.DetectorType]struct{}{
122122
detector_typepb.DetectorType_IPInfo: {},
123123
detector_typepb.DetectorType_Lob: {},
124124
detector_typepb.DetectorType_Rev: {},
125-
detector_typepb.DetectorType_TLy: {},
126125
detector_typepb.DetectorType_Tru: {},
127126
detector_typepb.DetectorType_User: {},
128127
detector_typepb.DetectorType_Wit: {},
@@ -134,6 +133,7 @@ var excludedFromDefaultList = map[detector_typepb.DetectorType]struct{}{
134133
detector_typepb.DetectorType_Enigma: {},
135134
detector_typepb.DetectorType_GitLabOauth2: {},
136135
detector_typepb.DetectorType_Pinecone: {},
136+
detector_typepb.DetectorType_TLy: {},
137137

138138
// Reserved / special types.
139139
detector_typepb.DetectorType_CustomRegex: {}, // added dynamically via engine config, not via buildDetectorList()

‎pkg/feature/feature.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ var (
2121
GitLabOAuthDetectorEnabled atomic.Bool
2222
EnigmaDetectorEnabled atomic.Bool
2323
DatadogApiKeyDetectorEnabled atomic.Bool
24+
TlyDetectorEnabled atomic.Bool
2425
)
2526

2627
type AtomicString struct {

0 commit comments

Comments
 (0)
⚡