Skip to content

Commit 2d008a5

Browse files
mustansir14mustansir14
authored
[INS-204] Abort Postman scan if monthly API request limit crosses 80% (#4586)
1 parent b6389e2 commit 2d008a5

File tree

3 files changed

+142
-21
lines changed

3 files changed

+142
-21
lines changed

‎pkg/sources/postman/postman.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,9 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
186186
for _, workspaceID := range s.conn.Workspaces {
187187
w, err := s.client.GetWorkspace(ctx, workspaceID)
188188
if err != nil {
189+
if errors.Is(err, errAbortScanDueToAPIRateLimit) {
190+
return err
191+
}
189192
// Log and move on, because sometimes the Postman API seems to give us workspace IDs
190193
// that we don't have access to, so we don't want to kill the scan because of it.
191194
ctx.Logger().Error(err, "error getting workspace %s", workspaceID)
@@ -206,6 +209,9 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
206209

207210
collection, err := s.client.GetCollection(ctx, collectionID)
208211
if err != nil {
212+
if errors.Is(err, errAbortScanDueToAPIRateLimit) {
213+
return err
214+
}
209215
// Log and move on, because sometimes the Postman API seems to give us collection IDs
210216
// that we don't have access to, so we don't want to kill the scan because of it.
211217
ctx.Logger().Error(err, "error getting collection %s", collectionID)
@@ -279,6 +285,9 @@ func (s *Source) scanWorkspace(ctx context.Context, chunksChan chan *sources.Chu
279285
for _, envID := range workspace.Environments {
280286
envVars, err := s.client.GetEnvironmentVariables(ctx, envID.Uid)
281287
if err != nil {
288+
if errors.Is(err, errAbortScanDueToAPIRateLimit) {
289+
return err
290+
}
282291
ctx.Logger().Error(err, "could not get env variables", "environment_uuid", envID.Uid)
283292
continue
284293
}
@@ -316,6 +325,9 @@ func (s *Source) scanWorkspace(ctx context.Context, chunksChan chan *sources.Chu
316325
}
317326
collection, err := s.client.GetCollection(ctx, collectionID.Uid)
318327
if err != nil {
328+
if errors.Is(err, errAbortScanDueToAPIRateLimit) {
329+
return err
330+
}
319331
// Log and move on, because sometimes the Postman API seems to give us collection IDs
320332
// that we don't have access to, so we don't want to kill the scan because of it.
321333
ctx.Logger().Error(err, "error getting collection %s", collectionID)

‎pkg/sources/postman/postman_client.go‎

Lines changed: 65 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,16 @@ import (
1818
)
1919

2020
const (
21-
WORKSPACE_URL = "https://api.getpostman.com/workspaces/%s"
22-
ENVIRONMENTS_URL = "https://api.getpostman.com/environments/%s"
23-
COLLECTIONS_URL = "https://api.getpostman.com/collections/%s"
24-
userAgent = "PostmanRuntime/7.26.8"
25-
defaultContentType = "*"
21+
WORKSPACE_URL = "https://api.getpostman.com/workspaces/%s"
22+
ENVIRONMENTS_URL = "https://api.getpostman.com/environments/%s"
23+
COLLECTIONS_URL = "https://api.getpostman.com/collections/%s"
24+
userAgent = "PostmanRuntime/7.26.8"
25+
defaultContentType = "*"
26+
abortScanAPIReqLimitThreshold = 0.8 // Abort scan if 80% of monthly api request limit is reached
2627
)
2728

29+
var errAbortScanDueToAPIRateLimit = fmt.Errorf("aborting scan due to Postman API monthly requests limit being used over %f%%", abortScanAPIReqLimitThreshold*100)
30+
2831
type Workspace struct {
2932
Id string `json:"id"`
3033
Name string `json:"name"`
@@ -284,22 +287,9 @@ func (c *Client) getPostmanResponseBodyBytes(ctx trContext.Context, urlString st
284287

285288
c.Metrics.apiRequests.WithLabelValues(urlString).Inc()
286289

287-
rateLimitRemainingMonthValue := resp.Header.Get("RateLimit-Remaining-Month")
288-
if rateLimitRemainingMonthValue == "" {
289-
rateLimitRemainingMonthValue = resp.Header.Get("X-RateLimit-Remaining-Month")
290-
}
291-
292-
if rateLimitRemainingMonthValue != "" {
293-
rateLimitRemainingMonth, err := strconv.Atoi(rateLimitRemainingMonthValue)
294-
if err != nil {
295-
ctx.Logger().Error(err, "Couldn't convert RateLimit-Remaining-Month to an int",
296-
"header_value", rateLimitRemainingMonthValue,
297-
)
298-
} else {
299-
c.Metrics.apiMonthlyRequestsRemaining.WithLabelValues().Set(
300-
float64(rateLimitRemainingMonth),
301-
)
302-
}
290+
err = c.handleRateLimits(ctx, resp)
291+
if err != nil {
292+
return nil, err
303293
}
304294

305295
body, err := io.ReadAll(resp.Body)
@@ -315,6 +305,60 @@ func (c *Client) getPostmanResponseBodyBytes(ctx trContext.Context, urlString st
315305
return body, nil
316306
}
317307

308+
// handleRateLimits processes the rate limit headers from the Postman API response
309+
// and updates the client's metrics accordingly. If the monthly rate limit usage exceeds
310+
// the set threshold, an error is returned to abort further processing.
311+
func (c *Client) handleRateLimits(ctx trContext.Context, resp *http.Response) error {
312+
rateLimitRemainingMonthValue := resp.Header.Get("RateLimit-Remaining-Month")
313+
if rateLimitRemainingMonthValue == "" {
314+
rateLimitRemainingMonthValue = resp.Header.Get("X-RateLimit-Remaining-Month")
315+
}
316+
if rateLimitRemainingMonthValue == "" {
317+
ctx.Logger().V(2).Info("RateLimit-Remaining-Month header not found in response")
318+
return nil
319+
}
320+
321+
rateLimitRemainingMonth, err := strconv.Atoi(rateLimitRemainingMonthValue)
322+
if err != nil {
323+
ctx.Logger().Error(err, "Couldn't convert RateLimit-Remaining-Month to an int",
324+
"header_value", rateLimitRemainingMonthValue,
325+
)
326+
return nil
327+
}
328+
c.Metrics.apiMonthlyRequestsRemaining.WithLabelValues().Set(
329+
float64(rateLimitRemainingMonth),
330+
)
331+
332+
rateLimitTotalMonthValue := resp.Header.Get("RateLimit-Limit-Month")
333+
if rateLimitTotalMonthValue == "" {
334+
rateLimitTotalMonthValue = resp.Header.Get("X-RateLimit-Limit-Month")
335+
}
336+
if rateLimitTotalMonthValue == "" {
337+
ctx.Logger().V(2).Info("RateLimit-Limit-Month header not found in response")
338+
return nil
339+
}
340+
rateLimitTotalMonth, err := strconv.Atoi(rateLimitTotalMonthValue)
341+
if err != nil {
342+
ctx.Logger().Error(err, "Couldn't convert RateLimit-Limit-Month to an int",
343+
"header_value", rateLimitTotalMonthValue,
344+
)
345+
return nil
346+
}
347+
348+
if rateLimitTotalMonth == 0 {
349+
ctx.Logger().V(2).Info("RateLimit-Limit-Month is zero, cannot compute usage percentage")
350+
return nil
351+
}
352+
353+
// failsafe to abandon scan if we are over the threshold of monthly API requests used
354+
percentageUsed := float64(rateLimitTotalMonth-rateLimitRemainingMonth) / float64(rateLimitTotalMonth)
355+
if percentageUsed > abortScanAPIReqLimitThreshold {
356+
return errAbortScanDueToAPIRateLimit
357+
}
358+
359+
return nil
360+
}
361+
318362
// EnumerateWorkspaces returns the workspaces for a given user (both private, public, team and personal).
319363
// Consider adding additional flags to support filtering.
320364
func (c *Client) EnumerateWorkspaces(ctx trContext.Context) ([]Workspace, error) {

‎pkg/sources/postman/postman_test.go‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"testing"
99
"time"
1010

11+
"github.com/go-errors/errors"
1112
"github.com/stretchr/testify/assert"
1213

1314
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
@@ -734,3 +735,67 @@ func TestSource_ResponseID(t *testing.T) {
734735
}
735736

736737
}
738+
739+
func TestSource_AbortScanRateLimitCrossedThreshold(t *testing.T) {
740+
defer gock.Off()
741+
// Mock the API response for getting collection
742+
gock.New("https://api.getpostman.com").
743+
Get("/collections/1234-abc1").
744+
Reply(200).
745+
AddHeader("X-RateLimit-Remaining-Month", "19").
746+
AddHeader("X-RateLimit-Limit-Month", "100").
747+
BodyString(`{"collection":{"info":{"_postman_id":"abc1","name":"test-collection-1","schema":"https://schema.postman.com/json/collection/v2.1.0/collection.json",
748+
"updatedAt":"2025-03-21T17:39:25.000Z","createdAt":"2025-03-21T17:37:13.000Z","lastUpdatedBy":"1234","uid":"1234-abc1"},
749+
"item":[]}}`)
750+
751+
ctx := context.Background()
752+
s, conn := createTestSource(&sourcespb.Postman{
753+
Credential: &sourcespb.Postman_Token{
754+
Token: "super-secret-token",
755+
},
756+
})
757+
758+
err := s.Init(ctx, "test - postman", 0, 1, false, conn, 1)
759+
if err != nil {
760+
t.Fatalf("init error: %v", err)
761+
}
762+
gock.InterceptClient(s.client.HTTPClient)
763+
defer gock.RestoreClient(s.client.HTTPClient)
764+
765+
_, err = s.client.GetCollection(ctx, "1234-abc1")
766+
if !errors.Is(err, errAbortScanDueToAPIRateLimit) {
767+
t.Errorf("expected abort scan error due to rate limit, got: %v", err)
768+
}
769+
}
770+
771+
func TestSource_AbortScanRateLimitBelowThreshold(t *testing.T) {
772+
defer gock.Off()
773+
// Mock the API response for getting collection
774+
gock.New("https://api.getpostman.com").
775+
Get("/collections/1234-abc1").
776+
Reply(200).
777+
AddHeader("X-RateLimit-Remaining-Month", "90").
778+
AddHeader("X-RateLimit-Limit-Month", "100").
779+
BodyString(`{"collection":{"info":{"_postman_id":"abc1","name":"test-collection-1","schema":"https://schema.postman.com/json/collection/v2.1.0/collection.json",
780+
"updatedAt":"2025-03-21T17:39:25.000Z","createdAt":"2025-03-21T17:37:13.000Z","lastUpdatedBy":"1234","uid":"1234-abc1"},
781+
"item":[]}}`)
782+
783+
ctx := context.Background()
784+
s, conn := createTestSource(&sourcespb.Postman{
785+
Credential: &sourcespb.Postman_Token{
786+
Token: "super-secret-token",
787+
},
788+
})
789+
790+
err := s.Init(ctx, "test - postman", 0, 1, false, conn, 1)
791+
if err != nil {
792+
t.Fatalf("init error: %v", err)
793+
}
794+
gock.InterceptClient(s.client.HTTPClient)
795+
defer gock.RestoreClient(s.client.HTTPClient)
796+
797+
_, err = s.client.GetCollection(ctx, "1234-abc1")
798+
if err != nil {
799+
t.Errorf("expected no error, got: %v", err)
800+
}
801+
}

0 commit comments

Comments
 (0)