fix(scim): replace the 1000-group hard cap with real pagination + a safety valve

The prior fix (a hard LIMIT/queryEqLimit cap at 1000) traded unbounded scan
cost for silent incorrectness: an org past the cap could get a false
"not found" on GetScimGroupByOrgAndDisplayName / GetScimGroupByOrgAndExternalID,
which breaks CreateGroup's dedup (creates a duplicate), the rename-uniqueness
gate (lets a duplicate name through), and the displayName filter (404s a
group that exists). Not hypothetical - large enterprises can plausibly sync
1000+ fine-grained IdP groups into one org.

Cassandra: drop the CQL LIMIT entirely (it truncates the combined result set
across every page, reintroducing the same silent-drop bug server-side instead
of client-side). gocql's Scanner already pages through the full ALLOW
FILTERING result set lazily as Next() is called, so a normal lookup runs to
full exhaustion or an early match. groupScanSafetyCap (100k) now bounds the
Go-side examined-item counter instead, purely as a circuit-breaker against a
pathologically large partition, not a realistic ceiling.

DynamoDB: new queryEqUntil() helper pages through the org_id GSI (like the
existing unbounded queryEq) but stops as soon as the caller's match predicate
reports a hit, instead of materializing every page before filtering, and
carries the same 100k safety valve. Both GetScimGroupByOrgAndDisplayName and
GetScimGroupByOrgAndExternalID (both previously capped at groupScanCap=1000)
now use it. The narrower queryEqLimit helper is untouched - it's still
correct for its many other exact-match, limit=1 callers.

Verified against real ScyllaDB and DynamoDB (make test-scylladb,
make test-dynamodb), not just SQLite: 31 packages, 0 failures, each.

Author: lakhansamani

internal/storage/db/cassandradb/scim_group.go

@@ -14,6 +14,15 @@ import (
 
 const scimGroupColumns = "id, org_id, display_name, external_id, created_at, updated_at"
 
+// groupScanSafetyCap bounds the org-scoped scan-and-compare-in-app
+// displayName lookup below. It is NOT a realistic ceiling on an org's group
+// count — gocql's Scanner already pages through the full ALLOW FILTERING
+// result set as Next() is called, so a normal lookup runs to full exhaustion
+// (or an early match) regardless of how many pages that takes. This exists
+// purely as a circuit-breaker against unbounded work on a pathologically
+// large partition, mirroring DynamoDB's groupScanSafetyCap.
+const groupScanSafetyCap = 100_000
+
 // scanScimGroup maps the scimGroupColumns projection onto a struct.
 func scanScimGroup(scan func(...interface{}) error, group *schemas.ScimGroup) error {
 	return scan(&group.ID, &group.OrgID, &group.DisplayName, &group.ExternalID, &group.CreatedAt, &group.UpdatedAt)
@@ -91,8 +100,15 @@ func (p *provider) GetScimGroupByID(ctx context.Context, id string) (*schemas.Sc
 // query to the org and compare displayName with strings.EqualFold in Go (an
 // org's group set is small), mirroring the DynamoDB fetch-then-filter shape.
 func (p *provider) GetScimGroupByOrgAndDisplayName(ctx context.Context, orgID, displayName string) (*schemas.ScimGroup, error) {
+	// No LIMIT here deliberately: a CQL LIMIT truncates the combined result set
+	// across every page, which would silently drop a match beyond it — the
+	// exact bug groupScanSafetyCap exists to avoid. gocql's Scanner pages
+	// through ALLOW FILTERING lazily as Next() is called, so leaving LIMIT off
+	// lets a real match anywhere in the org's group set still be found; the
+	// safety cap below bounds the Go-side loop instead, not the CQL query.
 	query := fmt.Sprintf("SELECT %s FROM %s WHERE org_id = ? ALLOW FILTERING", scimGroupColumns, KeySpace+"."+schemas.Collections.ScimGroup)
 	scanner := p.db.Query(query, orgID).Consistency(gocql.One).Iter().Scanner()
+	examined := 0
 	for scanner.Next() {
 		var group schemas.ScimGroup
 		if err := scanScimGroup(scanner.Scan, &group); err != nil {
@@ -101,6 +117,12 @@ func (p *provider) GetScimGroupByOrgAndDisplayName(ctx context.Context, orgID, d
 		if strings.EqualFold(group.DisplayName, displayName) {
 			return &group, nil
 		}
+		examined++
+		if examined >= groupScanSafetyCap {
+			p.dependencies.Log.Warn().Str("org_id", orgID).Int("examined", examined).
+				Msg("GetScimGroupByOrgAndDisplayName: hit the scan safety cap without a match")
+			return nil, gocql.ErrNotFound
+		}
 	}
 	if err := scanner.Err(); err != nil {
 		return nil, err

internal/storage/db/dynamodb/ops.go

@@ -211,6 +211,59 @@ func (p *provider) queryEqLimit(ctx context.Context, table, indexName, pkAttr, p
 	return res.Items, nil
 }
 
+// queryEqUntil pages through the pkAttr=pkVal partition (like queryEq) but
+// stops as soon as match reports a hit, instead of materializing every page
+// before filtering. maxItems is a defensive circuit-breaker against unbounded
+// scan cost on a pathologically large partition — NOT a realistic ceiling;
+// normal callers exhaust the partition long before reaching it. match may
+// return an error (e.g. a decode failure), which aborts the scan immediately
+// rather than being swallowed as a non-match; on a hit it is responsible for
+// capturing whatever it needs from the item (queryEqUntil only reports found/
+// not-found, it does not return the raw item — callers already decode it
+// inside match to test it, so returning it again would be redundant).
+func (p *provider) queryEqUntil(ctx context.Context, table, indexName, pkAttr, pkVal string, maxItems int, match func(map[string]types.AttributeValue) (bool, error)) (bool, error) {
+	kc := expression.Key(pkAttr).Equal(expression.Value(pkVal))
+	expr, err := expression.NewBuilder().WithKeyCondition(kc).Build()
+	if err != nil {
+		return false, err
+	}
+	var start map[string]types.AttributeValue
+	examined := 0
+	for {
+		in := &dynamodb.QueryInput{
+			TableName:                 aws.String(table),
+			IndexName:                 aws.String(indexName),
+			KeyConditionExpression:    expr.KeyCondition(),
+			ExpressionAttributeNames:  expr.Names(),
+			ExpressionAttributeValues: expr.Values(),
+			ExclusiveStartKey:         start,
+		}
+		res, err := p.client.Query(ctx, in)
+		if err != nil {
+			return false, err
+		}
+		for _, it := range res.Items {
+			ok, err := match(it)
+			if err != nil {
+				return false, err
+			}
+			if ok {
+				return true, nil
+			}
+			examined++
+			if examined >= maxItems {
+				p.dependencies.Log.Warn().Str("table", table).Str("partition_key", pkVal).Int("examined", examined).
+					Msg("queryEqUntil: hit the scan safety cap without a match")
+				return false, nil
+			}
+		}
+		if res.LastEvaluatedKey == nil {
+			return false, nil
+		}
+		start = res.LastEvaluatedKey
+	}
+}
+
 func (p *provider) scanFilteredLimit(ctx context.Context, table string, index *string, filter *expression.ConditionBuilder, limit int32) ([]map[string]types.AttributeValue, error) {
 	in := &dynamodb.ScanInput{
 		TableName: aws.String(table),

internal/storage/db/dynamodb/scim_group.go

@@ -7,16 +7,17 @@ import (
 	"strings"
 	"time"
 
+	"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
 	"github.com/google/uuid"
 
 	"github.com/authorizerdev/authorizer/internal/storage/schemas"
 )
 
-// groupScanCap bounds the org_id query used to resolve a group by displayName.
-// An org's group count is small; this closes over the realistic range without
-// paginating.
-// ponytail: cap at 1000; add pagination if an org ever exceeds it.
-const groupScanCap = 1000
+// groupScanSafetyCap bounds the org-scoped scan-and-compare-in-app lookups
+// below (queryEqUntil). It is NOT a realistic ceiling on an org's group
+// count — normal lookups page through the whole partition — it exists purely
+// as a circuit-breaker against unbounded work on a pathologically large one.
+const groupScanSafetyCap = 100_000
 
 // AddScimGroup creates a new SCIM group record.
 func (p *provider) AddScimGroup(ctx context.Context, group *schemas.ScimGroup) (*schemas.ScimGroup, error) {
@@ -76,20 +77,25 @@ func (p *provider) GetScimGroupByID(ctx context.Context, id string) (*schemas.Sc
 // caseExact:false (RFC 7644 §3.4.2.2), and a DynamoDB GSI lookup is exact-match
 // only, so the case-fold happens here in Go with strings.EqualFold.
 func (p *provider) GetScimGroupByOrgAndDisplayName(ctx context.Context, orgID, displayName string) (*schemas.ScimGroup, error) {
-	items, err := p.queryEqLimit(ctx, schemas.Collections.ScimGroup, "org_id", "org_id", orgID, nil, groupScanCap)
-	if err != nil {
-		return nil, err
-	}
-	for _, it := range items {
+	var found schemas.ScimGroup
+	ok, err := p.queryEqUntil(ctx, schemas.Collections.ScimGroup, "org_id", "org_id", orgID, groupScanSafetyCap, func(it map[string]types.AttributeValue) (bool, error) {
 		var group schemas.ScimGroup
 		if err := unmarshalItem(it, &group); err != nil {
-			return nil, err
+			return false, err
 		}
-		if strings.EqualFold(group.DisplayName, displayName) {
-			return &group, nil
+		if !strings.EqualFold(group.DisplayName, displayName) {
+			return false, nil
 		}
+		found = group
+		return true, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	if !ok {
+		return nil, errors.New("no document found")
 	}
-	return nil, errors.New("no document found")
+	return &found, nil
 }
 
 // GetScimGroupByOrgAndExternalID resolves the single group with the given
@@ -98,18 +104,23 @@ func (p *provider) GetScimGroupByOrgAndDisplayName(ctx context.Context, orgID, d
 // and match in-app (an org's group set is small).
 func (p *provider) GetScimGroupByOrgAndExternalID(ctx context.Context, orgID, externalID string) (*schemas.ScimGroup, error) {
 	want := orgID + ":" + externalID
-	items, err := p.queryEqLimit(ctx, schemas.Collections.ScimGroup, "org_id", "org_id", orgID, nil, groupScanCap)
-	if err != nil {
-		return nil, err
-	}
-	for _, it := range items {
+	var found schemas.ScimGroup
+	ok, err := p.queryEqUntil(ctx, schemas.Collections.ScimGroup, "org_id", "org_id", orgID, groupScanSafetyCap, func(it map[string]types.AttributeValue) (bool, error) {
 		var group schemas.ScimGroup
 		if err := unmarshalItem(it, &group); err != nil {
-			return nil, err
+			return false, err
 		}
-		if group.ExternalID != nil && *group.ExternalID == want {
-			return &group, nil
+		if group.ExternalID == nil || *group.ExternalID != want {
+			return false, nil
 		}
+		found = group
+		return true, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	if !ok {
+		return nil, errors.New("no document found")
 	}
-	return nil, errors.New("no document found")
+	return &found, nil
 }