Skip to content

Commit db3afe5

Browse files
xinhaozxinhaoz
committed
db: datadriven testing for span policy enforcer
1 parent b156909 commit db3afe5

4 files changed

Lines changed: 208 additions & 0 deletions

File tree

‎compaction_test.go‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1507,6 +1507,31 @@ func runCompactionTest(
15071507
s := blobRewriteLog.String()
15081508
return s
15091509

1510+
case "scan-policy-violations":
1511+
// Run the span policy enforcer's scan to detect violations and mark files.
1512+
if d.opts.Experimental.SpanPolicyFunc == nil {
1513+
return "no span policy configured"
1514+
}
1515+
// Wait for table stats to be loaded so that table properties
1516+
// are available for violation detection.
1517+
d.waitTableStats()
1518+
1519+
enforcer := newSpanPolicyEnforcer(d, SpanPolicyEnforcerOptions{})
1520+
enforcer.scanOnce()
1521+
return ""
1522+
1523+
case "pending-policy-enforcement":
1524+
// Show files pending policy enforcement compaction.
1525+
d.mu.Lock()
1526+
count := d.mu.compact.policyEnforcementFiles.Count()
1527+
var buf strings.Builder
1528+
fmt.Fprintf(&buf, "pending: %d\n", count)
1529+
for f, level := range d.mu.compact.policyEnforcementFiles.Ascending() {
1530+
fmt.Fprintf(&buf, " L%d: %s\n", level, f.TableNum)
1531+
}
1532+
d.mu.Unlock()
1533+
return buf.String()
1534+
15101535
case "set-span-policies":
15111536
var spanPolicies []SpanAndPolicy
15121537
for line := range crstrings.LinesSeq(td.Input) {
@@ -1552,6 +1577,11 @@ func runCompactionTest(
15521577
td.Fatalf(t, "parsing minimum-mvcc-garbage-size: %s", err)
15531578
}
15541579
policy.ValueStoragePolicy.MinimumMVCCGarbageSize = int(size)
1580+
case "prefer-fast-compression":
1581+
if len(parts) != 1 {
1582+
td.Fatalf(t, "expected prefer-fast-compression with no value, got: %s", arg)
1583+
}
1584+
policy.PreferFastCompression = true
15551585
default:
15561586
td.Fatalf(t, "unknown span policy arg: %s", arg)
15571587
}
@@ -1660,6 +1690,11 @@ func TestCompaction(t *testing.T) {
16601690
maxVersion: FormatNewest,
16611691
verbose: true,
16621692
},
1693+
"policy_enforcement": {
1694+
minVersion: FormatNewest,
1695+
maxVersion: FormatNewest,
1696+
cmp: DefaultComparer,
1697+
},
16631698
}
16641699
datadriven.Walk(t, "testdata/compaction", func(t *testing.T, path string) {
16651700
filename := filepath.Base(path)

‎data_test.go‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"github.com/cockroachdb/errors"
2626
"github.com/cockroachdb/pebble/internal/base"
2727
"github.com/cockroachdb/pebble/internal/blobtest"
28+
"github.com/cockroachdb/pebble/internal/compression"
2829
"github.com/cockroachdb/pebble/internal/humanize"
2930
"github.com/cockroachdb/pebble/internal/keyspan"
3031
"github.com/cockroachdb/pebble/internal/manifest"
@@ -36,6 +37,7 @@ import (
3637
"github.com/cockroachdb/pebble/objstorage/objstorageprovider"
3738
"github.com/cockroachdb/pebble/objstorage/remote"
3839
"github.com/cockroachdb/pebble/sstable"
40+
"github.com/cockroachdb/pebble/sstable/block"
3941
"github.com/cockroachdb/pebble/sstable/block/blockkind"
4042
"github.com/cockroachdb/pebble/sstable/tablefilters/bloom"
4143
"github.com/cockroachdb/pebble/valsep"
@@ -1899,6 +1901,32 @@ func parseDBOptionsArgs(opts *Options, args []datadriven.CmdArg) error {
18991901
Secondary: wal.Dir{FS: opts.FS, Dirname: cmdArg.Vals[0]},
19001902
}
19011903
opts.WALFailover.EnsureDefaults()
1904+
case "compression":
1905+
var profile block.CompressionProfile
1906+
switch cmdArg.Vals[0] {
1907+
case "zstd":
1908+
profile = *block.ZstdCompression
1909+
case "snappy":
1910+
profile = *block.SnappyCompression
1911+
case "none":
1912+
profile = *block.NoCompression
1913+
case "zstd-force":
1914+
// For testing: Zstd with MinReductionPercent=0 so even small
1915+
// values are stored compressed.
1916+
profile = block.CompressionProfile{
1917+
Name: "test-zstd-force",
1918+
DataBlocks: block.SimpleCompressionSetting(compression.ZstdLevel3),
1919+
ValueBlocks: block.SimpleCompressionSetting(compression.ZstdLevel3),
1920+
OtherBlocks: compression.ZstdLevel3,
1921+
MinReductionPercent: 0,
1922+
}
1923+
default:
1924+
return errors.Newf("unrecognized compression %q", cmdArg.Vals[0])
1925+
}
1926+
for i := range opts.Levels {
1927+
p := profile
1928+
opts.Levels[i].Compression = func() *block.CompressionProfile { return &p }
1929+
}
19021930
}
19031931
}
19041932
if len(spanPolicies) > 0 {

‎testdata/compaction/policy_enforcement‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
# Test span policy enforcer detecting compression violations and triggering
2+
# policy enforcement compactions.
3+
4+
# Create files with Zstd compression using zstd-force (MinReductionPercent=0).
5+
# The span policy requires fast compression for keys >= "m".
6+
# We use long repetitive values to ensure they compress with Zstd.
7+
8+
define compression=zstd-force
9+
L1
10+
a#10,SET:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa b#10,SET:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
11+
L1
12+
m#10,SET:mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmm n#10,SET:nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn
13+
L2
14+
x#5,SET:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx z#5,SET:zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
15+
----
16+
L1:
17+
000004:[a#10,SET-b#10,SET]
18+
000005:[m#10,SET-n#10,SET]
19+
L2:
20+
000006:[x#5,SET-z#5,SET]
21+
22+
# Configure span policy: keys >= "m" require fast compression.
23+
set-span-policies
24+
m,zzz prefer-fast-compression
25+
----
26+
27+
# Initially no pending enforcement files.
28+
pending-policy-enforcement
29+
----
30+
pending: 0
31+
32+
# Run the enforcer scan to detect violations and mark files.
33+
# Files 000005 (m-n) and 000006 (x-z) are in the policy span and use Zstd.
34+
scan-policy-violations
35+
----
36+
37+
# Verify files are now pending enforcement. The enforcer's run loop would pause
38+
# here until these are processed.
39+
pending-policy-enforcement
40+
----
41+
pending: 2
42+
L2: 000006
43+
L1: 000005
44+
45+
# Run compaction. The scheduler should pick up policy enforcement compactions.
46+
auto-compact
47+
----
48+
L1:
49+
000004:[a#10,SET-b#10,SET]
50+
000005:[m#10,SET-n#10,SET]
51+
L2:
52+
000007:[x#0,SET-z#0,SET]
53+
54+
# After auto-compact processes enforcement compactions, pending should be cleared.
55+
# (auto-compact may process one or both files depending on scheduling)
56+
pending-policy-enforcement
57+
----
58+
pending: 0
59+
60+
# Scan again to check for any remaining violations.
61+
scan-policy-violations
62+
----
63+
64+
# One file may still violate policy if not recompacted in first pass.
65+
pending-policy-enforcement
66+
----
67+
pending: 1
68+
L1: 000005
69+
70+
auto-compact
71+
----
72+
L1:
73+
000004:[a#10,SET-b#10,SET]
74+
000008:[m#0,SET-n#0,SET]
75+
L2:
76+
000007:[x#0,SET-z#0,SET]
77+
78+
# All enforcement compactions complete.
79+
pending-policy-enforcement
80+
----
81+
pending: 0
82+
83+
# Test interaction with manual compaction: a file marked for enforcement
84+
# gets moved by a manual compaction. Since the move keeps the same file
85+
# reference, the enforcement mark remains valid.
86+
87+
define compression=zstd-force
88+
L1
89+
a#10,SET:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa b#10,SET:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
90+
L2
91+
m#10,SET:mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmm n#10,SET:nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn
92+
L3
93+
x#5,SET:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx z#5,SET:zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
94+
----
95+
L1:
96+
000004:[a#10,SET-b#10,SET]
97+
L2:
98+
000005:[m#10,SET-n#10,SET]
99+
L3:
100+
000006:[x#5,SET-z#5,SET]
101+
102+
# Set policy and scan to mark files.
103+
set-span-policies
104+
m,zzz prefer-fast-compression
105+
----
106+
107+
scan-policy-violations
108+
----
109+
110+
# Manually compact the L2 file. This moves 000005 to L3 but keeps the same
111+
# file reference, so the enforcement mark remains valid.
112+
compact m-o L2
113+
----
114+
L1:
115+
000004:[a#10,SET-b#10,SET]
116+
L3:
117+
000005:[m#10,SET-n#10,SET]
118+
000006:[x#5,SET-z#5,SET]
119+
120+
# Auto-compact picks up the enforcement compaction for one of the marked files.
121+
# File 000006 is outside the policy span (x-z < m), so only 000005 was marked.
122+
auto-compact
123+
----
124+
L1:
125+
000004:[a#10,SET-b#10,SET]
126+
L3:
127+
000005:[m#10,SET-n#10,SET]
128+
000007:[x#0,SET-z#0,SET]
129+
130+
# Scan and compact again. This should compact file 000005.
131+
scan-policy-violations
132+
----
133+
134+
auto-compact
135+
----
136+
L1:
137+
000004:[a#10,SET-b#10,SET]
138+
L3:
139+
000008:[m#0,SET-n#0,SET]
140+
000007:[x#0,SET-z#0,SET]

‎tool/logs/compaction.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -321,6 +321,7 @@ const (
321321
compactionTypeRewrite
322322
compactionTypeBlobRewrite
323323
compactionTypeVirtualRewrite
324+
compactionTypePolicyEnforcement
324325
)
325326

326327
// String implements fmt.Stringer.
@@ -346,6 +347,8 @@ func (c compactionType) String() string {
346347
return "blob-rewrite"
347348
case compactionTypeVirtualRewrite:
348349
return "virtual-sst-rewrite"
350+
case compactionTypePolicyEnforcement:
351+
return "policy-enforcement"
349352
default:
350353
panic(errors.Newf("unknown compaction type: %s", c))
351354
}
@@ -375,6 +378,8 @@ func parseCompactionType(s string) (t compactionType, err error) {
375378
t = compactionTypeBlobRewrite
376379
case "virtual-sst-rewrite":
377380
t = compactionTypeVirtualRewrite
381+
case "policy-enforcement":
382+
t = compactionTypePolicyEnforcement
378383
default:
379384
err = errors.Newf("unknown compaction type: %s", s)
380385
}

0 commit comments

Comments
 (0)