Skip to content

Commit e7a099e

Browse files
bfarmer67bfarmer67
authored
Integration-test hardening + bootstrap race tolerance + XML docs (#120)
* Fix: OpenSearch dispatcher robustness + R-24c integration test coverage - StatementDispatcher: pass CAS query params (if_seq_no/if_primary_term) via IRequestParameters instead of embedding in path string (OpenSearch.Net rejects paths with query strings); add CasPutParameters class - StatementDispatcher: handle OpenSearchClientException in DispatchUpdateSettingsAsync and DispatchReindexAsync for ThrowExceptions=true clients - SafeDefaultMergeMiddleware: co-inject conflicts:proceed with op_type:create on REINDEX to prevent abort-on-conflict defeating idempotency (ADR-0011) - R-24c gap-fill tests: handle both ThrowExceptions modes in DynamicStrict assertion; make LedgerWrite cleanup defensive against container teardown races - InitializeTestContainers: add HYPERBEE_TESTS_PROVIDERS_ONLY env gate to scope container startup; add HYPERBEE_TESTS_SKIP_SINGLE_NODE support - CouchbaseTestContainer: remove erroneous port 80 mapping * Harden bootstrap and document public OpenSearch provider surface - LedgerIndexInitStep / LockIndexInitStep: tolerate the TOCTOU race between Exists() and Create() by detecting `resource_already_exists_exception` in both client modes (response-based and exception-based). On race-loss the ledger step still verifies the existing mapping; the lock step succeeds. - OpenSearchMigrationOptions: add XML docs to every public type, enum value, property, const, and ctor; lift R-19 / R-29 / PA-2 rationale from line comments into <summary>/<remarks>; add <see cref> links between related options (LockRenewInterval <-> LockStaleAfter, etc.). - OpenSearchExceptions: add XML docs to every exception type and ctor; document RecordId/StatementIndex/FailedStatementIndex properties.
1 parent 4035678 commit e7a099e

9 files changed

Lines changed: 469 additions & 111 deletions

File tree

‎src/Hyperbee.Migrations.Providers.OpenSearch/Internal/Bootstrap/Steps/LedgerIndexInitStep.cs‎

Lines changed: 51 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -84,14 +84,35 @@ public async Task<StepOutcome> ExecuteAsync( BootstrapContext context )
8484

8585
logger.LogInformation( "{step} creating ledger index `{idx}` with strict mapping", Name, indexName );
8686

87-
var createResponse = await context.Client.LowLevel.Indices.CreateAsync<StringResponse>(
88-
indexName,
89-
PostData.String( DefaultMappingJson ),
90-
ctx: context.CancellationToken
91-
).ConfigureAwait( false );
87+
StringResponse createResponse;
88+
try
89+
{
90+
createResponse = await context.Client.LowLevel.Indices.CreateAsync<StringResponse>(
91+
indexName,
92+
PostData.String( DefaultMappingJson ),
93+
ctx: context.CancellationToken
94+
).ConfigureAwait( false );
95+
}
96+
catch ( OpenSearchClientException ex ) when ( IsResourceAlreadyExists( ex.Response ) )
97+
{
98+
// TOCTOU race: another runner created the index between our Exists()
99+
// check and Create(). Verify the mapping and treat as success.
100+
logger.LogDebug( "{step} ledger index `{idx}` created concurrently by another runner; verifying mapping", Name, indexName );
101+
var verifyDetail = await VerifyMappingAsync( context, indexName, logger ).ConfigureAwait( false );
102+
var raceElapsed = context.TimeProvider.GetElapsedTime( start );
103+
return StepOutcome.Succeeded( Name, raceElapsed, $"{verifyDetail} (raced)" );
104+
}
92105

93106
if ( !createResponse.Success )
94107
{
108+
if ( IsResourceAlreadyExists( createResponse ) )
109+
{
110+
logger.LogDebug( "{step} ledger index `{idx}` created concurrently by another runner; verifying mapping", Name, indexName );
111+
var verifyDetail = await VerifyMappingAsync( context, indexName, logger ).ConfigureAwait( false );
112+
var raceElapsed = context.TimeProvider.GetElapsedTime( start );
113+
return StepOutcome.Succeeded( Name, raceElapsed, $"{verifyDetail} (raced)" );
114+
}
115+
95116
var detail = createResponse.OriginalException?.Message ?? createResponse.Body ?? "Unknown create failure";
96117
var ex = new OpenSearchProviderException(
97118
$"{Name} could not create ledger index `{indexName}`. {detail}",
@@ -155,4 +176,29 @@ private static async Task<string> VerifyMappingAsync( BootstrapContext context,
155176
logger.LogDebug( "{step} ledger schema verified ({count} required fields present)", "ledger-init", RequiredFields.Length );
156177
return "verified existing schema";
157178
}
179+
180+
// Detects the OpenSearch-specific 400 body that signals a TOCTOU race
181+
// between Exists() and Create() — another runner won. Inspect the body
182+
// string rather than the status code alone because OS reuses 400 for
183+
// genuine bad-request shapes (malformed mapping, invalid settings).
184+
private static bool IsResourceAlreadyExists( IApiCallDetails? response )
185+
{
186+
if ( response is null || response.HttpStatusCode != 400 )
187+
return false;
188+
189+
var body = response.ResponseBodyInBytes is { Length: > 0 } bytes
190+
? System.Text.Encoding.UTF8.GetString( bytes )
191+
: null;
192+
193+
return body is not null && body.Contains( "resource_already_exists_exception", StringComparison.Ordinal );
194+
}
195+
196+
private static bool IsResourceAlreadyExists( StringResponse response )
197+
{
198+
if ( response.HttpStatusCode != 400 )
199+
return false;
200+
201+
return !string.IsNullOrEmpty( response.Body )
202+
&& response.Body.Contains( "resource_already_exists_exception", StringComparison.Ordinal );
203+
}
158204
}

‎src/Hyperbee.Migrations.Providers.OpenSearch/Internal/Bootstrap/Steps/LockIndexInitStep.cs‎

Lines changed: 49 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -67,14 +67,33 @@ public async Task<StepOutcome> ExecuteAsync( BootstrapContext context )
6767

6868
logger.LogInformation( "{step} creating lock index `{idx}` (replicas=0)", Name, indexName );
6969

70-
var createResponse = await context.Client.LowLevel.Indices.CreateAsync<StringResponse>(
71-
indexName,
72-
PostData.String( DefaultMappingJson ),
73-
ctx: context.CancellationToken
74-
).ConfigureAwait( false );
70+
StringResponse createResponse;
71+
try
72+
{
73+
createResponse = await context.Client.LowLevel.Indices.CreateAsync<StringResponse>(
74+
indexName,
75+
PostData.String( DefaultMappingJson ),
76+
ctx: context.CancellationToken
77+
).ConfigureAwait( false );
78+
}
79+
catch ( OpenSearchClientException ex ) when ( IsResourceAlreadyExists( ex.Response ) )
80+
{
81+
// TOCTOU race: another runner created the lock index between our
82+
// Exists() check and Create(). Treat as success.
83+
logger.LogDebug( "{step} lock index `{idx}` created concurrently by another runner", Name, indexName );
84+
var raceElapsed = context.TimeProvider.GetElapsedTime( start );
85+
return StepOutcome.Succeeded( Name, raceElapsed, "exists (raced)" );
86+
}
7587

7688
if ( !createResponse.Success )
7789
{
90+
if ( IsResourceAlreadyExists( createResponse ) )
91+
{
92+
logger.LogDebug( "{step} lock index `{idx}` created concurrently by another runner", Name, indexName );
93+
var raceElapsed = context.TimeProvider.GetElapsedTime( start );
94+
return StepOutcome.Succeeded( Name, raceElapsed, "exists (raced)" );
95+
}
96+
7897
var detail = createResponse.OriginalException?.Message ?? createResponse.Body ?? "Unknown create failure";
7998
var ex = new OpenSearchProviderException(
8099
$"{Name} could not create lock index `{indexName}`. {detail}",
@@ -97,4 +116,29 @@ public async Task<StepOutcome> ExecuteAsync( BootstrapContext context )
97116
$"{Name} threw an unexpected exception. {ex.Message}", ex ) );
98117
}
99118
}
119+
120+
// Detects the OpenSearch-specific 400 body that signals a TOCTOU race
121+
// between Exists() and Create() — another runner won. Inspect the body
122+
// string rather than the status code alone because OS reuses 400 for
123+
// genuine bad-request shapes (malformed mapping, invalid settings).
124+
private static bool IsResourceAlreadyExists( IApiCallDetails? response )
125+
{
126+
if ( response is null || response.HttpStatusCode != 400 )
127+
return false;
128+
129+
var body = response.ResponseBodyInBytes is { Length: > 0 } bytes
130+
? System.Text.Encoding.UTF8.GetString( bytes )
131+
: null;
132+
133+
return body is not null && body.Contains( "resource_already_exists_exception", StringComparison.Ordinal );
134+
}
135+
136+
private static bool IsResourceAlreadyExists( StringResponse response )
137+
{
138+
if ( response.HttpStatusCode != 400 )
139+
return false;
140+
141+
return !string.IsNullOrEmpty( response.Body )
142+
&& response.Body.Contains( "resource_already_exists_exception", StringComparison.Ordinal );
143+
}
100144
}

0 commit comments

Comments
 (0)