-
Notifications
You must be signed in to change notification settings - Fork 567
Fix parallel forked containers causing IdCompressor "Ranges finalized out of order" #26520
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -370,6 +370,145 @@ describe("Runtime", () => { | |||||
| "All Ids should have been finalized after calling createSummary()", | ||||||
| ); | ||||||
| }); | ||||||
|
|
||||||
| describe("Duplicate IdAllocation range detection", () => { | ||||||
| let containerRuntime: ContainerRuntime; | ||||||
| const remoteClientId = "remoteClient1"; | ||||||
| const forkClientId = "remoteClient2"; | ||||||
| // Use the runtime's own session ID (obtained after creation) for realism, | ||||||
| // but also test with an arbitrary remote session ID. | ||||||
| let localSessionId: string; | ||||||
|
|
||||||
| beforeEach(async () => { | ||||||
| containerRuntime = await ContainerRuntime.loadRuntime2({ | ||||||
| context: getMockContext() as IContainerContext, | ||||||
| registry: new FluidDataStoreRegistry([]), | ||||||
| existing: false, | ||||||
| runtimeOptions: { | ||||||
| enableRuntimeIdCompressor: "on", | ||||||
| }, | ||||||
| provideEntryPoint: mockProvideEntryPoint, | ||||||
| }); | ||||||
| const compressor = containerRuntime.idCompressor; | ||||||
| assert(compressor !== undefined, "IdCompressor should be available"); | ||||||
| localSessionId = compressor.localSessionId; | ||||||
| nextClientSequenceNumber = 1; | ||||||
| }); | ||||||
|
|
||||||
| let nextClientSequenceNumber = 1; | ||||||
|
|
||||||
| function makeIdAllocationOp( | ||||||
| sessionId: string, | ||||||
| firstGenCount: number, | ||||||
| count: number, | ||||||
| clientId: string, | ||||||
| sequenceNumber: number, | ||||||
| ): ISequencedDocumentMessage { | ||||||
| const op: Partial<ISequencedDocumentMessage> = { | ||||||
| contents: JSON.stringify({ | ||||||
| type: ContainerMessageType.IdAllocation, | ||||||
| contents: { | ||||||
| sessionId, | ||||||
| ids: { | ||||||
| firstGenCount, | ||||||
| count, | ||||||
| requestedClusterSize: 128, | ||||||
| localIdRanges: [[firstGenCount, count]], | ||||||
| }, | ||||||
| }, | ||||||
| }), | ||||||
| type: MessageType.Operation, | ||||||
| sequenceNumber, | ||||||
| clientSequenceNumber: nextClientSequenceNumber++, | ||||||
| clientId, | ||||||
| minimumSequenceNumber: 0, | ||||||
| referenceSequenceNumber: 0, | ||||||
| timestamp: Date.now(), | ||||||
| }; | ||||||
| return op as ISequencedDocumentMessage; | ||||||
| } | ||||||
|
|
||||||
| it("Throws DataCorruptionError when overlapping IdAllocation ranges arrive for the same session from different clients", () => { | ||||||
| // First IdAllocation from remoteClient1 — should succeed | ||||||
| containerRuntime.process( | ||||||
| makeIdAllocationOp(localSessionId, 1, 5, remoteClientId, 1), | ||||||
| false /* local */, | ||||||
| ); | ||||||
|
|
||||||
| // Second IdAllocation from forkClient with the SAME session and overlapping range — should throw | ||||||
| assert.throws( | ||||||
| () => | ||||||
| containerRuntime.process( | ||||||
| makeIdAllocationOp(localSessionId, 1, 5, forkClientId, 2), | ||||||
| false /* local */, | ||||||
| ), | ||||||
| (error: IErrorBase) => { | ||||||
| assert(isFluidError(error)); | ||||||
| assert.strictEqual(error.errorType, ContainerErrorTypes.dataCorruptionError); | ||||||
| assert( | ||||||
| error.message.includes("Duplicate IdAllocation range detected"), | ||||||
| `Unexpected error message: ${error.message}`, | ||||||
| ); | ||||||
| return true; | ||||||
| }, | ||||||
| "Processing overlapping IdAllocation ranges for the same session should throw DataCorruptionError", | ||||||
| ); | ||||||
| }); | ||||||
|
|
||||||
| it("Allows sequential (non-overlapping) IdAllocation ranges for the same session", () => { | ||||||
| // First range: genCounts 1-5 | ||||||
| containerRuntime.process( | ||||||
| makeIdAllocationOp(localSessionId, 1, 5, remoteClientId, 1), | ||||||
| false /* local */, | ||||||
| ); | ||||||
|
|
||||||
| // Second range: genCounts 6-10 (sequential, no overlap) | ||||||
| containerRuntime.process( | ||||||
| makeIdAllocationOp(localSessionId, 6, 5, remoteClientId, 2), | ||||||
| false /* local */, | ||||||
| ); | ||||||
| // No error — sequential ranges for the same session are fine (normal reconnection) | ||||||
| }); | ||||||
|
|
||||||
| it("Allows IdAllocation ranges from different sessions", () => { | ||||||
| const otherSessionId = "aaaaaaaa-bbbb-4ccc-dddd-eeeeeeeeeeee"; | ||||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
Haha I'm kidding, it doesn't matter what it is but I thought it was funny the "4" in the middle
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I wonder what on earth Claude was thinking writing that 4 mmm |
||||||
|
|
||||||
| containerRuntime.process( | ||||||
| makeIdAllocationOp(localSessionId, 1, 5, remoteClientId, 1), | ||||||
| false /* local */, | ||||||
| ); | ||||||
|
|
||||||
| // Different session — no conflict | ||||||
| containerRuntime.process( | ||||||
| makeIdAllocationOp(otherSessionId, 1, 5, forkClientId, 2), | ||||||
| false /* local */, | ||||||
| ); | ||||||
| // No error — different sessions can independently allocate the same genCount ranges | ||||||
| }); | ||||||
|
|
||||||
| it("Detects partial overlap where new range starts before the end of the previous one", () => { | ||||||
| // First range: genCounts 1-5 | ||||||
| containerRuntime.process( | ||||||
| makeIdAllocationOp(localSessionId, 1, 5, remoteClientId, 1), | ||||||
| false /* local */, | ||||||
| ); | ||||||
|
|
||||||
| // Overlapping range starting at genCount 3 (before nextExpected=6) | ||||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I wonder if we should make this case an assert - no known way to trigger this case in reality is there?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't think so but given I'm no expert on id compressor I wouldn't assert that. Seems only relevant to internal id compressor logic so I wouldn't add that assert in this PR |
||||||
| assert.throws( | ||||||
| () => | ||||||
| containerRuntime.process( | ||||||
| makeIdAllocationOp(localSessionId, 3, 5, forkClientId, 2), | ||||||
| false /* local */, | ||||||
| ), | ||||||
| (error: IErrorBase) => { | ||||||
| assert(isFluidError(error)); | ||||||
| assert.strictEqual(error.errorType, ContainerErrorTypes.dataCorruptionError); | ||||||
| return true; | ||||||
| }, | ||||||
| "Partially overlapping ranges should be detected as duplicates", | ||||||
| ); | ||||||
| }); | ||||||
| }); | ||||||
| }); | ||||||
|
|
||||||
| describe("Batching", () => { | ||||||
|
|
||||||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
What would it look like to move this duplicate tracking into ID Compressor itself? (Inside finalizeCreationRange)
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I've been playing with it and by doing so it breaks some other id compressor tests. I think this is a better place to target the forked scenario better, given that finalizeCreationRange is used in different places and id compressor already has other mechanisms to detect similar scenarios.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Ok. Yeah I can see how trying to modify the core
finalizeCreationRangefn could be troublesome.I'm always leery of adding a too much niche logic to this file/class. Can we add a new fn to
IIdCompressorCoreliketryFinalizeCreationRangethat returns status about whether it succeeded (rather than throwing), and includes this check?That type is legacy-beta so you'd have to add it as optional but I know Taylor wants to move it to internal anyway so we can consider it pretty flexible.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@taylorsw04 can you please review this new logic? Any thoughts on moving it into the ID Compressor impl rather than adding it here?