Skip to content

Commit 2ccc88a

Browse files
committed
address review comments
Signed-off-by: Mikhail Koviazin <github@mkmk.aleeas.com>
1 parent 29e6223 commit 2ccc88a

2 files changed

Lines changed: 87 additions & 10 deletions

File tree

tests/test_asyncio/test_cluster.py

Lines changed: 68 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -113,15 +113,34 @@ async def pipe(
113113

114114

115115
class BlockingDisconnectConnection:
116-
def __init__(self, event: asyncio.Event) -> None:
116+
def __init__(
117+
self,
118+
event: asyncio.Event,
119+
started_event: Optional[asyncio.Event] = None,
120+
) -> None:
117121
self._event = event
122+
self._started_event = started_event
118123
self.is_connected = True
119124

120125
async def disconnect(self) -> None:
126+
if self._started_event is not None:
127+
self._started_event.set()
121128
await self._event.wait()
122129
self.is_connected = False
123130

124131

132+
class FlakyDisconnectConnection:
133+
def __init__(self) -> None:
134+
self.is_connected = True
135+
self.calls = 0
136+
137+
async def disconnect(self) -> None:
138+
self.calls += 1
139+
if self.calls == 1:
140+
raise RuntimeError("disconnect failed")
141+
self.is_connected = False
142+
143+
125144
@pytest_asyncio.fixture()
126145
async def slowlog(r: ValkeyCluster) -> None:
127146
"""
@@ -2470,10 +2489,13 @@ async def test_init_slots_cache(self) -> None:
24702489

24712490
async def test_initialize_waits_for_removed_node_disconnect(self) -> None:
24722491
disconnect_event = asyncio.Event()
2492+
disconnect_started_event = asyncio.Event()
24732493
old_node = ClusterNode("127.0.0.1", 7009, PRIMARY)
2474-
old_node._connections.append(BlockingDisconnectConnection(disconnect_event))
2494+
old_node._connections.append(
2495+
BlockingDisconnectConnection(disconnect_event, disconnect_started_event)
2496+
)
24752497
startup_node = ClusterNode(default_host, default_port)
2476-
manager = NodesManager([startup_node], False, {})
2498+
manager = NodesManager([startup_node], False, {}, dynamic_startup_nodes=False)
24772499
manager.nodes_cache = {old_node.name: old_node}
24782500

24792501
captured_contexts = []
@@ -2495,7 +2517,7 @@ async def mocked_execute_command(self, *args, **kwargs):
24952517
side_effect=mocked_execute_command,
24962518
):
24972519
initialize_task = asyncio.create_task(manager.initialize())
2498-
await asyncio.sleep(0)
2520+
await disconnect_started_event.wait()
24992521

25002522
assert not initialize_task.done()
25012523

@@ -2517,6 +2539,48 @@ async def mocked_execute_command(self, *args, **kwargs):
25172539
disconnect_event.set()
25182540
loop.set_exception_handler(previous_handler)
25192541

2542+
async def test_initialize_warns_and_retries_stale_disconnect_errors(self) -> None:
2543+
flaky_connection = FlakyDisconnectConnection()
2544+
old_node = ClusterNode("127.0.0.1", 7009, PRIMARY)
2545+
old_node._connections.append(flaky_connection)
2546+
startup_node = ClusterNode(default_host, default_port)
2547+
manager = NodesManager([startup_node], False, {}, dynamic_startup_nodes=False)
2548+
manager.nodes_cache = {old_node.name: old_node}
2549+
2550+
async def mocked_execute_command(self, *args, **kwargs):
2551+
assert args[0] == "CLUSTER SLOTS"
2552+
return [[0, 16383, [default_host, default_port, "node_0"]]]
2553+
2554+
with mock.patch.object(
2555+
ClusterNode,
2556+
"execute_command",
2557+
autospec=True,
2558+
side_effect=mocked_execute_command,
2559+
):
2560+
with pytest.warns(
2561+
RuntimeWarning, match="disconnecting stale cluster nodes"
2562+
):
2563+
await manager.initialize()
2564+
2565+
assert manager._pending_node_disconnects == {old_node.name: old_node}
2566+
assert flaky_connection.calls == 1
2567+
2568+
await manager.aclose()
2569+
2570+
assert manager._pending_node_disconnects == {}
2571+
assert flaky_connection.calls == 2
2572+
2573+
with warnings.catch_warnings(record=True) as caught:
2574+
warnings.simplefilter("always", ResourceWarning)
2575+
old_node.__del__()
2576+
2577+
resource_warnings = [
2578+
warning
2579+
for warning in caught
2580+
if issubclass(warning.category, ResourceWarning)
2581+
]
2582+
assert resource_warnings == []
2583+
25202584
async def test_init_slots_cache_cluster_mode_disabled(self) -> None:
25212585
"""
25222586
Test that creating a ValkeyCluster fails if one of the startup nodes

valkey/asyncio/cluster.py

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1233,14 +1233,27 @@ async def _close_pending_node_disconnects(self) -> None:
12331233
if not self._pending_node_disconnects:
12341234
return
12351235

1236-
nodes = tuple(self._pending_node_disconnects.values())
1237-
self._pending_node_disconnects.clear()
1236+
pending_items = tuple(self._pending_node_disconnects.items())
12381237
ret = await asyncio.gather(
1239-
*(node.disconnect() for node in nodes), return_exceptions=True
1238+
*(node.disconnect() for _, node in pending_items), return_exceptions=True
12401239
)
1241-
exc = next((res for res in ret if isinstance(res, Exception)), None)
1242-
if exc:
1243-
raise exc
1240+
errors = []
1241+
for (name, node), res in zip(pending_items, ret):
1242+
if self._pending_node_disconnects.get(name) is not node:
1243+
continue
1244+
if isinstance(res, Exception):
1245+
errors.append(res)
1246+
continue
1247+
self._pending_node_disconnects.pop(name, None)
1248+
1249+
if errors:
1250+
msg = ", ".join(f"{type(err).__name__}: {err}" for err in errors)
1251+
warnings.warn(
1252+
"Error(s) while disconnecting stale cluster nodes during "
1253+
f"topology refresh: {msg}",
1254+
RuntimeWarning,
1255+
stacklevel=2,
1256+
)
12441257

12451258
def _update_moved_slots(self) -> None:
12461259
e = self._moved_exception

0 commit comments

Comments
 (0)