Skip to content

Commit ee220f4

Browse files
authored
Fix/concurrent summarization loop
1 parent ceec961 commit ee220f4

5 files changed

Lines changed: 150 additions & 19 deletions

File tree

echo/server/dembrane/api/webhooks.py

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -73,15 +73,9 @@ async def assemblyai_webhook_callback(
7373

7474
if normalized_status == "error":
7575
from dembrane.tasks import _on_chunk_transcription_done
76-
from dembrane.transcribe import _save_chunk_error, fetch_assemblyai_result
76+
from dembrane.transcribe import _save_chunk_error
7777

78-
error_detail = f"AssemblyAI error for transcript {payload.transcript_id}"
79-
try:
80-
fetch_assemblyai_result(payload.transcript_id)
81-
except Exception as fetch_exc:
82-
error_detail = str(fetch_exc)
83-
84-
_save_chunk_error(chunk_id, error_detail)
78+
_save_chunk_error(chunk_id, f"AssemblyAI error for transcript {payload.transcript_id}")
8579
_on_chunk_transcription_done(conversation_id, chunk_id, logger)
8680
delete_assemblyai_webhook_metadata(payload.transcript_id)
8781
return {"status": "error_handled"}

echo/server/dembrane/async_helpers.py

Lines changed: 22 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -172,17 +172,31 @@ def _get_thread_event_loop() -> asyncio.AbstractEventLoop:
172172

173173
def run_async_in_new_loop(coro: Coroutine[Any, Any, T]) -> T:
174174
"""
175-
Execute an async coroutine on this thread's persistent event loop.
175+
Execute an async coroutine in a fresh, isolated event loop.
176176
177177
Use from synchronous contexts such as Dramatiq actors or CLI scripts to
178-
invoke async FastAPI handlers without hitting "Future attached to a
179-
different loop" errors.
178+
invoke async FastAPI handlers.
179+
180+
A fresh loop is created per call rather than reusing a cached thread loop.
181+
This prevents "Future attached to a different loop" errors when multiple
182+
concurrent Dramatiq greenlets (dramatiq-gevent uses one OS thread with many
183+
greenlets) share the same thread ID and would otherwise share the same loop.
184+
The coroutines invoked here (summarize_conversation, get_conversation_content)
185+
use only stateless async operations so fresh loops per call is safe.
180186
"""
181187
if not asyncio.iscoroutine(coro) and not asyncio.isfuture(coro):
182188
raise TypeError("run_async_in_new_loop expects a coroutine or Future.")
183189

184-
loop = _get_thread_event_loop()
185-
logger.debug("Running async coroutine in thread loop: %s", coro)
186-
result = loop.run_until_complete(coro)
187-
logger.debug("Completed async coroutine: %s", coro)
188-
return result
190+
import nest_asyncio
191+
192+
loop = asyncio.new_event_loop()
193+
# Apply nest_asyncio in case dramatiq-gevent has patched asyncio's running
194+
# loop detection on this thread.
195+
nest_asyncio.apply(loop)
196+
logger.debug("Running async coroutine in fresh event loop: %s", coro)
197+
try:
198+
result = loop.run_until_complete(coro)
199+
logger.debug("Completed async coroutine: %s", coro)
200+
return result
201+
finally:
202+
loop.close()

echo/server/dembrane/transcribe.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ def transcribe_audio_assemblyai(
107107

108108
data: dict[str, Any] = {
109109
"audio_url": audio_file_uri,
110-
"speech_models": ["universal-3-pro", "universal-2"],
110+
"speech_models": ["universal-3-pro"],
111111
"language_detection": True,
112112
"language_detection_options": {
113113
"expected_languages": list(set(get_allowed_languages()) | {"pt"}),
Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
"""
2+
Tests for run_async_in_new_loop — specifically the concurrent-greenlet scenario
3+
that caused "Future attached to a different loop" errors under load.
4+
5+
Regression test for: multiple concurrent callers sharing the same OS thread
6+
(as dramatiq-gevent greenlets do) must not share an event loop.
7+
"""
8+
9+
import asyncio
10+
import threading
11+
from concurrent.futures import ThreadPoolExecutor, as_completed
12+
13+
import pytest
14+
15+
from dembrane.async_helpers import run_async_in_new_loop
16+
17+
18+
async def _simple_coro(value: int) -> int:
19+
"""Minimal async coroutine that does a thread-pool round-trip (like run_in_thread_pool)."""
20+
loop = asyncio.get_running_loop()
21+
# Simulate run_in_thread_pool: submit blocking work to the executor
22+
result = await loop.run_in_executor(None, lambda: value * 2)
23+
return result
24+
25+
26+
async def _gather_coro(value: int) -> int:
27+
"""Uses asyncio.gather internally — matches what summarize_conversation does."""
28+
loop = asyncio.get_running_loop()
29+
a, b = await asyncio.gather(
30+
loop.run_in_executor(None, lambda: value + 1),
31+
loop.run_in_executor(None, lambda: value + 2),
32+
)
33+
return a + b
34+
35+
36+
def test_run_async_in_new_loop_basic():
37+
"""Single call works correctly."""
38+
result = run_async_in_new_loop(_simple_coro(5))
39+
assert result == 10
40+
41+
42+
def test_run_async_in_new_loop_with_gather():
43+
"""Gather inside coroutine works correctly."""
44+
result = run_async_in_new_loop(_gather_coro(3))
45+
assert result == 9 # (3+1) + (3+2) = 9
46+
47+
48+
def test_run_async_in_new_loop_concurrent_threads():
49+
"""
50+
Simulates the dramatiq-gevent scenario: N threads all calling
51+
run_async_in_new_loop concurrently. Before the fix, they shared
52+
a cached loop by thread ID, causing "Future attached to a different
53+
loop" errors under concurrent load.
54+
"""
55+
errors = []
56+
results = []
57+
58+
def worker(value: int):
59+
try:
60+
r = run_async_in_new_loop(_gather_coro(value))
61+
results.append(r)
62+
except Exception as e:
63+
errors.append(str(e))
64+
65+
# Simulate 10 concurrent callers (matches or exceeds Stage 3 load test concurrency)
66+
with ThreadPoolExecutor(max_workers=10) as pool:
67+
futures = [pool.submit(worker, i) for i in range(10)]
68+
for f in as_completed(futures):
69+
f.result() # re-raises if the thread itself crashed
70+
71+
assert errors == [], f"Concurrent run_async_in_new_loop raised errors: {errors}"
72+
assert len(results) == 10
73+
74+
75+
def test_run_async_in_new_loop_same_thread_sequential():
76+
"""
77+
Calls from the same thread are safe when sequential.
78+
Verifies loop is properly closed between calls (no 'loop is closed' error).
79+
"""
80+
for i in range(5):
81+
result = run_async_in_new_loop(_simple_coro(i))
82+
assert result == i * 2
83+
84+
85+
def test_run_async_in_new_loop_same_thread_id_concurrent():
86+
"""
87+
Reproduces the exact bug: multiple coroutines submitted from threads
88+
that all share the same thread ID (simulated by patching get_ident).
89+
90+
Before the fix (persistent loop per thread ID), all concurrent callers
91+
shared one loop → "Future attached to a different loop".
92+
After the fix (fresh loop per call), each call is isolated.
93+
"""
94+
original_get_ident = threading.get_ident
95+
# Make all threads report the same thread ID — exactly what gevent does
96+
threading.get_ident = lambda: 99999
97+
98+
errors = []
99+
results = []
100+
101+
def worker(value: int):
102+
try:
103+
r = run_async_in_new_loop(_gather_coro(value))
104+
results.append(r)
105+
except Exception as e:
106+
errors.append(str(e))
107+
108+
try:
109+
with ThreadPoolExecutor(max_workers=5) as pool:
110+
futures = [pool.submit(worker, i) for i in range(5)]
111+
for f in as_completed(futures):
112+
f.result()
113+
finally:
114+
threading.get_ident = original_get_ident
115+
116+
assert errors == [], f"Same-thread-ID concurrent calls raised errors: {errors}"
117+
assert len(results) == 5
118+
119+
120+
def test_run_async_in_new_loop_rejects_non_coroutine():
121+
"""Type guard still works."""
122+
with pytest.raises(TypeError, match="expects a coroutine or Future"):
123+
run_async_in_new_loop(42) # type: ignore

echo/server/tests/test_transcribe_webhook.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ def _fake_post(url: str, **kwargs: Any) -> _FakeResponse:
5151
assert transcript is None
5252
assert payload == {"transcript_id": "tx-1"}
5353
assert captured["url"].endswith("/v2/transcript")
54-
assert captured["json"]["speech_models"] == ["universal-3-pro", "universal-2"]
54+
assert captured["json"]["speech_models"] == ["universal-3-pro"]
5555
assert "speech_model" not in captured["json"]
5656
assert "prompt" not in captured["json"]
5757
assert captured["json"]["keyterms_prompt"] == ["Dembrane"]
@@ -92,7 +92,7 @@ def _fake_get(_url: str, **_kwargs: Any) -> _FakeResponse:
9292
assert response["status"] == "completed"
9393
assert payloads["polls"] == 2
9494
post_payload = payloads["posts"][0]
95-
assert post_payload["speech_models"] == ["universal-3-pro", "universal-2"]
95+
assert post_payload["speech_models"] == ["universal-3-pro"]
9696
assert "speech_model" not in post_payload
9797
assert "webhook_url" not in post_payload
9898

0 commit comments

Comments
 (0)