Hello,
I’m having some issues with my flow running on Prefect 2.10.17.
My flow consists of a generator that yields chunks extracted from a database (this is not a prefect task). Then, inside of a for loop iterating over that generator, there is a prefect task sending the chunk of data to an API.
I’m using the ConcurrentTaskRunner.
The flow runs run well for approx. 4000 tasks (or for 1h30min), but then it crashes with the following error message below.
Could you please help me with that ?
Thanks,
Pierre
Crash details:
Traceback (most recent call last):
File "/usr/local/lib/python3.10/site-packages/httpcore/_async/http11.py", line 99, in handle_async_request
) = await self._receive_response_headers(**kwargs)
File "/usr/local/lib/python3.10/site-packages/httpcore/_async/http11.py", line 163, in _receive_response_headers
event = await self._receive_event(timeout=timeout)
File "/usr/local/lib/python3.10/site-packages/httpcore/_async/http11.py", line 199, in _receive_event
data = await self._network_stream.read(
File "/usr/local/lib/python3.10/site-packages/httpcore/backends/asyncio.py", line 34, in read
return await self._stream.receive(max_bytes=max_bytes)
File "/usr/local/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 1203, in receive
await self._protocol.read_event.wait()
File "/usr/local/lib/python3.10/asyncio/locks.py", line 214, in wait
await fut
asyncio.exceptions.CancelledError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.10/site-packages/httpcore/_async/connection_pool.py", line 245, in handle_async_request
response = await connection.handle_async_request(request)
File "/usr/local/lib/python3.10/site-packages/httpcore/_async/connection.py", line 96, in handle_async_request
return await self._connection.handle_async_request(request)
File "/usr/local/lib/python3.10/site-packages/httpcore/_async/http11.py", line 119, in handle_async_request
await self._response_closed()
asyncio.exceptions.CancelledError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.10/site-packages/prefect/engine.py", line 1940, in report_task_run_crashes
yield
File "/usr/local/lib/python3.10/site-packages/prefect/engine.py", line 1532, in begin_task_run
state = await orchestrate_task_run(
File "/usr/local/lib/python3.10/site-packages/prefect/engine.py", line 1760, in orchestrate_task_run
state = await propose_state(client, terminal_state, task_run_id=task_run.id)
File "/usr/local/lib/python3.10/site-packages/prefect/engine.py", line 2145, in propose_state
response = await set_state_and_handle_waits(set_state)
File "/usr/local/lib/python3.10/site-packages/prefect/engine.py", line 2132, in set_state_and_handle_waits
response = await set_state_func()
File "/usr/local/lib/python3.10/site-packages/prefect/client/orchestration.py", line 1965, in set_task_run_state
response = await self._client.post(
File "/usr/local/lib/python3.10/site-packages/httpx/_client.py", line 1848, in post
return await self.request(
File "/usr/local/lib/python3.10/site-packages/httpx/_client.py", line 1530, in request
return await self.send(request, auth=auth, follow_redirects=follow_redirects)
File "/usr/local/lib/python3.10/site-packages/prefect/client/base.py", line 251, in send
response = await self._send_with_retry(
File "/usr/local/lib/python3.10/site-packages/prefect/client/base.py", line 193, in _send_with_retry
response = await request()
File "/usr/local/lib/python3.10/site-packages/httpx/_client.py", line 1617, in send
response = await self._send_handling_auth(
File "/usr/local/lib/python3.10/site-packages/httpx/_client.py", line 1645, in _send_handling_auth
response = await self._send_handling_redirects(
File "/usr/local/lib/python3.10/site-packages/httpx/_client.py", line 1682, in _send_handling_redirects
response = await self._send_single_request(request)
File "/usr/local/lib/python3.10/site-packages/httpx/_client.py", line 1719, in _send_single_request
response = await transport.handle_async_request(request)
File "/usr/local/lib/python3.10/site-packages/httpx/_transports/default.py", line 353, in handle_async_request
resp = await self._pool.handle_async_request(req)
File "/usr/local/lib/python3.10/site-packages/httpcore/_async/connection_pool.py", line 260, in handle_async_request
await self.response_closed(status)
asyncio.exceptions.CancelledError
