I tested this using S3 as default storage and got same error. FYO the full (very long) traceback is below.
09:57:31.115 | INFO | Task run 'test1-e6c6c25b-0' - Crash detected! Request to http://host.docker.internal:4200/api/blocks/get_default_storage_block timed out.
09:58:01.144 | ERROR | Flow run 'aboriginal-seahorse' - Encountered exception during execution:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/httpcore/backends/asyncio.py", line 104, in connect_tcp
local_host=local_address,
File "/usr/local/lib/python3.7/site-packages/anyio/_core/_sockets.py", line 180, in connect_tcp
await event.wait()
File "/usr/local/lib/python3.7/site-packages/anyio/_backends/_asyncio.py", line 574, in __aexit__
raise exceptions[0]
File "/usr/local/lib/python3.7/site-packages/anyio/_backends/_asyncio.py", line 611, in _run_wrapped_task
await coro
File "/usr/local/lib/python3.7/site-packages/anyio/_core/_sockets.py", line 127, in try_connect
stream = await asynclib.connect_tcp(remote_host, remote_port, local_address)
File "/usr/local/lib/python3.7/site-packages/anyio/_backends/_asyncio.py", line 1519, in connect_tcp
local_addr=local_addr)
File "/usr/local/lib/python3.7/asyncio/base_events.py", line 949, in create_connection
await self.sock_connect(sock, address)
File "/usr/local/lib/python3.7/asyncio/selector_events.py", line 473, in sock_connect
return await fut
concurrent.futures._base.CancelledError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/httpcore/_exceptions.py", line 8, in map_exceptions
yield
File "/usr/local/lib/python3.7/site-packages/httpcore/backends/asyncio.py", line 104, in connect_tcp
local_host=local_address,
File "/usr/local/lib/python3.7/site-packages/anyio/_core/_tasks.py", line 103, in __exit__
raise TimeoutError
TimeoutError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/httpx/_transports/default.py", line 60, in map_httpcore_exceptions
yield
File "/usr/local/lib/python3.7/site-packages/httpx/_transports/default.py", line 353, in handle_async_request
resp = await self._pool.handle_async_request(req)
File "/usr/local/lib/python3.7/site-packages/httpcore/_async/connection_pool.py", line 253, in handle_async_request
raise exc
File "/usr/local/lib/python3.7/site-packages/httpcore/_async/connection_pool.py", line 237, in handle_async_request
response = await connection.handle_async_request(request)
File "/usr/local/lib/python3.7/site-packages/httpcore/_async/connection.py", line 86, in handle_async_request
raise exc
File "/usr/local/lib/python3.7/site-packages/httpcore/_async/connection.py", line 63, in handle_async_request
stream = await self._connect(request)
File "/usr/local/lib/python3.7/site-packages/httpcore/_async/connection.py", line 111, in _connect
stream = await self._network_backend.connect_tcp(**kwargs)
File "/usr/local/lib/python3.7/site-packages/httpcore/backends/auto.py", line 24, in connect_tcp
host, port, timeout=timeout, local_address=local_address
File "/usr/local/lib/python3.7/site-packages/httpcore/backends/asyncio.py", line 104, in connect_tcp
local_host=local_address,
File "/usr/local/lib/python3.7/contextlib.py", line 130, in __exit__
self.gen.throw(type, value, traceback)
File "/usr/local/lib/python3.7/site-packages/httpcore/_exceptions.py", line 12, in map_exceptions
raise to_exc(exc)
httpcore.ConnectTimeout
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/prefect/engine.py", line 459, in orchestrate_flow_run
flow_run_context.task_run_futures, client=client
File "/usr/local/lib/python3.7/site-packages/prefect/engine.py", line 820, in wait_for_task_runs_and_report_crashes
task_run_id=future.task_run.id, state=state, force=True
File "/usr/local/lib/python3.7/site-packages/prefect/client.py", line 1581, in set_task_run_state
json=dict(state=state_data_json, force=force),
File "/usr/local/lib/python3.7/site-packages/prefect/utilities/httpx.py", line 151, in post
raise_for_status=raise_for_status,
File "/usr/local/lib/python3.7/site-packages/prefect/utilities/httpx.py", line 61, in request
request, auth=auth, follow_redirects=follow_redirects
File "/usr/local/lib/python3.7/site-packages/httpx/_client.py", line 1597, in send
history=[],
File "/usr/local/lib/python3.7/site-packages/httpx/_client.py", line 1624, in _send_handling_auth
history=history,
File "/usr/local/lib/python3.7/site-packages/httpx/_client.py", line 1658, in _send_handling_redirects
response = await self._send_single_request(request)
File "/usr/local/lib/python3.7/site-packages/httpx/_client.py", line 1695, in _send_single_request
response = await transport.handle_async_request(request)
File "/usr/local/lib/python3.7/site-packages/httpx/_transports/default.py", line 353, in handle_async_request
resp = await self._pool.handle_async_request(req)
File "/usr/local/lib/python3.7/contextlib.py", line 130, in __exit__
self.gen.throw(type, value, traceback)
File "/usr/local/lib/python3.7/site-packages/httpx/_transports/default.py", line 77, in map_httpcore_exceptions
raise mapped_exc(message) from exc
httpx.ConnectTimeout
--- Orion logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/httpcore/backends/asyncio.py", line 104, in connect_tcp
local_host=local_address,
File "/usr/local/lib/python3.7/site-packages/anyio/_core/_sockets.py", line 180, in connect_tcp
await event.wait()
File "/usr/local/lib/python3.7/site-packages/anyio/_backends/_asyncio.py", line 574, in __aexit__
raise exceptions[0]
File "/usr/local/lib/python3.7/site-packages/anyio/_backends/_asyncio.py", line 611, in _run_wrapped_task
await coro
File "/usr/local/lib/python3.7/site-packages/anyio/_core/_sockets.py", line 127, in try_connect
stream = await asynclib.connect_tcp(remote_host, remote_port, local_address)
File "/usr/local/lib/python3.7/site-packages/anyio/_backends/_asyncio.py", line 1519, in connect_tcp
local_addr=local_addr)
File "/usr/local/lib/python3.7/asyncio/base_events.py", line 949, in create_connection
await self.sock_connect(sock, address)
File "/usr/local/lib/python3.7/asyncio/selector_events.py", line 473, in sock_connect
return await fut
concurrent.futures._base.CancelledError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/httpcore/_exceptions.py", line 8, in map_exceptions
yield
File "/usr/local/lib/python3.7/site-packages/httpcore/backends/asyncio.py", line 104, in connect_tcp
local_host=local_address,
File "/usr/local/lib/python3.7/site-packages/anyio/_core/_tasks.py", line 103, in __exit__
raise TimeoutError
TimeoutError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/httpx/_transports/default.py", line 60, in map_httpcore_exceptions
yield
File "/usr/local/lib/python3.7/site-packages/httpx/_transports/default.py", line 353, in handle_async_request
resp = await self._pool.handle_async_request(req)
File "/usr/local/lib/python3.7/site-packages/httpcore/_async/connection_pool.py", line 253, in handle_async_request
raise exc
File "/usr/local/lib/python3.7/site-packages/httpcore/_async/connection_pool.py", line 237, in handle_async_request
response = await connection.handle_async_request(request)
File "/usr/local/lib/python3.7/site-packages/httpcore/_async/connection.py", line 86, in handle_async_request
raise exc
File "/usr/local/lib/python3.7/site-packages/httpcore/_async/connection.py", line 63, in handle_async_request
stream = await self._connect(request)
File "/usr/local/lib/python3.7/site-packages/httpcore/_async/connection.py", line 111, in _connect
stream = await self._network_backend.connect_tcp(**kwargs)
File "/usr/local/lib/python3.7/site-packages/httpcore/backends/auto.py", line 24, in connect_tcp
host, port, timeout=timeout, local_address=local_address
File "/usr/local/lib/python3.7/site-packages/httpcore/backends/asyncio.py", line 104, in connect_tcp
local_host=local_address,
File "/usr/local/lib/python3.7/contextlib.py", line 130, in __exit__
self.gen.throw(type, value, traceback)
File "/usr/local/lib/python3.7/site-packages/httpcore/_exceptions.py", line 12, in map_exceptions
raise to_exc(exc)
httpcore.ConnectTimeout
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/prefect/logging/handlers.py", line 133, in send_logs
await client.create_logs(self._pending_logs)
File "/usr/local/lib/python3.7/site-packages/prefect/client.py", line 1614, in create_logs
await self._client.post(f"/logs/", json=serialized_logs)
File "/usr/local/lib/python3.7/site-packages/prefect/utilities/httpx.py", line 151, in post
raise_for_status=raise_for_status,
File "/usr/local/lib/python3.7/site-packages/prefect/utilities/httpx.py", line 61, in request
request, auth=auth, follow_redirects=follow_redirects
File "/usr/local/lib/python3.7/site-packages/httpx/_client.py", line 1597, in send
history=[],
File "/usr/local/lib/python3.7/site-packages/httpx/_client.py", line 1624, in _send_handling_auth
history=history,
File "/usr/local/lib/python3.7/site-packages/httpx/_client.py", line 1658, in _send_handling_redirects
response = await self._send_single_request(request)
File "/usr/local/lib/python3.7/site-packages/httpx/_client.py", line 1695, in _send_single_request
response = await transport.handle_async_request(request)
File "/usr/local/lib/python3.7/site-packages/httpx/_transports/default.py", line 353, in handle_async_request
resp = await self._pool.handle_async_request(req)
File "/usr/local/lib/python3.7/contextlib.py", line 130, in __exit__
self.gen.throw(type, value, traceback)
File "/usr/local/lib/python3.7/site-packages/httpx/_transports/default.py", line 77, in map_httpcore_exceptions
raise mapped_exc(message) from exc
httpx.ConnectTimeout
Worker information:
Approximate queue length: 1
Pending log batch length: 1
Pending log batch size: 376
The log worker will attempt to send these logs again in 2.0s
09:58:31.239 | ERROR | Flow run 'aboriginal-seahorse' - Crash detected! Request to http://host.docker.internal:4200/api/blocks/get_default_storage_block timed out.
--- Orion logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/httpcore/backends/asyncio.py", line 104, in connect_tcp
local_host=local_address,
File "/usr/local/lib/python3.7/site-packages/anyio/_core/_sockets.py", line 180, in connect_tcp
await event.wait()
File "/usr/local/lib/python3.7/site-packages/anyio/_backends/_asyncio.py", line 574, in __aexit__
raise exceptions[0]
File "/usr/local/lib/python3.7/site-packages/anyio/_backends/_asyncio.py", line 611, in _run_wrapped_task
await coro
File "/usr/local/lib/python3.7/site-packages/anyio/_core/_sockets.py", line 127, in try_connect
stream = await asynclib.connect_tcp(remote_host, remote_port, local_address)
File "/usr/local/lib/python3.7/site-packages/anyio/_backends/_asyncio.py", line 1519, in connect_tcp
local_addr=local_addr)
File "/usr/local/lib/python3.7/asyncio/base_events.py", line 949, in create_connection
await self.sock_connect(sock, address)
File "/usr/local/lib/python3.7/asyncio/selector_events.py", line 473, in sock_connect
return await fut
concurrent.futures._base.CancelledError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/httpcore/_exceptions.py", line 8, in map_exceptions
yield
File "/usr/local/lib/python3.7/site-packages/httpcore/backends/asyncio.py", line 104, in connect_tcp
local_host=local_address,
File "/usr/local/lib/python3.7/site-packages/anyio/_core/_tasks.py", line 103, in __exit__
raise TimeoutError
TimeoutError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/httpx/_transports/default.py", line 60, in map_httpcore_exceptions
yield
File "/usr/local/lib/python3.7/site-packages/httpx/_transports/default.py", line 353, in handle_async_request
resp = await self._pool.handle_async_request(req)
File "/usr/local/lib/python3.7/site-packages/httpcore/_async/connection_pool.py", line 253, in handle_async_request
raise exc
File "/usr/local/lib/python3.7/site-packages/httpcore/_async/connection_pool.py", line 237, in handle_async_request
response = await connection.handle_async_request(request)
File "/usr/local/lib/python3.7/site-packages/httpcore/_async/connection.py", line 86, in handle_async_request
raise exc
File "/usr/local/lib/python3.7/site-packages/httpcore/_async/connection.py", line 63, in handle_async_request
stream = await self._connect(request)
File "/usr/local/lib/python3.7/site-packages/httpcore/_async/connection.py", line 111, in _connect
stream = await self._network_backend.connect_tcp(**kwargs)
File "/usr/local/lib/python3.7/site-packages/httpcore/backends/auto.py", line 24, in connect_tcp
host, port, timeout=timeout, local_address=local_address
File "/usr/local/lib/python3.7/site-packages/httpcore/backends/asyncio.py", line 104, in connect_tcp
local_host=local_address,
File "/usr/local/lib/python3.7/contextlib.py", line 130, in __exit__
self.gen.throw(type, value, traceback)
File "/usr/local/lib/python3.7/site-packages/httpcore/_exceptions.py", line 12, in map_exceptions
raise to_exc(exc)
httpcore.ConnectTimeout
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/prefect/logging/handlers.py", line 133, in send_logs
await client.create_logs(self._pending_logs)
File "/usr/local/lib/python3.7/site-packages/prefect/client.py", line 1614, in create_logs
await self._client.post(f"/logs/", json=serialized_logs)
File "/usr/local/lib/python3.7/site-packages/prefect/utilities/httpx.py", line 151, in post
raise_for_status=raise_for_status,
File "/usr/local/lib/python3.7/site-packages/prefect/utilities/httpx.py", line 61, in request
request, auth=auth, follow_redirects=follow_redirects
File "/usr/local/lib/python3.7/site-packages/httpx/_client.py", line 1597, in send
history=[],
File "/usr/local/lib/python3.7/site-packages/httpx/_client.py", line 1624, in _send_handling_auth
history=history,
File "/usr/local/lib/python3.7/site-packages/httpx/_client.py", line 1658, in _send_handling_redirects
response = await self._send_single_request(request)
File "/usr/local/lib/python3.7/site-packages/httpx/_client.py", line 1695, in _send_single_request
response = await transport.handle_async_request(request)
File "/usr/local/lib/python3.7/site-packages/httpx/_transports/default.py", line 353, in handle_async_request
resp = await self._pool.handle_async_request(req)
File "/usr/local/lib/python3.7/contextlib.py", line 130, in __exit__
self.gen.throw(type, value, traceback)
File "/usr/local/lib/python3.7/site-packages/httpx/_transports/default.py", line 77, in map_httpcore_exceptions
raise mapped_exc(message) from exc
httpx.ConnectTimeout
Worker information:
Approximate queue length: 1
Pending log batch length: 2
Pending log batch size: 752
The log worker will attempt to send these logs again in 2.0s