Hi Community,
We need help with Prefect agent setup.
We have hosted Prefect server in EC2 instance and created prefect agent in ECS Fargate.
We are using docker image as “prefecthq/prefect:2.10.8-python3.10”.
We configured environment variables for "‘PREFECT_API_URL’ , ‘PREFECT_CLOUD_API’ ‘PREFECT_LOGGING_LEVEL’: ‘INFO’,‘AWS_RETRY_MODE’:‘adaptive’,‘AWS_MAX_ATTEMPTS’:‘10’.
In ECS Fargate while running services and task we are getting below error.
If someone can point out which environment variable setup missing then it help us to complete setup.
Error logs :
Agent started! Looking for work from work pool ‘default-agent-pool’…
Failed the last 3 attempts. Please check your environment and configuration.
Examples of recent errors:
Traceback (most recent call last):
File “/usr/local/lib/python3.10/site-packages/httpcore/backends/asyncio.py”,
line 111, in connect_tcp
stream: anyio.abc.ByteStream = await anyio.connect_tcp(
File “/usr/local/lib/python3.10/site-packages/anyio/_core/_sockets.py”, line
213, in connect_tcp
async with create_task_group() as tg:
File “/usr/local/lib/python3.10/site-packages/anyio/_backends/_asyncio.py”,
line 658, in aexit
raise CancelledError
asyncio.exceptions.CancelledError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File “/usr/local/lib/python3.10/site-packages/httpcore/_exceptions.py”, line
10, in map_exceptions
yield
File “/usr/local/lib/python3.10/site-packages/httpcore/backends/asyncio.py”,
line 110, in connect_tcp
with anyio.fail_after(timeout):
File “/usr/local/lib/python3.10/site-packages/anyio/_core/_tasks.py”, line
118, in exit
raise TimeoutError
TimeoutError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File “/usr/local/lib/python3.10/site-packages/httpx/_transports/default.py”,
line 60, in map_httpcore_exceptions
yield
File “/usr/local/lib/python3.10/site-packages/httpx/_transports/default.py”,
line 353, in handle_async_request
resp = await self._pool.handle_async_request(req)
File
“/usr/local/lib/python3.10/site-packages/httpcore/_async/connection_pool.py”,
line 253, in handle_async_request
raise exc
File
“/usr/local/lib/python3.10/site-packages/httpcore/_async/connection_pool.py”,
line 237, in handle_async_request
response = await connection.handle_async_request(request)
File “/usr/local/lib/python3.10/site-packages/httpcore/_async/connection.py”,
line 86, in handle_async_request
raise exc
File “/usr/local/lib/python3.10/site-packages/httpcore/_async/connection.py”,
line 63, in handle_async_request
stream = await self._connect(request)
File “/usr/local/lib/python3.10/site-packages/httpcore/_async/connection.py”,
line 111, in _connect
stream = await self._network_backend.connect_tcp(**kwargs)
File “/usr/local/lib/python3.10/site-packages/httpcore/backends/auto.py”, line
29, in connect_tcp
return await self._backend.connect_tcp(
File “/usr/local/lib/python3.10/site-packages/httpcore/backends/asyncio.py”,
line 109, in connect_tcp
with map_exceptions(exc_map):
File “/usr/local/lib/python3.10/contextlib.py”, line 153, in exit
self.gen.throw(typ, value, traceback)
File “/usr/local/lib/python3.10/site-packages/httpcore/_exceptions.py”, line
14, in map_exceptions
raise to_exc(exc)
httpcore.ConnectTimeout
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File “/usr/local/lib/python3.10/site-packages/prefect/utilities/services.py”,
line 46, in critical_service_loop
await workload()
File “/usr/local/lib/python3.10/site-packages/prefect/agent.py”, line 196, in
get_and_submit_flow_runs
work_queue_names=,
File “/usr/local/lib/python3.10/site-packages/prefect/agent.py”, line 196, in
work_queue_names=,
File “/usr/local/lib/python3.10/site-packages/prefect/agent.py”, line 144, in
get_work_queues
work_queue = await self.client.read_work_queue_by_name(
File
“/usr/local/lib/python3.10/site-packages/prefect/client/orchestration.py”, line
844, in read_work_queue_by_name
response = await self._client.get(
File “/usr/local/lib/python3.10/site-packages/httpx/_client.py”, line 1754, in
get
return await self.request(
File “/usr/local/lib/python3.10/site-packages/httpx/_client.py”, line 1530, in
request
return await self.send(request, auth=auth,
follow_redirects=follow_redirects)
File “/usr/local/lib/python3.10/site-packages/prefect/client/base.py”, line
246, in send
response = await self._send_with_retry(
File “/usr/local/lib/python3.10/site-packages/prefect/client/base.py”, line
192, in _send_with_retry
response = await request()
File “/usr/local/lib/python3.10/site-packages/httpx/_client.py”, line 1617, in
send
response = await self._send_handling_auth(
File “/usr/local/lib/python3.10/site-packages/httpx/_client.py”, line 1645, in
_send_handling_auth
response = await self._send_handling_redirects(
File “/usr/local/lib/python3.10/site-packages/httpx/_client.py”, line 1682, in
_send_handling_redirects
response = await self._send_single_request(request)
File “/usr/local/lib/python3.10/site-packages/httpx/_client.py”, line 1719, in
_send_single_request
response = await transport.handle_async_request(request)
File “/usr/local/lib/python3.10/site-packages/httpx/_transports/default.py”,
line 352, in handle_async_request
with map_httpcore_exceptions():
File “/usr/local/lib/python3.10/contextlib.py”, line 153, in exit
self.gen.throw(typ, value, traceback)
File “/usr/local/lib/python3.10/site-packages/httpx/_transports/default.py”,
line 77, in map_httpcore_exceptions
raise mapped_exc(message) from exc
httpx.ConnectTimeout
Failed the last 3 attempts. Please check your environment and configuration.
Examples of recent errors:
Traceback (most recent call last):
File “/usr/local/lib/python3.10/site-packages/httpcore/backends/asyncio.py”,
line 111, in connect_tcp
stream: anyio.abc.ByteStream = await anyio.connect_tcp(
File “/usr/local/lib/python3.10/site-packages/anyio/_core/_sockets.py”, line
213, in connect_tcp
async with create_task_group() as tg:
File “/usr/local/lib/python3.10/site-packages/anyio/_backends/_asyncio.py”,
line 658, in aexit
raise CancelledError
asyncio.exceptions.CancelledError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File “/usr/local/lib/python3.10/site-packages/httpcore/_exceptions.py”, line
10, in map_exceptions
yield
File “/usr/local/lib/python3.10/site-packages/httpcore/backends/asyncio.py”,
line 110, in connect_tcp
with anyio.fail_after(timeout):
File “/usr/local/lib/python3.10/site-packages/anyio/_core/_tasks.py”, line
118, in exit
raise TimeoutError
TimeoutError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File “/usr/local/lib/python3.10/site-packages/httpx/_transports/default.py”,
line 60, in map_httpcore_exceptions
yield
File “/usr/local/lib/python3.10/site-packages/httpx/_transports/default.py”,
line 353, in handle_async_request
resp = await self._pool.handle_async_request(req)
File
“/usr/local/lib/python3.10/site-packages/httpcore/_async/connection_pool.py”,
line 253, in handle_async_request
raise exc
File
“/usr/local/lib/python3.10/site-packages/httpcore/_async/connection_pool.py”,
line 237, in handle_async_request
response = await connection.handle_async_request(request)
File “/usr/local/lib/python3.10/site-packages/httpcore/_async/connection.py”,
line 86, in handle_async_request
raise exc
File “/usr/local/lib/python3.10/site-packages/httpcore/_async/connection.py”,
line 63, in handle_async_request
stream = await self._connect(request)
File “/usr/local/lib/python3.10/site-packages/httpcore/_async/connection.py”,
line 111, in _connect
stream = await self._network_backend.connect_tcp(**kwargs)
File “/usr/local/lib/python3.10/site-packages/httpcore/backends/auto.py”, line
29, in connect_tcp
return await self._backend.connect_tcp(
File “/usr/local/lib/python3.10/site-packages/httpcore/backends/asyncio.py”,
line 109, in connect_tcp
with map_exceptions(exc_map):
File “/usr/local/lib/python3.10/contextlib.py”, line 153, in exit
self.gen.throw(typ, value, traceback)
File “/usr/local/lib/python3.10/site-packages/httpcore/_exceptions.py”, line
14, in map_exceptions
raise to_exc(exc)
httpcore.ConnectTimeout
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File “/usr/local/lib/python3.10/site-packages/prefect/utilities/services.py”,
line 46, in critical_service_loop
await workload()
File “/usr/local/lib/python3.10/site-packages/prefect/agent.py”, line 261, in
check_for_cancelled_flow_runs
async for work_queue in self.get_work_queues():
File “/usr/local/lib/python3.10/site-packages/prefect/agent.py”, line 144, in
get_work_queues
work_queue = await self.client.read_work_queue_by_name(
File
“/usr/local/lib/python3.10/site-packages/prefect/client/orchestration.py”, line
844, in read_work_queue_by_name
response = await self._client.get(
File “/usr/local/lib/python3.10/site-packages/httpx/_client.py”, line 1754, in
get
return await self.request(
File “/usr/local/lib/python3.10/site-packages/httpx/_client.py”, line 1530, in
request
return await self.send(request, auth=auth,
follow_redirects=follow_redirects)
File “/usr/local/lib/python3.10/site-packages/prefect/client/base.py”, line
246, in send
response = await self._send_with_retry(
File “/usr/local/lib/python3.10/site-packages/prefect/client/base.py”, line
192, in _send_with_retry
response = await request()
File “/usr/local/lib/python3.10/site-packages/httpx/_client.py”, line 1617, in
send
response = await self._send_handling_auth(
File “/usr/local/lib/python3.10/site-packages/httpx/_client.py”, line 1645, in
_send_handling_auth
response = await self._send_handling_redirects(
File “/usr/local/lib/python3.10/site-packages/httpx/_client.py”, line 1682, in
_send_handling_redirects
response = await self._send_single_request(request)
File “/usr/local/lib/python3.10/site-packages/httpx/_client.py”, line 1719, in
_send_single_request
response = await transport.handle_async_request(request)
File “/usr/local/lib/python3.10/site-packages/httpx/_transports/default.py”,
line 352, in handle_async_request
with map_httpcore_exceptions():
File “/usr/local/lib/python3.10/contextlib.py”, line 153, in exit
self.gen.throw(typ, value, traceback)
File “/usr/local/lib/python3.10/site-packages/httpx/_transports/default.py”,
line 77, in map_httpcore_exceptions
raise mapped_exc(message) from exc
httpx.ConnectTimeout
Agent stopped!