Hi. I am running a prefect server, a database, and a worker using docker compose. It works beautifully on most machines, but on a Linux VM I keep on getting an error from the server service:
Traceback (most recent call last):
File "/usr/local/lib/python3.11/site-packages/starlette/routing.py", line 677, in lifespan
async with self.lifespan_context(app) as maybe_state:
File "/usr/local/lib/python3.11/contextlib.py", line 204, in __aenter__
return await anext(self.gen)
^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/prefect/server/api/server.py", line 519, in lifespan
await run_migrations()
File "/usr/local/lib/python3.11/site-packages/prefect/server/api/server.py", line 442, in run_migrations
await db.create_db()
File "/usr/local/lib/python3.11/site-packages/prefect/server/database/interface.py", line 56, in create_db
await self.run_migrations_upgrade()
File "/usr/local/lib/python3.11/site-packages/prefect/server/database/interface.py", line 64, in run_migrations_upgrade
await run_sync_in_worker_thread(alembic_upgrade)
File "/usr/local/lib/python3.11/site-packages/prefect/utilities/asyncutils.py", line 91, in run_sync_in_worker_thread
return await anyio.to_thread.run_sync(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/anyio/to_thread.py", line 33, in run_sync
return await get_asynclib().run_sync_in_worker_thread(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/anyio/_backends/_asyncio.py", line 877, in run_sync_in_worker_thread
return await future
^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/anyio/_backends/_asyncio.py", line 807, in run
result = context.run(func, *args)
^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/prefect/server/database/alembic_commands.py", line 24, in wrapper
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/prefect/server/database/alembic_commands.py", line 53, in alembic_upgrade
alembic.command.upgrade(alembic_config(), revision, sql=dry_run)
File "/usr/local/lib/python3.11/site-packages/alembic/command.py", line 382, in upgrade
script.run_env()
File "/usr/local/lib/python3.11/site-packages/alembic/script/base.py", line 578, in run_env
util.load_python_file(self.dir, "env.py")
File "/usr/local/lib/python3.11/site-packages/alembic/util/pyfiles.py", line 93, in load_python_file
module = load_module_py(module_id, path)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/alembic/util/pyfiles.py", line 109, in load_module_py
spec.loader.exec_module(module) # type: ignore
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "<frozen importlib._bootstrap_external>", line 940, in exec_module
File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
File "/usr/local/lib/python3.11/site-packages/prefect/server/database/migrations/env.py", line 175, in <module>
apply_migrations()
File "/usr/local/lib/python3.11/site-packages/prefect/utilities/asyncutils.py", line 243, in coroutine_wrapper
return run_async_from_worker_thread(async_fn, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/prefect/utilities/asyncutils.py", line 177, in run_async_from_worker_thread
return anyio.from_thread.run(call)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/anyio/from_thread.py", line 47, in run
return asynclib.run_async_from_thread(func, *args)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/anyio/_backends/_asyncio.py", line 906, in run_async_from_thread
return f.result()
^^^^^^^^^^
File "/usr/local/lib/python3.11/concurrent/futures/_base.py", line 456, in result
return self.__get_result()
^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/concurrent/futures/_base.py", line 401, in __get_result
raise self._exception
TimeoutError
Application startup failed. Exiting.
My docker-compose.yml file looks like this:
version: "3.9"
services:
### PostgreSQL Database
database:
image: postgres:15.2-alpine
restart: always
environment:
- POSTGRES_USER=postgres
- POSTGRES_PASSWORD=postgres
- POSTGRES_DB=prefect
expose:
- 5432
volumes:
- postgres:/var/lib/postgresql/data
### Prefect Server
server:
image: prefecthq/prefect:2.11.5-python3.11
restart: always
volumes:
- prefect:/home/azureuser/.prefect
entrypoint:
[
"/opt/prefect/entrypoint.sh",
"prefect",
"server",
"start"
]
environment:
- PREFECT_API_URL=http://127.0.0.1:4200/api
- PREFECT_UI_URL=http://127.0.0.1:4200/api
- PREFECT_SERVER_API_HOST=0.0.0.0
- PREFECT_API_DATABASE_CONNECTION_URL=postgresql+asyncpg://postgres:postgres@database:5432/prefect
ports:
- 4200:4200
depends_on:
- database
### Prefect Worker
worker:
image: prefect-worker:dev
restart: always
entrypoint:
[
"/opt/prefect/entrypoint.sh",
"prefect",
"worker",
"start",
"--pool",
"default-worker-pool"
]
environment:
- PREFECT_API_URL=http://server:4200/api
depends_on:
- database
- server
profiles: [ "production" ]
### Prefect CLI
cli:
image: prefect-worker:dev
entrypoint: "bash"
environment:
- PREFECT_API_URL=http://server:4200/api
profiles: [ "cli" ]
volumes:
prefect:
postgres:
To start it up I run docker compose --profile production up.
Any idea what could be wrong here? I tried flushing the volumes, force restarting etc. but nothing helps. I also tried increasing the timeout and indeed it’s correlated (if I increase to 10 seconds it will take that amount to raise the error) so I assume it’s an issue with connecting to the database, but I am not sure why.
Thanks!