I’m running a hybrid model with Prefect Cloud connected to an agent continuously running on a virtual machine on our local network. Flows are running without any issues, but I’m getting a long error message randomly throughout the day. It doesn’t seem to happen when I’m running flows and it always manages to reconnect.
Has anyone else experienced the (<ConnectionState.CLOSED: 3>, <ConnectionInputs.SEND_HEADERS: 0>)
error message? If so, has a root cause been identified and/or a workaround?
Failed the last 3 attempts. Please check your environment and configuration.
Examples of recent errors:
Traceback (most recent call last):
File "C:\Users\King\Documents\GitHub\Prefect\lib\site-packages\h2\connection.py", line 224, in process_input
func, target_state = self._transitions[(self.state, input_)]
KeyError: (<ConnectionState.CLOSED: 3>, <ConnectionInputs.SEND_HEADERS: 0>)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\King\Documents\GitHub\Prefect\lib\site-packages\httpcore\_async\http2.py", line 116, in handle_async_request
await self._send_request_headers(request=request, stream_id=stream_id)
File "C:\Users\King\Documents\GitHub\Prefect\lib\site-packages\httpcore\_async\http2.py", line 213, in _send_request_headers
self._h2_state.send_headers(stream_id, headers, end_stream=end_stream)
File "C:\Users\King\Documents\GitHub\Prefect\lib\site-packages\h2\connection.py", line 766, in send_headers
self.state_machine.process_input(ConnectionInputs.SEND_HEADERS)
File "C:\Users\King\Documents\GitHub\Prefect\lib\site-packages\h2\connection.py", line 228, in process_input
raise ProtocolError(
h2.exceptions.ProtocolError: Invalid input ConnectionInputs.SEND_HEADERS in state ConnectionState.CLOSED
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\King\Documents\GitHub\Prefect\lib\site-packages\httpx\_transports\default.py", line 60, in map_httpcore_exceptions
yield
File "C:\Users\King\Documents\GitHub\Prefect\lib\site-packages\httpx\_transports\default.py", line 353, in handle_async_request
resp = await self._pool.handle_async_request(req)
File "C:\Users\King\Documents\GitHub\Prefect\lib\site-packages\httpcore\_async\connection_pool.py", line 253, in handle_async_request
raise exc
File "C:\Users\King\Documents\GitHub\Prefect\lib\site-packages\httpcore\_async\connection_pool.py", line 237, in handle_async_request
response = await connection.handle_async_request(request)
File "C:\Users\King\Documents\GitHub\Prefect\lib\site-packages\httpcore\_async\connection.py", line 90, in handle_async_request
return await self._connection.handle_async_request(request)
File "C:\Users\King\Documents\GitHub\Prefect\lib\site-packages\httpcore\_async\http2.py", line 152, in handle_async_request
raise LocalProtocolError(exc) # pragma: nocover
httpcore.LocalProtocolError: Invalid input ConnectionInputs.SEND_HEADERS in state ConnectionState.CLOSED
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:\Users\King\Documents\GitHub\Prefect\lib\site-packages\prefect\utilities\services.py", line 53, in critical_service_loop
await workload()
File "C:\Users\King\Documents\GitHub\Prefect\lib\site-packages\prefect\agent.py", line 264, in check_for_cancelled_flow_runs
named_cancelling_flow_runs = await self.client.read_flow_runs(
File "C:\Users\King\Documents\GitHub\Prefect\lib\site-packages\prefect\client\orchestration.py", line 1700, in read_flow_runs
response = await self._client.post("/flow_runs/filter", json=body)
File "C:\Users\King\Documents\GitHub\Prefect\lib\site-packages\httpx\_client.py", line 1845, in post
return await self.request(
File "C:\Users\King\Documents\GitHub\Prefect\lib\site-packages\httpx\_client.py", line 1530, in request
return await self.send(request, auth=auth, follow_redirects=follow_redirects)
File "C:\Users\King\Documents\GitHub\Prefect\lib\site-packages\prefect\client\base.py", line 251, in send
response = await self._send_with_retry(
File "C:\Users\King\Documents\GitHub\Prefect\lib\site-packages\prefect\client\base.py", line 194, in _send_with_retry
response = await request()
File "C:\Users\King\Documents\GitHub\Prefect\lib\site-packages\httpx\_client.py", line 1617, in send
response = await self._send_handling_auth(
File "C:\Users\King\Documents\GitHub\Prefect\lib\site-packages\httpx\_client.py", line 1645, in _send_handling_auth
response = await self._send_handling_redirects(
File "C:\Users\King\Documents\GitHub\Prefect\lib\site-packages\httpx\_client.py", line 1682, in _send_handling_redirects
response = await self._send_single_request(request)
File "C:\Users\King\Documents\GitHub\Prefect\lib\site-packages\httpx\_client.py", line 1719, in _send_single_request
response = await transport.handle_async_request(request)
File "C:\Users\King\Documents\GitHub\Prefect\lib\site-packages\httpx\_transports\default.py", line 353, in handle_async_request
resp = await self._pool.handle_async_request(req)
File "C:\Program Files\Python39\lib\contextlib.py", line 137, in __exit__
self.gen.throw(typ, value, traceback)
File "C:\Users\King\Documents\GitHub\Prefect\lib\site-packages\httpx\_transports\default.py", line 77, in map_httpcore_exceptions
raise mapped_exc(message) from exc
httpx.LocalProtocolError: Invalid input ConnectionInputs.SEND_HEADERS in state ConnectionState.CLOSED
Backing off due to consecutive errors, using increased interval of 60.0s.
Resetting backoff due to successful run.