Now that I took a closer look at the work pool in Prefect UI, there are many fields in the job template that are missing.
Here is the full template that I use when creating or updating the work pool from the CLI.
{
"variables": {
"type": "object",
"required": [
"resource_group_name",
"subscription_id"
],
"properties": {
"cpu": {
"type": "number",
"title": "CPU",
"default": 4,
"description": "The number of virtual CPUs to assign to the task container. If not provided, a default value of 1.0 will be used."
},
"env": {
"type": "object",
"title": "Environment Variables",
"default": {
"ENV": "$ENV",
"DATABRICKS_HOST": "$DATABRICKS_HOST",
"DATABRICKS_HTTP_PATH": "$DATABRICKS_HTTP_PATH",
"AZURE_SUBSCRIPTION_ID": "$AZURE_SUBSCRIPTION_ID"
},
"description": "Environment variables to set when starting a flow run.",
"additionalProperties": {
"type": "string"
}
},
"name": {
"type": "string",
"title": "Name",
"description": "Name given to infrastructure created by a worker."
},
"image": {
"type": "string",
"title": "Image",
"default": "prefect/worker:latest",
"description": "The image to use for the Prefect container in the task. This value defaults to a Prefect base image matching your local versions."
},
"labels": {
"type": "object",
"title": "Labels",
"description": "Labels applied to infrastructure created by a worker.",
"additionalProperties": {
"type": "string"
}
},
"memory": {
"type": "number",
"title": "Memory",
"default": 4,
"description": "The amount of memory in gigabytes to provide to the ACI task. Valid amounts are specified in the Azure documentation. If not provided, a default value of 1.0 will be used unless present on the task definition."
},
"command": {
"type": "string",
"title": "Command",
"description": "The command to use when starting a flow run. In most cases, this should be left blank and the command will be automatically generated by the worker."
},
"gpu_sku": {
"type": "string",
"title": "GPU SKU",
"description": "The Azure GPU SKU to use. See the ACI documentation for a list of GPU SKUs available in each Azure region."
},
"gpu_count": {
"type": "integer",
"title": "GPU Count",
"default": 0,
"description": "The number of GPUs to assign to the task container. If not provided, no GPU will be used."
},
"entrypoint": {
"type": "string",
"title": "Entrypoint",
"default": "/opt/prefect/entrypoint.sh",
"description": "The entrypoint of the container you wish you run. This value defaults to the entrypoint used by Prefect images and should only be changed when using a custom image that is not based on an official Prefect image. Any commands set on deployments will be passed to the entrypoint as parameters."
},
"identities": {
"type": "array",
"items": {
"type": "string"
},
"title": "Identities",
"default": [
"/subscriptions/$AZURE_SUBSCRIPTION_ID/resourceGroups/leap-$ENV-we-rg-aks/providers/Microsoft.ManagedIdentity/userAssignedIdentities/leap-$ENV-we-mi-prefect-001"
],
"description": "A list of user-assigned identities to associate with the container group. The identities should be an ARM resource IDs in the form: '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}'."
},
"subnet_ids": {
"type": "array",
"items": {
"type": "string"
},
"title": "Subnet IDs",
"default": [
"/subscriptions/$AZURE_SUBSCRIPTION_ID/resourceGroups/leap-$ENV-we-rg-network/providers/Microsoft.Network/virtualNetworks/leap-$ENV-we-vnet-main-001/subnets/leap-$ENV-we-snet-aci-001"
],
"description": "A list of subnet IDs to associate with the container group. "
},
"dns_servers": {
"type": "array",
"items": {
"type": "string"
},
"title": "DNS Servers",
"description": "A list of DNS servers to associate with the container group."
},
"stream_output": {
"type": "boolean",
"title": "Stream Output",
"default": true,
"description": "If `True`, logs will be streamed from the Prefect container to the local console."
},
"image_registry": {
"anyOf": [
{
"$ref": "#/definitions/DockerRegistry"
},
{
"$ref": "#/definitions/ACRManagedIdentity"
}
],
"title": "Image Registry (Optional)",
"default": {
"identity": "/subscriptions/$AZURE_SUBSCRIPTION_ID/resourcegroups/leap-$ENV-we-rg-aks/providers/Microsoft.ManagedIdentity/userAssignedIdentities/leap-$ENV-we-mi-prefect-001",
"registry_url": "leap$ENVweops.azurecr.io"
},
"description": "To use any private container registry with a username and password, choose DockerRegistry. To use a private Azure Container Registry with a managed identity, choose ACRManagedIdentity."
},
"aci_credentials": {
"allOf": [
{
"$ref": "#/definitions/AzureContainerInstanceCredentials"
}
],
"title": "Aci Credentials",
"description": "The credentials to use to authenticate with Azure."
},
"subscription_id": {
"type": "string",
"title": "Azure Subscription ID",
"format": "password",
"default": "$AZURE_SUBSCRIPTION_ID",
"writeOnly": true,
"description": "The ID of the Azure subscription to create containers under."
},
"resource_group_name": {
"type": "string",
"title": "Azure Resource Group Name",
"default": "leap-$ENV-we-rg-aci",
"description": "The name of the Azure Resource Group in which to run Prefect ACI tasks."
},
"task_watch_poll_interval": {
"type": "number",
"title": "Task Watch Poll Interval",
"default": 5,
"description": "The number of seconds to wait between Azure API calls while monitoring the state of an Azure Container Instances task."
},
"task_start_timeout_seconds": {
"type": "integer",
"title": "Task Start Timeout Seconds",
"default": 240,
"description": "The amount of time to watch for the start of the ACI container. before marking it as failed."
}
},
"definitions": {
"DockerRegistry": {
"type": "object",
"title": "DockerRegistry",
"required": [
"username",
"password",
"registry_url"
],
"properties": {
"reauth": {
"type": "boolean",
"title": "Reauth",
"default": true,
"description": "Whether or not to reauthenticate on each interaction."
},
"password": {
"type": "string",
"title": "Password",
"format": "password",
"writeOnly": true,
"description": "The password to log into the registry with."
},
"username": {
"type": "string",
"title": "Username",
"description": "The username to log into the registry with."
},
"registry_url": {
"type": "string",
"title": "Registry Url",
"description": "The URL to the registry. Generally, \"http\" or \"https\" can be omitted."
}
},
"description": "Connects to a Docker registry.\n\nRequires a Docker Engine to be connectable.",
"secret_fields": [
"password"
],
"block_type_slug": "docker-registry",
"block_schema_references": {}
},
"ACRManagedIdentity": {
"type": "object",
"title": "ACRManagedIdentity",
"required": [
"registry_url",
"identity"
],
"properties": {
"identity": {
"type": "string",
"title": "Identity",
"description": "The user-assigned Azure managed identity for the private registry."
},
"registry_url": {
"type": "string",
"title": "Registry URL",
"description": "The URL to the registry, such as myregistry.azurecr.io. Generally, 'http' or 'https' can be omitted."
}
},
"description": "Use a Managed Identity to access Azure Container registry. Requires the\nuser-assigned managed identity be available to the ACI container group."
},
"AzureContainerInstanceCredentials": {
"type": "object",
"title": "AzureContainerInstanceCredentials",
"properties": {
"client_id": {
"type": "string",
"title": "Client ID",
"description": "The service principal client ID. If none of client_id, tenant_id, and client_secret are provided, will use DefaultAzureCredential; else will need to provide all three to use ClientSecretCredential."
},
"tenant_id": {
"type": "string",
"title": "Tenant ID",
"description": "The service principal tenant ID.If none of client_id, tenant_id, and client_secret are provided, will use DefaultAzureCredential; else will need to provide all three to use ClientSecretCredential."
},
"client_secret": {
"type": "string",
"title": "Client Secret",
"format": "password",
"writeOnly": true,
"description": "The service principal client secret.If none of client_id, tenant_id, and client_secret are provided, will use DefaultAzureCredential; else will need to provide all three to use ClientSecretCredential."
},
"credential_kwargs": {
"type": "object",
"title": "Additional Credential Keyword Arguments",
"description": "Additional keyword arguments to pass to `ClientSecretCredential` or `DefaultAzureCredential`."
}
},
"description": "Block used to manage Azure Container Instances authentication. Stores Azure Service\nPrincipal authentication data.",
"secret_fields": [
"client_secret"
],
"block_type_slug": "azure-container-instance-credentials",
"block_schema_references": {}
}
},
"description": "Variables for an Azure Container Instance flow run."
},
"job_configuration": {
"cpu": "{{ cpu }}",
"env": "{{ env }}",
"name": "{{ name }}",
"image": "{{ image }}",
"labels": "{{ labels }}",
"memory": "{{ memory }}",
"command": "{{ command }}",
"gpu_sku": "{{ gpu_sku }}",
"gpu_count": "{{ gpu_count }}",
"entrypoint": "{{ entrypoint }}",
"identities": "{{ identities }}",
"subnet_ids": "{{ subnet_ids }}",
"dns_servers": "{{ dns_servers }}",
"arm_template": {
"$schema": "https://schema.management.azure.com/schemas/2019-08-01/deploymentTemplate.json#",
"resources": [
{
"name": "[parameters('container_group_name')]",
"type": "Microsoft.ContainerInstance/containerGroups",
"location": "[parameters('location')]",
"apiVersion": "2022-09-01",
"properties": {
"osType": "Linux",
"containers": [
{
"name": "[parameters('container_name')]",
"properties": {
"image": "{{ image }}",
"command": "{{ command }}",
"resources": {
"requests": {
"cpu": "{{ cpu }}",
"memoryInGB": "{{ memory }}"
}
},
"environmentVariables": []
}
}
],
"restartPolicy": "Never"
}
}
],
"parameters": {
"location": {
"type": "string",
"metadata": {
"description": "Location for all resources."
},
"defaultValue": "[resourceGroup().location]"
},
"container_name": {
"type": "string",
"metadata": {
"description": "The name of the container to create."
},
"defaultValue": "[uniqueString(resourceGroup().id)]"
},
"container_group_name": {
"type": "string",
"metadata": {
"description": "The name of the container group to create."
},
"defaultValue": "[uniqueString(resourceGroup().id)]"
}
},
"contentVersion": "1.0.0.0"
},
"stream_output": "{{ stream_output }}",
"image_registry": "{{ image_registry }}",
"aci_credentials": "{{ aci_credentials }}",
"subscription_id": "{{ subscription_id }}",
"resource_group_name": "{{ resource_group_name }}",
"task_watch_poll_interval": "{{ task_watch_poll_interval }}",
"task_start_timeout_seconds": "{{ task_start_timeout_seconds }}"
}
}
Before the template is used I run it through envsubst to replace all environment variables. And the i run
prefect work-pool create \
--base-job-template infra/worker/aci_job_template_$(ENV).json \
--type azure-container-instance \
$(ENV)-worker