Setting up custom logging configurations in Kubernetes

Help
, ,
1

This is related to this archived post about suppressing certain logs in Prefect. By following this article I created a logging.yml file that works wonderfully when running Prefect on my local machine. However, when I run the same command in my Kubernetes cluster, no logs are emitted. We are setting it up with a configMap. The classes used to filter logs and set up the console handler are in the module project_tasks.config.logger. I’m wondering if I’m missing something in this set up? Has anyone set up customized logging in Kubernetes successfully? I’ve included the k8s config and the logging.yml file below.

{
    "name": "P{REFECT_LOGGING_SETTINGS_PATH",
    "value": "/etc/prefect/logging/logging.yml"
},
...
{
    "name": "config-volume",
    "configMap": { "name": "prefect-logging-config" }
},
...
{
    "name": "config-volume",
    "mountPath": "/etc/prefect/logging/logging.yml",
    "subPath:": "logging.yml"
}

# Prefect logging config file.
#
# Any item in this file can be overridden with an environment variable:
#    `PREFECT_LOGGING_[PATH]_[TO]_[KEY]=VALUE`
#
# Templated values can be used to insert values from the Prefect settings at runtime.
# Modified based on https://github.com/PrefectHQ/prefect/blob/main/src/prefect/logging/logging.yml

version: 1
disable_existing_loggers: False

formatters:
    simple:
        format: "%(asctime)s.%(msecs)03d | %(message)s"
        datefmt: "%H:%M:%S"

    standard:
        (): prefect.logging.formatters.PrefectFormatter
        format: "%(asctime)s.%(msecs)03d | %(levelname)-7s | %(name)s - %(message)s"
        flow_run_fmt: "%(asctime)s.%(msecs)03d | %(levelname)-7s | Flow run %(flow_run_name)r - %(message)s"
        task_run_fmt: "%(asctime)s.%(msecs)03d | %(levelname)-7s | Task run %(task_run_name)r - %(message)s"
        datefmt: "%H:%M:%S"

    debug:
        format: "%(asctime)s.%(msecs)03d | %(levelname)-7s | %(threadName)-12s | %(name)s - %(message)s"
        datefmt: "%H:%M:%S"

    json:
        class: prefect.logging.formatters.JsonFormatter
        format: "default"

filters:
    remove_created_task_run:
        (): project_tasks.config.logger.RemoveCreatedTaskRun
    remove_executing_task_log:
        (): project_tasks.config.logger.RemoveExecutingTaskRun

handlers:
    project:
        level: 0
        class: project_tasks.config.logger.ProjectConsoleHandler
        formatter: standard
        filters: [remove_created_task_run, remove_executing_task_log]
        styles:
            log.web_url: bright_blue
            log.local_url: bright_blue

            log.debug_level: black
            log.info_level: cyan
            log.warning_level: yellow3
            log.error_level: red3
            log.critical_level: bright_red

            log.completed_state: green
            log.cancelled_state: yellow1
            log.failed_state: red3
            log.crashed_state: bright_red

            log.flow_run_name: magenta
            log.flow_name: bold magenta
            log.task_run_name: dark_magenta
            log.dry_run: "bold black on yellow3"

    api:
        level: 0
        class: prefect.logging.handlers.APILogHandler

    debug:
        level: 0
        class: logging.StreamHandler
        formatter: debug

loggers:
    prefect:
        level: "${PREFECT_LOGGING_LEVEL}"

    prefect.extra:
        level: "${PREFECT_LOGGING_LEVEL}"
        handlers: [project]
        propagate: false

    prefect.flow_runs:
        level: NOTSET
        handlers: [project]
        propagate: false

    prefect.task_runs:
        level: NOTSET
        handlers: [project]
        propagate: false

    prefect.server:
        level: "${PREFECT_LOGGING_SERVER_LEVEL}"

    prefect.client:
        level: "${PREFECT_LOGGING_LEVEL}"

    prefect.infrastructure:
        level: "${PREFECT_LOGGING_LEVEL}"

    prefect._internal:
        level: "${PREFECT_LOGGING_INTERNAL_LEVEL}"
        propagate: false
        handlers: [debug]

root:
    level: WARNING
    handlers: [project]
    propagate: false