analytics/configuration/flint-jmx-prometheus.yaml

#****************************************************************
#  Licensed Materials - Property of IBM
#  IBM Cognos Products: Flint
#  
#  (C) Copyright IBM Corp. 2018, 2018
#  
#  US Government Users Restricted Rights - 
#  Use, duplication or disclosure restricted by GSA ADP Schedule 
#  Contract with IBM Corp.
#*****************************************************************

# Configuration for the JMX Prometheus Exporter Java Agent

rules:

  # System CPU load reported by JVM.
  - pattern: "java\\.lang<type=OperatingSystem><>SystemCpuLoad:"
    name: jvm_cpu_system_load
  
  # Process CPU load reported by JVM.
  - pattern: "java\\.lang<type=OperatingSystem><>ProcessCpuLoad:"
    name: jvm_cpu_process_load

  # These come from the master
  # Example: master.aliveWorkers
  - pattern: "metrics<name=master\\.(.*)><>(.*):"
    name: spark_master_$1_$2

  # These come from the worker
  # Example: worker.coresFree
  - pattern: "metrics<name=worker\\.(.*)><>(.*):"
    name: spark_worker_$1_$2

  # These come from the application driver
  # Example: app-20160809000059-0000.driver.DAGScheduler.stage.failedStages
  - pattern: "metrics<name=(.*)\\.driver\\.(DAGScheduler|BlockManager)\\.(.*)><>(.*):"
    name: spark_driver_$2_$3_$4
    labels:
      app_id: "$1"
      
  # These come from the FlintServer running in application driver
  # Example: app-20160809000059-0000.driver.FlintServer.UserMetrics.request.TOUCH_DATASET.duration_millis
  - pattern: "metrics<name=(.*)\\.driver\\.FlintServer\\.UserMetrics\\.request\\.(.*)\\.(.*)><>(.*):"
    name: spark_driver_FlintServer_request_$3_$4
    labels:
      app_id: "$1"
      request_type: "$2"
  - pattern: "metrics<name=(.*)\\.driver\\.FlintServer\\.UserMetrics\\.transport\\.(.*)><>(.*):"
    name: spark_driver_FlintServer_transport_$2_$3
    labels:
      app_id: "$1"
  - pattern: "metrics<name=(.*)\\.driver\\.FlintServer\\.UserMetrics\\.dataset\\.(.*)><>(.*):"
    name: spark_driver_FlintServer_dataset_$2_$3
    labels:
      app_id: "$1"
  - pattern: "metrics<name=(.*)\\.driver\\.FlintServer\\.UserMetrics\\.query\\.(.*)><>(.*):"
    name: spark_driver_FlintServer_query_$2_$3
    labels:
      app_id: "$1"
  - pattern: "metrics<name=(.*)\\.driver\\.FlintServer\\.UserMetrics\\.wlm\\.under_pressure><>(.*):"
    name: spark_driver_FlintServer_wlm_under_pressure_$2
    labels:
      app_id: "$1"
  - pattern: "metrics<name=(.*)\\.driver\\.FlintServer\\.UserMetrics\\.wlm\\.pool\\.(.*)\\.(.*)><>(.*):"
    name: spark_driver_FlintServer_wlm_pool_$3_$4
    labels:
      app_id: "$1"
      permit_pool: "$2"
  - pattern: "metrics<name=(.*)\\.driver\\.FlintServer\\.UserMetrics\\.memory\\.max_off_heap_bytes><>(.*):"
    name: spark_driver_FlintServer_memory_max_off_heap_bytes_$2
    labels:
      app_id: "$1"
  - pattern: "metrics<name=(.*)\\.driver\\.FlintServer\\.UserMetrics\\.memory\\.max_off_heap_storage_bytes><>(.*):"
    name: spark_driver_FlintServer_memory_max_off_heap_storage_bytes_$2
    labels:
      app_id: "$1"
  - pattern: "metrics<name=(.*)\\.driver\\.FlintServer\\.UserMetrics\\.memory\\.max_on_heap_storage_bytes><>(.*):"
    name: spark_driver_FlintServer_memory_max_on_heap_storage_bytes_$2
    labels:
      app_id: "$1"
  - pattern: "metrics<name=(.*)\\.driver\\.FlintServer\\.UserMetrics\\.memory\\.used_storage_bytes><>(.*):"
    name: spark_driver_FlintServer_memory_used_storage_bytes_$2
    labels:
      app_id: "$1"
  - pattern: "metrics<name=(.*)\\.driver\\.FlintServer\\.UserMetrics\\.memory\\.used_execution_bytes><>(.*):"
    name: spark_driver_FlintServer_memory_used_execution_bytes_$2
    labels:
      app_id: "$1"
  - pattern: "metrics<name=(.*)\\.driver\\.FlintServer\\.UserMetrics\\.memory\\.pool\\.(.*)\\.(.*)><>(.*):"
    name: spark_driver_FlintServer_memory_pool_$3_$4
    labels:
      app_id: "$1"
      mem_pool: "$2"

  # These come from the application driver if it's a streaming application
  # Example: app-20160809000059-0000.driver.com.example.ClassName.StreamingMetrics.streaming.lastCompletedBatch_schedulingDelay
  - pattern: "metrics<name=(.*)\\.driver\\.(.*)\\.StreamingMetrics\\.streaming\\.(.*)><>(.*):"
    name: spark_streaming_driver_$3_$4
    labels:
      app_id: "$1"
      app_name: "$2"

  # These come from the application driver if it's a structured streaming application
  # Example: app-20160809000059-0000.driver.spark.streaming.QueryName.inputRate-total
  - pattern: "metrics<name=(.*)\\.driver\\.spark\\.streaming\\.(.*)\\.(.*)><>(.*):"
    name: spark_structured_streaming_driver_$3_$4
    labels:
      app_id: "$1"
      query_name: "$2"

  # These come from the application executors
  # Example: app-20160809000059-0000.0.executor.threadpool.activeTasks
  - pattern: "metrics<name=(.*)\\.(.*)\\.executor\\.(.*)><>(.*):"
    name: spark_executor_$3_$4
    labels:
      app_id: "$1"
      executor_id: "$2"

  # These come from the master
  # Example: application.com.example.ClassName.1470700859054.cores
  - pattern: "metrics<name=application\\.(.*)\\.([0-9]+)\\.(.*)><>(.*):"
    name: spark_application_$3_$4
    labels:
      app_name: "$1"
      app_start_epoch: "$2"