Understand kube-state-metrics Service
Note:
kube-state-metrics is supported as a beta-only feature.
kube-state-metrics is a third-party metrics monitoring service that generates metrics based on the current state of Kubernetes clusters. You can use kube-state-metrics to monitor the health of Kubernetes cluster and services. kube-state-metrics service is supported as a beta-only feature. kube-state-metrics runs as a cluster service, and is installed automatically when you install Paragon Automation. Once this service is installed, you can enable this service to generate, monitor, and expose metrics of various objects within a Kubernetes cluster.
kube-state-metrics service provides metrics on pods, DaemonSets, deployments, persistent volume, endpoints, ingress, job, lease, and configmap objects that are part of a Kubernetes cluster.
List of Metrics that are Exposed
The following is the list of metrics that are exposed:
- Pods running in a namespace
- Pods that are available
- Information on successful/failed deployments
- State of persistent volumes
- Information on currently running, successful, and failed jobs
- Pods that are in error state
- Health of deployment and DaemonSets
- Status and condition of Kubernetes nodes
Enable kube-state-metrics
You can enable kube-state-metrics from the CLI as well as from the Paragon
Automation UI.
The following is an overview of steps to enable kube-state-metrics from the UI:
-
Create an unmanaged device.
The unmanaged device represents the cluster. The device hostname must be the kube-state-metrics service IP. For example,
kube-state-metrics.healthbot.svc.cluster.local. -
Add the device to a device group.
-
Create rules.
-
Apply playbooks.
Sample Rules and Playbooks
check-daemonset-status.rule
healthbot {
topic kube-metrics {
rule check-daemonset-status {
keys [ daemonset namespace ];
synopsis "";
description "Checks daemon set unavailable status";
sensor daemonset-status {
description "Checks daemon set unavailable status";
server-monitoring {
sensor-name /kube/daemonset;
frequency 60s;
}
}
field daemonset {
sensor daemonset-status {
path daemonset;
}
type string;
description "Checks status for demonset key";
}
field daemonset_status {
sensor daemonset-status {
path /kube/daemonset/status/number/unavailable;
}
type float;
description "Field to check condition";
}
field namespace {
sensor daemonset-status {
path namespace;
}
type string;
description "Checks status for namespace key";
}
trigger daemonset-status {
frequency 1offset;
term available {
when {
equal-to "$daemonset_status" 0;
}
then {
status {
color green;
message "Unavailable demons set is 0 for $namespace $daemonset";
}
}
}
term notavailable {
then {
status {
color red;
message "Unavailable demons set is not 0 for $namespace $daemonset";
}
}
}
}
}
}
}
check-deployment-status-condition.rule
healthbot {
topic kube-metrics {
rule check-deployment-status-condition {
keys [ condition deployment namespace ];
synopsis "";
description "Checks kube metrics deployment status condition";
sensor deployment-status {
description "Checks kube metrics deployment status condition";
server-monitoring {
sensor-name /kube/deployment;
frequency 60s;
}
}
field condition {
sensor deployment-status {
path condition;
}
type string;
description "Deployment condition";
}
field deployment {
sensor deployment-status {
path deployment;
}
type string;
description "Deployment pod name";
}
field namespace {
sensor deployment-status {
path namespace;
}
type string;
description "Deployment namespace";
}
field status {
sensor deployment-status {
path status;
}
type string;
description "Checks for true or false condition";
}
trigger deployment-status {
frequency 1offset;
term available {
when {
matches-with "$status" true;
}
then {
status {
color green;
message "Deployment status for $deployment is $status";
}
}
}
term notavailable {
then {
status {
color red;
message "Deployment status for $deployment is $status";
}
}
}
}
}
}
} check-deployment-status-replicas.rule
healthbot {
topic kube-metrics {
rule check-deployment-status-replicas {
keys [ deployment namespace ];
synopsis "";
description "Checks kube metrics deployment replica status";
sensor deployment-status {
description "Checks kube metrics deployment replica status";
server-monitoring {
sensor-name /kube/deployment;
frequency 60s;
}
}
field deployment {
sensor deployment-status {
path deployment;
}
type string;
description "Deployment pod name";
}
field deployment_status {
sensor deployment-status {
path /kube/deployment/status/replicas;
}
type float;
description "Field to check 0 or other values";
}
field namespace {
sensor deployment-status {
path namespace;
}
type string;
description "Namespace key";
}
trigger deployment-status {
frequency 1offset;
term available {
when {
not-equal-to "$deployment_status" 0;
}
then {
status {
color green;
message "Deployment status for replicaset is $deployment_status for $namespace $deployment";
}
}
}
term notavailable {
then {
status {
color red;
message "Deployment status for replicaset is $deployment_status for $namespace $deployment";
}
}
}
}
}
}
} check-node-status.rule
healthbot {
topic kube-metrics {
rule check-node-status {
keys [ condition node ];
synopsis "";
description "Checks node status";
sensor node-status {
description "Checks node status";
server-monitoring {
sensor-name /kube/node;
frequency 60s;
}
}
field condition {
sensor node-status {
path condition;
}
type string;
description "Node condition";
}
field node {
sensor node-status {
path node;
}
type string;
description "Node name";
}
field node_status {
constant {
value "{{value}}";
}
type integer;
description "Field to check condition";
}
field status {
sensor node-status {
path status;
}
type string;
description "Status of the node";
}
trigger node-status {
frequency 1offset;
term available {
when {
matches-with "$status" false {
ignore-case;
}
}
then {
status {
color green;
message "$condition for $node is $status";
}
}
}
term notavailable {
then {
status {
color red;
message "$condition for $node is $status.";
}
}
}
}
variable value {
value 0;
description "Variable to match true(0) condition.";
type int;
}
}
}
} check-pod-container-restarts.rule
healthbot {
topic kube-metrics {
rule check-pod-container-restarts {
keys [ container namespace pod uid ];
synopsis "";
description "Checks pod container status restarts";
sensor pod-init-container-status {
description "Checks pod container status restarts";
server-monitoring {
sensor-name /kube/pod/container;
frequency 60s;
}
}
field container {
sensor pod-init-container-status {
path container;
}
type string;
description "container name";
}
field namespace {
sensor pod-init-container-status {
path namespace;
}
type string;
description "namespace of the pod";
}
field pod {
sensor pod-init-container-status {
path pod;
}
type string;
description "pod name";
}
field restart-value {
sensor pod-init-container-status {
path /kube/pod/container/status/restarts/total;
}
type integer;
description "restart status value";
}
field uid {
sensor pod-init-container-status {
path uid;
}
type string;
description "Id ";
}
trigger restart-total-status {
frequency 1offset;
term less-than-ten {
when {
less-than-or-equal-to "$restart-value" 10;
}
then {
status {
color green;
message "$namespace pod $pod container $container restart total is $restart-value ";
}
}
}
term between-ten-and-twenty {
when {
range "$restart-value" {
min 10;
max 20;
}
}
then {
status {
color yellow;
message "$namespace pod $pod container $container restart total is $restart-value ";
}
}
}
term more-than-twenty {
then {
status {
color red;
message "$namespace pod $pod container $container restart total is $restart-value";
}
}
}
}
}
}
} check-pod-init-container-status.rule
healthbot {
topic kube-metrics {
rule check-pod-init-container-status {
keys [ container namespace pod uid ];
synopsis "";
description "Checks pod init container status waiting";
sensor pod-init-container-status {
description "Checks pod init container status waiting";
server-monitoring {
sensor-name /kube/pod/init/container;
frequency 60s;
}
}
field container {
sensor pod-init-container-status {
path container;
}
type string;
description "container name";
}
field namespace {
sensor pod-init-container-status {
path namespace;
}
type string;
description "namespace of the pod";
}
field pod {
sensor pod-init-container-status {
path pod;
}
type string;
description "pod name";
}
field status {
sensor pod-init-container-status {
path /kube/pod/init/container/status/waiting;
}
type integer;
description "Statius value (0/1)";
}
field uid {
sensor pod-init-container-status {
path uid;
}
type string;
description "Id ";
}
trigger waiting-status {
frequency 1offset;
term matches-zero {
when {
equal-to "$status" 0 {
time-range 10offset;
}
}
then {
status {
color green;
message "$namespace $pod $container status is $status";
}
}
}
term is-not-zero {
then {
status {
color red;
message "$namespace $pod $container status is $status";
}
}
}
}
}
}
} kube-metrics.playbook
healthbot {
playbook kube-metrics {
rules [ kube-metrics/check-daemonset-status kube-metrics/check-deployment-status-replicas kube-metrics/check-node-status kube-metrics/check-pod-container-restarts kube-metrics/check-pod-init-container-status kube-metrics/check-deployment-status-condition ];
description "Rules to check for kube-metrics ";
synopsis "Rules to check for kube-metrics ";
}
}