// ============================================================================ // Grafana Alloy — the ONLY monitoring agent on a Linux host. // // Collects metrics (node_exporter, built into Alloy) and logs (journal + // /var/log), and PUSHES both over HTTPS to the monitoring server. Nothing // listens on this host; only outbound 443 is needed. // // Set by install.sh in /etc/default/alloy: // NODE_NAME this host's name (from its enrollment) // INGEST_URL https:// // Push token: /etc/alloy/monitor-token (root:alloy 0640), from enrollment. // ============================================================================ logging { level = "info" format = "logfmt" } // ---- metrics ------------------------------------------------------------------ // node_exporter, built in. Same metric names as the standalone exporter, so // the dashboards and node.rules.yml apply unchanged. prometheus.exporter.unix "node" { enable_collectors = ["systemd", "processes"] filesystem { mount_points_exclude = "^/(sys|proc|dev|run)($|/)" } // Network interfaces: keep the real ones (eth*/en*, bonds, VLANs) and // WireGuard (wg*). Skip loopback and container/virtual plumbing — Docker // creates a bridge per network and a veth per container (dozens on a // Docker host, ~35 series each, renamed on every restart), and that // traffic is already counted on the real interface. netdev { device_exclude = "^(lo|docker[0-9]+|br-.+|veth.+|virbr.+|vnet[0-9]+|cni.+|flannel.+|cali.+|vxlan.+|tunl[0-9]+|kube-.+)$" } netclass { ignored_devices = "^(lo|docker[0-9]+|br-.+|veth.+|virbr.+|vnet[0-9]+|cni.+|flannel.+|cali.+|vxlan.+|tunl[0-9]+|kube-.+)$" } } // Label the series like every other host: job="node", instance=. discovery.relabel "node" { targets = prometheus.exporter.unix.node.targets rule { target_label = "job" replacement = "node" } rule { target_label = "instance" replacement = sys.env("NODE_NAME") } } prometheus.scrape "node" { targets = discovery.relabel.node.output scrape_interval = "30s" forward_to = [prometheus.relabel.trim.receiver] } // The systemd collector reports five state series per unit (most of a host's // series). Only "failed" is used (alerts + dashboard), so keep just that. prometheus.relabel "trim" { forward_to = [prometheus.remote_write.ingest.receiver] rule { source_labels = ["__name__", "state"] regex = "node_systemd_unit_state;(activating|active|deactivating|inactive)" action = "drop" } } // Buffered in a write-ahead log: if the server is unreachable, metrics are // sent once it is back. prometheus.remote_write "ingest" { endpoint { url = sys.env("INGEST_URL") + "/api/v1/write" bearer_token_file = "/etc/alloy/monitor-token" } } // ---- logs --------------------------------------------------------------------- loki.write "default" { endpoint { url = sys.env("INGEST_URL") + "/loki/api/v1/push" bearer_token_file = "/etc/alloy/monitor-token" } external_labels = { host = sys.env("NODE_NAME"), } } // systemd journal (primary log source on modern Ubuntu/Debian). loki.source.journal "journal" { max_age = "12h" labels = { job = "systemd-journal" } forward_to = [loki.relabel.journal.receiver] } // Promote a couple of useful journal fields to Loki labels. loki.relabel "journal" { forward_to = [loki.write.default.receiver] rule { source_labels = ["__journal__systemd_unit"] target_label = "unit" } rule { source_labels = ["__journal_priority_keyword"] target_label = "level" } } // Classic text logs, if the host still writes them (rsyslog). local.file_match "system" { path_targets = [ { __path__ = "/var/log/syslog", job = "syslog" }, { __path__ = "/var/log/auth.log", job = "auth" }, ] } loki.source.file "system" { targets = local.file_match.system.targets forward_to = [loki.write.default.receiver] }