[metrics] Gets Loki and Prometheus set up with Grafana

system/hosts/mcp/configuration.nix

@@ -103,6 +103,10 @@
         enable = true;
         enabledCollectors = [ "systemd" ];
         port = 9002;
+        # Open the firewall, but only listen on the internal address
+        # TODO: Add some form authentication
+        openFirewall = true;
+        listenAddress = "10.88.0.1";
       };
     };
   };

system/hosts/mcp/containers/grafana.nix

@@ -10,8 +10,10 @@ in {
     image = "grafana/grafana-enterprise";
     dependsOn = [
       "db"
+      "loki"
     ];
     hostName = "grafana";
+    domain = lib.blazestar;
     port = 3000;
     homepageOpts = {
       group = "Infra";
@@ -23,7 +25,42 @@ in {
       "grafana-storage:/var/lib/grafana"
     ];
     environment = {
-      GF_SERVER_ROOT_URL = "https://grafna.havenisms.com";
+      GF_SERVER_ROOT_URL = "https://grafana.${lib.blazestar}";
+    };
+  };
+
+  virtualisation.oci-containers.containers.loki = lib.mkContainer {
+    image = "grafana/loki";
+    hostName = "loki";
+    domain = lib.blazestar;
+    port = 3100;
+    homepageOpts = {
+      group = "Infra";
+      icon = "loki.png";
+      name = "Loki";
+      description = "Log Database";
+    };
+    environment = {};
+  };
+
+  services.promtail = {
+    enable = true;
+    configuration = {
+      server.http_listen_port = 9080;
+      clients = [
+        { url = "https://loki.blazestar.net/loki/api/v1/push"; }
+      ];
+      scrape_configs = [{
+        job_name = "journal";
+        journal = {
+          max_age = "12h";
+          labels = { job = "systemd-journal"; };
+        };
+        relabel_configs = [{
+          source_labels = ["__journal__systemd_unit"];
+          target_label = "unit";
+        }];
+      }];
     };
   };
 }

system/hosts/mcp/containers/lib.nix

@@ -5,7 +5,8 @@ let
   hostRule = host: domain: "Host(`${host}.${domain}`)";
   hostRuleHavenisms = host: hostRule host havenisms;
   localNet = "192.168.0.0/16";
-  localNetRule = "ClientIP(`${localNet}`)";
+  dockerNet = "10.88.0.0/16";
+  localNetRule = "(ClientIP(`${localNet}`) || ClientIP(`${dockerNet}`))";
   localHostRule = host: domain: "${localNetRule} && ${hostRule host domain}";
   localHostRuleHavenisms = host: localHostRule host havenisms;
 in 

system/hosts/mcp/containers/prometheus.nix

@@ -1,9 +1,10 @@
-{ config, pkgs, ... }:
-let inherit (import ./lib.nix config) mkContainer; in
+{ config, ... }:
+let inherit (import ./lib.nix config) mkContainer blazestar; in
 {
   virtualisation.oci-containers.containers.prometheus = mkContainer {
     image = "prom/prometheus";
     hostName = "prometheus";
+    domain = blazestar;
     port = 9090;
     volumes = [
       "/tank/config/prometheus:/etc/prometheus"