{
  config,
  lib,
  pkgs,
  ...
}:

let
  vars = import ./vars.nix;
  loki_storage = vars.primary_loki;
in
{
  # loki: port 3030 (8030)
  #
  services = {
    loki = {
      enable = true;
      configuration = {
        server.http_listen_port = 3030;
        # auth_enabled = false;

        ingester = {
          lifecycler = {
            address = "127.0.0.1";
            ring = {
              kvstore = {
                store = "inmemory";
              };
              replication_factor = 1;
            };
          };
          chunk_idle_period = "1h";
          max_chunk_age = "1h";
          chunk_target_size = 999999;
          chunk_retain_period = "30s";
          max_transfer_retries = 0;
        };

        schema_config = {
          configs = [
            {
              from = "2023-07-01";
              store = "tsdb";
              object_store = "aws";
              schema = "v13";
              index = {
                prefix = "index_";
                period = "24h";
              };
            }
          ];
        };

        storage_config = {
          tsdb_shipper = {
            active_index_directory = "${loki_storage}/boltdb-shipper-active";
            cache_location = "${loki_storage}/boltdb-shipper-cache";
            cache_ttl = "24h";
            shared_store = "filesystem";
          };

          aws = {
            directory = "${loki_storage}/chunks";
            s3 = "s3://access_key:\${LOKI_S3_KEY}@custom_endpoint/bucket_name";
          };
        };

        limits_config = {
          reject_old_samples = true;
          reject_old_samples_max_age = "168h";
        };

        chunk_store_config = {
          max_look_back_period = "0s";
        };

        table_manager = {
          retention_deletes_enabled = false;
          retention_period = "0s";
        };

        compactor = {
          working_directory = loki_storage;
          shared_store = "filesystem";
          compactor_ring = {
            kvstore = {
              store = "inmemory";
            };
          };
        };
      };
      # user, group, dataDir, extraFlags, (configFile)
    };

    # promtail: port 3031 (8031)
    #
    promtail = {
      enable = true;
      configuration = {
        server = {
          http_listen_port = 3031;
          grpc_listen_port = 0;
        };
        positions = {
          filename = "/tmp/positions.yaml";
        };
        clients = [
          {
            url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
          }
        ];
        scrape_configs = [
          {
            job_name = "journal";
            journal = {
              max_age = "12h";
              labels = {
                job = "systemd-journal";
                host = "pihole";
              };
            };
            relabel_configs = [
              {
                source_labels = [ "__journal__systemd_unit" ];
                target_label = "unit";
              }
            ];
          }
        ];
      };
      # extraFlags
    };

    # grafana: port 3010 (8010)
    #
    grafana = {
      port = 3010;
      # WARNING: this should match nginx setup!
      # prevents "Request origin is not authorized"
      rootUrl = "http://192.168.1.10:8010"; # helps with nginx / ws / live

      protocol = "http";
      addr = "127.0.0.1";
      analytics.reporting.enable = false;
      enable = true;

      provision = {
        enable = true;
        datasources = [
          {
            name = "Prometheus";
            type = "prometheus";
            access = "proxy";
            url = "http://127.0.0.1:${toString config.services.prometheus.port}";
          }
          {
            name = "Loki";
            type = "loki";
            access = "proxy";
            url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}";
          }
        ];
      };
    };
  };

  /*
    # nginx reverse proxy
    services.nginx = {
      enable = true;
      recommendedProxySettings = true;
      recommendedOptimisation = true;
      recommendedGzipSettings = true;
      # recommendedTlsSettings = true;

      upstreams = {
        "grafana" = {
          servers = {
            "127.0.0.1:${toString config.services.grafana.port}" = {};
          };
        };
        "prometheus" = {
          servers = {
            "127.0.0.1:${toString config.services.prometheus.port}" = {};
          };
        };
        "loki" = {
          servers = {
            "127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}" = {};
          };
        };
        "promtail" = {
          servers = {
            "127.0.0.1:${toString config.services.promtail.configuration.server.http_listen_port}" = {};
          };
        };
      };

      virtualHosts.grafana = {
        locations."/" = {
          proxyPass = "http://grafana";
          proxyWebsockets = true;
        };
        listen = [{
          addr = "192.168.1.10";
          port = 8010;
        }];
      };

      virtualHosts.prometheus = {
        locations."/".proxyPass = "http://prometheus";
        listen = [{
          addr = "192.168.1.10";
          port = 8020;
        }];
      };

      # confirm with http://192.168.1.10:8030/loki/api/v1/status/buildinfo
      #     (or)     /config /metrics /ready
      virtualHosts.loki = {
        locations."/".proxyPass = "http://loki";
        listen = [{
          addr = "192.168.1.10";
          port = 8030;
        }];
      };

      virtualHosts.promtail = {
        locations."/".proxyPass = "http://promtail";
        listen = [{
          addr = "192.168.1.10";
          port = 8031;
        }];
      };
    };
  */

  systemd.services.loki.serviceConfig.environmentFile = config.sops.secrets."minio/loki".path;
  sops.secrets = {
    "minio/loki".owner = "root";
  };
}