From 5a628bffd214a8dcd2200169164aa3d880ff172d Mon Sep 17 00:00:00 2001 From: Martin Nikov Date: Thu, 18 Sep 2025 15:08:47 +0300 Subject: [PATCH 1/3] config(shells): Add some usefull packages --- shells/default.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/shells/default.nix b/shells/default.nix index 6ba3657a..55bdaeb2 100644 --- a/shells/default.nix +++ b/shells/default.nix @@ -35,9 +35,14 @@ nix-eval-jobs nixos-rebuild nix-output-monitor + curl + openssl + zlib + pkg-config repl rage dub + dub-to-nix ldc ] ++ pkgs.lib.optionals (pkgs.stdenv.system == "x86_64-linux") [ From c2c454dfac00d0d9d970f5319e96d63a41a0dab9 Mon Sep 17 00:00:00 2001 From: Martin Nikov Date: Thu, 18 Sep 2025 15:09:48 +0300 Subject: [PATCH 2/3] feat(packages): Add cachix-deploy-metrics package --- packages/cachix-deploy-metrics/default.nix | 45 ++++ packages/cachix-deploy-metrics/dub.lock.json | 76 ++++++ packages/cachix-deploy-metrics/dub.sdl | 10 + .../cachix-deploy-metrics/dub.selections.json | 23 ++ packages/cachix-deploy-metrics/main.d | 232 ++++++++++++++++++ packages/default.nix | 1 + 6 files changed, 387 insertions(+) create mode 100644 packages/cachix-deploy-metrics/default.nix create mode 100644 packages/cachix-deploy-metrics/dub.lock.json create mode 100644 packages/cachix-deploy-metrics/dub.sdl create mode 100644 packages/cachix-deploy-metrics/dub.selections.json create mode 100644 packages/cachix-deploy-metrics/main.d diff --git a/packages/cachix-deploy-metrics/default.nix b/packages/cachix-deploy-metrics/default.nix new file mode 100644 index 00000000..2b02dba3 --- /dev/null +++ b/packages/cachix-deploy-metrics/default.nix @@ -0,0 +1,45 @@ +{ + lib, + buildDubPackage, + pkgs, + ... +}: +buildDubPackage rec { + pname = "cachix-deploy-metrics"; + version = "unstable"; + src = lib.fileset.toSource { + root = ./.; + fileset = lib.fileset.fileFilter ( + file: + builtins.any file.hasExt [ + "d" + "sdl" + "json" + ] + ) ./.; + }; + dubLock = ./dub.lock.json; # Auto generated with `dub-to-nix`. + dubBuildFlags = [ ]; + + installPhase = '' + runHook preInstall + mkdir -p $out/bin + install -m755 ./build/${pname} $out/bin/${pname} + runHook postInstall + ''; + + nativeBuildInputs = [ + pkgs.pkg-config + pkgs.makeWrapper + ]; + + buildInputs = [ + pkgs.curl + ]; + + postFixup = '' + wrapProgram $out/bin/${pname} + ''; + + meta.mainProgram = pname; +} diff --git a/packages/cachix-deploy-metrics/dub.lock.json b/packages/cachix-deploy-metrics/dub.lock.json new file mode 100644 index 00000000..819c2605 --- /dev/null +++ b/packages/cachix-deploy-metrics/dub.lock.json @@ -0,0 +1,76 @@ +{ + "dependencies": { + "argparse": { + "version": "1.4.1", + "sha256": "1hwpc9y9jqcw849g77pxbz6rvvpslgaxwid1rxpaz4nyhi93fq4g" + }, + "diet-ng": { + "version": "1.8.4", + "sha256": "03c3pmc9w0s3yf1xipmiw5jiy3aifk6bby3z015vvn80s59mk18c" + }, + "during": { + "version": "0.3.0", + "sha256": "1xkfwhvnr67k0vy664sadj0hjifbs0257cbfjw86jp7rvjkn0myi" + }, + "eventcore": { + "version": "0.9.37", + "sha256": "17fs8lsm1l3nf26k9a21jr7chmzs5ryl926nppzh6a1i2k4jmia4" + }, + "mir-linux-kernel": { + "version": "1.2.1", + "sha256": "12i0sa7dnh70vd22cyhymwn0837bxdjcrs3hilnn5cpfxamaqavq" + }, + "openssl": { + "version": "3.4.0", + "sha256": "0y71p03v05v797bkk0b3sbdkncqb5ch3a48nxaly6dxfdz4w247m" + }, + "openssl-static": { + "version": "1.0.5+3.0.8", + "sha256": "0wpqz29yrbbh39g3cwlgd6h6hh1msws7w5baw1kywdkgj761gx2k" + }, + "prometheus2": { + "version": "1.1.0", + "sha256": "0i3r4i5dyfxxlayk1gzbf0w28mpysnwg33khxndp7wwh3vadwyis" + }, + "silly": { + "version": "1.1.1", + "sha256": "0fz7ib715sfk3w69i6xns5pwd4caahvfqjf32v13daxm1ms8xdzz" + }, + "stdx-allocator": { + "version": "2.77.5", + "sha256": "1g8382wr49sjyar0jay8j7y2if7h1i87dhapkgxphnizp24d7kaj" + }, + "taggedalgebraic": { + "version": "1.0.1", + "sha256": "1xwczhidhb4d6lw342fwx8da4vdwryv9gq8p41bh7184300aafl9" + }, + "vibe-container": { + "version": "1.7.0", + "sha256": "04m3yvrhjq231m71yvk0wc4n2y4bwpd7nvivnbvc65sqjxipqpgf" + }, + "vibe-core": { + "version": "2.13.1", + "sha256": "0fcl1n9lsw2gg5flbjv9wimb18jxby40iip2zdzh1zksis2rg9b5" + }, + "vibe-d": { + "version": "0.10.2", + "sha256": "1vb2alzvvlmn4gjycsgcjjj57gv2mh5faci57qvkxhs73x818rc8" + }, + "vibe-http": { + "version": "1.3.1", + "sha256": "0w01qw3ajnwij3pflv77brik0gf5qhla0wkcq8gcj79hj7gkf49c" + }, + "vibe-inet": { + "version": "1.2.0", + "sha256": "11qdm5qyv95flfrdm32xscjsx0khhpin7jvrk9v43hsvyy8lzrvf" + }, + "vibe-serialization": { + "version": "1.1.2", + "sha256": "1mlqd7z5b3qdr7crj7ci76whqrnz2bahndp3fxn5k8zsd1yl294x" + }, + "vibe-stream": { + "version": "1.3.0", + "sha256": "0q8bxi07z46dp7fk1xdfxmcq9n10jfxhq3zbd5p44w0681qgzf7g" + } + } +} diff --git a/packages/cachix-deploy-metrics/dub.sdl b/packages/cachix-deploy-metrics/dub.sdl new file mode 100644 index 00000000..cedc68db --- /dev/null +++ b/packages/cachix-deploy-metrics/dub.sdl @@ -0,0 +1,10 @@ +name "cachix-deploy-metrics" + +targetType "executable" +targetPath "build" + +sourceFiles "main.d" + +dependency "prometheus2" version="~>1.1.0" +dependency "vibe-d" version="~>0.10.2" +dependency "argparse" version=">=1.3.0 <2.0.0" diff --git a/packages/cachix-deploy-metrics/dub.selections.json b/packages/cachix-deploy-metrics/dub.selections.json new file mode 100644 index 00000000..1fd9037d --- /dev/null +++ b/packages/cachix-deploy-metrics/dub.selections.json @@ -0,0 +1,23 @@ +{ + "fileVersion": 1, + "versions": { + "argparse": "1.4.1", + "diet-ng": "1.8.4", + "during": "0.3.0", + "eventcore": "0.9.37", + "mir-linux-kernel": "1.2.1", + "openssl": "3.4.0", + "openssl-static": "1.0.5+3.0.8", + "prometheus2": "1.1.0", + "silly": "1.1.1", + "stdx-allocator": "2.77.5", + "taggedalgebraic": "1.0.1", + "vibe-container": "1.7.0", + "vibe-core": "2.13.1", + "vibe-d": "0.10.2", + "vibe-http": "1.3.1", + "vibe-inet": "1.2.0", + "vibe-serialization": "1.1.2", + "vibe-stream": "1.3.0" + } +} diff --git a/packages/cachix-deploy-metrics/main.d b/packages/cachix-deploy-metrics/main.d new file mode 100644 index 00000000..3298d9e8 --- /dev/null +++ b/packages/cachix-deploy-metrics/main.d @@ -0,0 +1,232 @@ +import prometheus.registry : Registry; +import prometheus.gauge : Gauge; + +import core.thread : Thread; +import core.time : dur; +import std.conv : to; +import std.process : environment; +import std.exception : ErrnoException; +import std.file : readText; +import std.format : format; +import argparse; // Andrey Zherikov's argparse (UDA-based) +import std.json : JSONType, JSONValue, parseJSON; +import std.logger : LogLevel, logf; +import std.string : strip; +import std.datetime : SysTime, Clock; +import vibe.core.args : setCommandLineArgs; +import vibe.d: HTTPServerSettings, URLRouter, listenHTTP, runApplication, HTTPServerRequest, HTTPServerResponse; +import std.net.curl : HTTP, get; + +const string[] CACHIX_DEPLOY_STATES = ["Pending", "InProgress", "Cancelled", "Failed", "Succeeded"]; + +__gshared Gauge statusGauge; +__gshared Gauge indexGauge; +__gshared Gauge startedGauge; +__gshared Gauge finishedGauge; +__gshared Gauge inProgressDurationGauge; +__gshared string gWorkspace; +const string[] FINISHED_KEYS = ["endedOn", "finishedOn", "completedOn"]; + +void promInit() { + statusGauge = new Gauge("cachix_deploy_status", "Status of the last deploy", ["workspace", "agent", "status"]); + statusGauge.register; + indexGauge = new Gauge("cachix_deploy_counter", "Counter/index of deploys.", ["workspace", "agent"]); + indexGauge.register; + startedGauge = new Gauge("cachix_deploy_last_started_time", "Unix time when the last deploy started.", ["workspace", "agent"]); + startedGauge.register; + finishedGauge = new Gauge("cachix_deploy_last_finished_time", "Unix time when the last deploy finished (if any).", ["workspace", "agent"]); + finishedGauge.register; + inProgressDurationGauge = new Gauge("cachix_deploy_in_progress_duration_seconds", "Seconds elapsed for the current in-progress deploy.", ["workspace", "agent"]); + inProgressDurationGauge.register; +} + +void promSetStatus(string agentName, string status, long indexVal) { + auto ws = gWorkspace; + foreach (s; CACHIX_DEPLOY_STATES) { + statusGauge.set(s == status ? 1.0 : 0.0, [ws, agentName, s]); + } + if (indexVal != long.min) { + indexGauge.set(cast(double) indexVal, [ws, agentName]); + } +} + +JSONValue httpGetJson(string url, string authToken) { + logf(LogLevel.trace, "GET %s", url); + auto conn = HTTP(); + conn.connectTimeout = dur!"seconds"(10); + conn.operationTimeout = dur!"seconds"(20); + conn.addRequestHeader("Authorization", "Bearer " ~ authToken); + auto bodyArr = get!(HTTP, char)(url, conn); + auto body = bodyArr.idup; + return parseJSON(body); +} + +private: +bool tryIsoToUnix(string iso, out double outVal) { + try { + auto t = SysTime.fromISOExtString(iso); + outVal = cast(double) t.toUnixTime(); + return true; + } catch (Exception) { + return false; + } +} + +void promSetTimes(string agentName, string startedOn, string finishedOn) { + auto ws = gWorkspace; + double v; + if (startedOn.length && tryIsoToUnix(startedOn, v)) { + startedGauge.set(v, [ws, agentName]); + } + if (finishedOn.length && tryIsoToUnix(finishedOn, v)) { + finishedGauge.set(v, [ws, agentName]); + } +} + +void promSetInProgressDuration(string agentName, string status, string startedOn) { + auto ws = gWorkspace; + if (status == "InProgress") { + double startUnix; + if (startedOn.length && tryIsoToUnix(startedOn, startUnix)) { + auto nowUnix = cast(double) Clock.currTime().toUnixTime(); + auto diff = nowUnix - startUnix; + if (diff < 0) diff = 0; + inProgressDurationGauge.set(diff, [ws, agentName]); + return; + } + } + inProgressDurationGauge.set(0, [ws, agentName]); +} + +void fetchAgentMetrics(string workspace, string authToken, string agentName) { + auto url = format("%s/api/v1/deploy/agent/%s/%s", "https://app.cachix.org", workspace, agentName); + try { + auto data = httpGetJson(url, authToken); + JSONValue last; + if (data.type == JSONType.object && "lastDeployment" in data.object) { + last = data["lastDeployment"]; + } + + string status; + long indexVal = long.min; + string startedOn; + string finishedOn; + + if (last.type == JSONType.object) { + if ("status" in last.object && last["status"].type == JSONType.string) { + status = last["status"].str; + } + if ("index" in last.object && (last["index"].type == JSONType.integer || last["index"].type == JSONType.uinteger)) { + indexVal = last["index"].integer; + } + if ("startedOn" in last.object && last["startedOn"].type == JSONType.string) { + startedOn = last["startedOn"].str; + } + foreach (k; FINISHED_KEYS) { + if (k in last.object && last[k].type == JSONType.string) { + finishedOn = last[k].str; + break; + } + } + } + + promSetStatus(agentName, status, indexVal); + promSetTimes(agentName, startedOn, finishedOn); + promSetInProgressDuration(agentName, status, startedOn); + + auto started = startedOn.length ? startedOn : ""; + auto finished = finishedOn.length ? finishedOn : ""; + auto idx = (indexVal == long.min) ? "" : to!string(indexVal); + logf(LogLevel.trace, "Agent %s startedOn=%s finishedOn=%s index=%s status=%s", agentName, started, finished, idx, (status.length ? status : "")); + } catch (Exception e) { + logf(LogLevel.error, "Error fetching metrics for agent '%s' (%s): %s", agentName, url, e.msg); + } +} + +void scrapeLoop(string workspace, string authToken, string[] agents, int scrapeIntervalSec) { + while (true) { + foreach (agentName; agents) { + fetchAgentMetrics(workspace, authToken, agentName); + } + Thread.sleep(dur!"seconds"(scrapeIntervalSec)); + } +} + +int main(string[] args) { + struct CliArgs { + @(NamedArgument(["port"]) + .Description("Port to listen on (default: 9160)")) + int port = 9160; + + @(NamedArgument(["listen-address"]) + .Description("Address to bind (default: 127.0.0.1)")) + string listenAddress = "127.0.0.1"; + + @(NamedArgument(["scrape-interval"]) + .Description("Scrape interval in seconds (default: 10)")) + int scrapeInterval = 10; + + @(NamedArgument(["auth-token-path"]) + .Description("Path to Cachix auth token (required if CACHIX_AUTH_TOKEN is unset)")) + string tokenPath; + + @(NamedArgument(["workspace"]) + .Description("Cachix workspace name (required)") + .Required()) + string workspace; + + @(NamedArgument(["agent-names", "a"]) + .Description("Agent names (repeatable)") + .Required()) + string[] agents; + } + + CliArgs opts; + auto res = CLI!(Config.init, CliArgs).parseArgs(opts, args.length > 1 ? args[1 .. $] : []); + if (!res) return res.resultCode; + + if (args.length > 0) setCommandLineArgs([args[0]]); + + string authToken = environment.get("CACHIX_AUTH_TOKEN"); + try { + if (!authToken) authToken = readText(opts.tokenPath).strip(); + } catch (Exception e) { + logf(LogLevel.error, "Token file '%s' not found or unreadable.", opts.tokenPath); + return 2; + } + + gWorkspace = opts.workspace; + promInit(); + foreach (agentName; opts.agents) { + foreach (s; CACHIX_DEPLOY_STATES) { + promSetStatus(agentName, s, long.min); + } + } + + auto settings = new HTTPServerSettings; + settings.port = cast(ushort)opts.port; + bool listenSpecified = opts.listenAddress.length != 0; + if (listenSpecified) settings.bindAddresses = [opts.listenAddress]; + + auto router = new URLRouter; + router.get("/metrics", (HTTPServerRequest req, HTTPServerResponse res) { + string buf; + foreach (m; Registry.global.metrics) { + auto snap = m.collect(); + buf ~= snap.encode(); + } + res.writeBody(cast(ubyte[])buf, "text/plain; version=0.0.4; charset=utf-8"); + }); + + if (opts.agents.length) { + auto t = new Thread({ scrapeLoop(opts.workspace, authToken, opts.agents, opts.scrapeInterval); }); + t.isDaemon = true; + t.start(); + } else { + logf(LogLevel.warning, "No --agent-names provided; only /metrics with static counters will be served."); + } + + listenHTTP(settings, router); + runApplication; + return 0; +} diff --git a/packages/default.nix b/packages/default.nix index c73d456b..548634e4 100644 --- a/packages/default.nix +++ b/packages/default.nix @@ -54,6 +54,7 @@ }; packages = { + cachix-deploy-metrics = pkgs.callPackage ./cachix-deploy-metrics { }; lido-withdrawals-automation = pkgs.callPackage ./lido-withdrawals-automation { }; pyroscope = pkgs.callPackage ./pyroscope { }; random-alerts = pkgs.callPackage ./random-alerts { }; From b7ad58dbfa20dc452d7524202e4c77738ccf6614 Mon Sep 17 00:00:00 2001 From: Martin Nikov Date: Thu, 18 Sep 2025 15:10:22 +0300 Subject: [PATCH 3/3] config(modules): Add cachix-deploy-metrics module --- modules/cachix-deploy-metrics/default.nix | 91 +++++++++++++++++++++++ modules/default.nix | 1 + 2 files changed, 92 insertions(+) create mode 100644 modules/cachix-deploy-metrics/default.nix diff --git a/modules/cachix-deploy-metrics/default.nix b/modules/cachix-deploy-metrics/default.nix new file mode 100644 index 00000000..b88932a9 --- /dev/null +++ b/modules/cachix-deploy-metrics/default.nix @@ -0,0 +1,91 @@ +{ withSystem, ... }: +{ + flake.modules.nixos.cachix-deploy-metrics = + { + pkgs, + config, + lib, + ... + }: + let + inherit (lib) + types + mkEnableOption + mkOption + mkIf + concatMapStringsSep + ; + cfg = config.services.cachix-deploy-metrics; + defaultPackage = withSystem pkgs.stdenv.hostPlatform.system ( + { config, ... }: config.packages.cachix-deploy-metrics + ); + in + { + options.services.cachix-deploy-metrics = with lib; { + enable = mkEnableOption (lib.mdDoc "Cachix Deploy Metrics"); + + package = mkOption { + type = types.package; + default = defaultPackage; + description = "Package providing the deploy-metrics binary."; + }; + + scrape-interval = mkOption { + type = types.int; + default = 1; + description = "Scrape interval in seconds."; + }; + + auth-token-path = mkOption { + type = types.path; + description = "Cachix auth token path."; + }; + + workspace = mkOption { + type = types.str; + description = "Cachix workspace."; + }; + + agent-names = mkOption { + type = types.listOf types.str; + default = [ ]; + description = "List of Cachix deploy agents."; + example = [ + "machine-01" + "machine-02" + ]; + }; + + port = mkOption { + type = types.port; + default = 9160; + description = "Port number of Cachix Deployments Exporter service."; + }; + }; + + config = mkIf cfg.enable { + systemd.services.cachix-deploy-metrics = { + description = "Prometheus exporter for Cachix Deploy"; + wantedBy = [ "multi-user.target" ]; + path = [ cfg.package ]; + serviceConfig = { + ExecStart = '' + ${lib.getExe cfg.package} \ + --port ${toString cfg.port} \ + --scrape-interval ${toString cfg.scrape-interval} \ + --auth-token-path ${cfg.auth-token-path} \ + --workspace ${cfg.workspace} \ + ${ + if cfg.agent-names == [ ] then + "" + else + concatMapStringsSep " \\\n" (agent: "--agent-names=${agent}") cfg.agent-names + } + ''; + Restart = "on-failure"; + RestartSec = 10; + }; + }; + }; + }; +} diff --git a/modules/default.nix b/modules/default.nix index 5698129d..014cdbc3 100644 --- a/modules/default.nix +++ b/modules/default.nix @@ -1,5 +1,6 @@ { imports = [ + ./cachix-deploy-metrics ./lido ./pyroscope ./folder-size-metrics