Introduce metrics framework based on telemetry events

config/docker-testval.config.src

@@ -56,14 +56,17 @@
    %% we only really need the params below if this file is changed to specify a radio device
    %% as without one miner_lora is not started
    %% including the params anyway in case someone needs it in this env
-   {region_override, 'US915'}
+   {region_override, 'US915'},
+   {metrics, [block_metrics, txn_metrics, grpc_metrics, snapshot_metrics]},
+   {metrics_port, ${METRICS_PORT:-9090}}
   ]},
  {grpcbox,
   [
    {servers, [#{grpc_opts =>
                     #{
                       service_protos => [gateway_pb],
-                      services => #{'helium.gateway' => helium_gateway_service}
+                      services => #{'helium.gateway' => helium_gateway_service},
+                      stats_handler => grpcbox_telemetry_stats_handler
                      },
 
                 transport_opts => #{ssl => false},

config/docker-val.config.src

@@ -52,14 +52,17 @@
    %% we only really need the params below if this file is changed to specify a radio device
    %% as without one miner_lora is not started
    %% including the params anyway in case someone needs it in this env
-   {region_override, 'US915'}
+   {region_override, 'US915'},
+   {metrics, [block_metrics, txn_metrics, grpc_metrics, snapshot_metrics]},
+   {metrics_port, ${METRICS_PORT:-9090}}
   ]},
  {grpcbox,
   [
    {servers, [#{grpc_opts =>
                     #{
                       service_protos => [gateway_pb],
-                      services => #{'helium.gateway' => helium_gateway_service}
+                      services => #{'helium.gateway' => helium_gateway_service},
+                      stats_handler => grpcbox_telemetry_stats_handler
                      },
 
                 transport_opts => #{ssl => false},

config/sys.config

@@ -23,6 +23,17 @@
      {lager_file_backend, [{file, "error.log"}, {level, error}]}
     ]}
   ]},
+ {prometheus, [
+    {collectors, [
+        prometheus_boolean,
+        prometheus_counter,
+        prometheus_gauge,
+        prometheus_histogram,
+        prometheus_vm_memory_collector,
+        prometheus_vm_system_info_collector
+    ]},
+    {vm_system_info_collector_metrics, [process_count]}
+ ]},
  {libp2p,
   [
    {use_dns_for_seeds, true},

config/test_val.config.src

@@ -63,6 +63,8 @@
    {region_override, 'US915'},
    {gateway_and_mux_enable, false},
    {default_routers, ["${DEFAULT_ROUTERS:-/p2p/11w77YQLhgUt8HUJrMtntGGr97RyXmot1ofs5Ct2ELTmbFoYsQa}"]},
+   {metrics, [block_metrics, txn_metrics, grpc_metrics, snapshot_metrics]},
+   {metrics_port, ${METRICS_PORT:-9090}},
    {seed_validators, [
        {"1ZPNnNd9k5qiQXXigKifQpCPiy5HTbszQDSyLM56ywk7ihNRvt6", "18.223.171.149", 8080},     %% test-val2
        {"1ZYe21WzqJGkWjXvyEt2c8ALSrufPfjzqfQP2SGy61UJd2h9EbL", "3.17.164.253", 8080},       %% test-val3
@@ -75,7 +77,8 @@
    {servers, [#{grpc_opts =>
                     #{
                       service_protos => [gateway_pb],
-                      services => #{'helium.gateway' => helium_gateway_service}
+                      services => #{'helium.gateway' => helium_gateway_service},
+                      stats_handler => grpcbox_telemetry_stats_handler
                      },
 
                 transport_opts => #{ssl => false},

config/testnet-sys.config

@@ -23,6 +23,17 @@
      {lager_file_backend, [{file, "error.log"}, {level, error}]}
     ]}
   ]},
+ {prometheus, [
+    {collectors, [
+        prometheus_boolean,
+        prometheus_counter,
+        prometheus_gauge,
+        prometheus_histogram,
+        prometheus_vm_memory_collector,
+        prometheus_vm_system_info_collector
+    ]},
+    {vm_system_info_collector_metrics, [process_count]}
+ ]},
  {libp2p,
   [
    {use_dns_for_seeds, false},
@@ -151,7 +162,8 @@
             #{
                 grpc_opts => #{
                     service_protos => [gateway_pb],
-                    services => #{'helium.gateway' => helium_gateway_service}
+                    services => #{'helium.gateway' => helium_gateway_service},
+                    stats_handler => grpcbox_telemetry_stats_handler
                 },
 
                 transport_opts => #{ssl => false},

config/val.config.src

@@ -50,6 +50,8 @@
    {sidecar_parallelism_limit, ${SIDECAR_PARALLELISM_LIMIT:-3}},
    {hotfix_dir, "${HOTFIX_DIR:-/opt/miner/hotfix}"},
    {update_dir, "${UPDATE_DIR:-/opt/miner/update}"},
+   {metrics, [block_metrics, txn_metrics, grpc_metrics, snapshot_metrics]},
+   {metrics_port, ${METRICS_PORT:-9090}},
    %% these two now disable all the poc stuff
    {use_ebus, false},
    {radio_device, undefined},
@@ -64,7 +66,8 @@
    {servers, [#{grpc_opts =>
                     #{
                       service_protos => [gateway_pb],
-                      services => #{'helium.gateway' => helium_gateway_service}
+                      services => #{'helium.gateway' => helium_gateway_service},
+                      stats_handler => grpcbox_telemetry_stats_handler
                      },
 
                 transport_opts => #{ssl => false},

include/metrics.hrl

@@ -0,0 +1,76 @@
+-define(METRICS_HISTOGRAM_BUCKETS, [50, 100, 250, 500, 1000, 2000, 5000, 10000, 30000, 60000]).
+
+-define(METRICS_BLOCK_ABSORB, "blockchain_block_absorb_duration").
+-define(METRICS_BLOCK_UNVAL_ABSORB, "blockchain_block_unval_absorb_duration").
+-define(METRICS_BLOCK_HEIGHT, "blockchain_block_height").
+-define(METRICS_BLOCK_UNVAL_HEIGHT, "blockchain_block_unval_height").
+-define(METRICS_TXN_ABSORB_DURATION, "blockchain_txn_absorb_duration").
+-define(METRICS_TXN_BLOCK_SPAN, "blockchain_txn_mgr_block_span").
+-define(METRICS_TXN_QUEUE, "blockchain_txn_mgr_queue").
+-define(METRICS_TXN_SUBMIT_COUNT, "blockchain_txn_mgr_submited_count").
+-define(METRICS_TXN_REJECT_COUNT, "blockchain_txn_mgr_rejected_count").
+-define(METRICS_TXN_ACCEPT_COUNT, "blockchain_txn_mgr_accepted_count").
+-define(METRICS_TXN_UPDATE_COUNT, "blockchain_txn_mgr_updated_count").
+-define(METRICS_TXN_PROCESS_DURATION, "blockchain_txn_mgr_process_duration").
+-define(METRICS_TXN_CACHE_SIZE, "blockchain_txn_mgr_cache_size").
+-define(METRICS_TXN_BLOCK_TIME, "blockchain_txn_mgr_block_time").
+-define(METRICS_TXN_BLOCK_AGE, "blockchain_txn_mgr_block_age").
+-define(METRICS_GRPC_SESSIONS, "grpcbox_session_count").
+-define(METRICS_GRPC_LATENCY, "grpcbox_session_latency").
+-define(METRICS_SC_COUNT, "blockchain_state_channel_count").
+-define(METRICS_SNAP_LOAD_SIZE, "blockchain_snapshot_load_size").
+-define(METRICS_SNAP_LOAD_DURATION, "blockchain_snapshot_load_duration").
+-define(METRICS_SNAP_GEN_SIZE, "blockchain_snapshot_gen_size").
+-define(METRICS_SNAP_GEN_DURATION, "blockchain_snapshot_gen_duration").
+
+-define(METRICS, #{
+    block_metrics => {
+        [ [blockchain, block, absorb],
+          [blockchain, block, height],
+          [blockchain, block, unvalidated_absorb],
+          [blockchain, block, unvalidated_height] ],
+        [ {?METRICS_BLOCK_ABSORB, prometheus_histogram, [stage], "Block absorb duration"},
+          {?METRICS_BLOCK_HEIGHT, prometheus_gauge, [time], "Most recent block height"},
+          {?METRICS_BLOCK_UNVAL_ABSORB, prometheus_histogram, [stage], "Block unvalidated absorb duration"},
+          {?METRICS_BLOCK_UNVAL_HEIGHT, prometheus_gauge, [time], "Most recent unvalidated block height"} ]
+    },
+    txn_metrics => {
+        [ [blockchain, txn, absorb],
+          [blockchain, txn_mgr, submit],
+          [blockchain, txn_mgr, reject],
+          [blockchain, txn_mgr, accept],
+          [blockchain, txn_mgr, update],
+          [blockchain, txn_mgr, process],
+          [blockchain, txn_mgr, add_block] ],
+        [ {?METRICS_TXN_ABSORB_DURATION, prometheus_histogram, [stage], "Txn absorb duration"},
+          {?METRICS_TXN_BLOCK_SPAN, prometheus_gauge, [], "Block span of transactions"},
+          {?METRICS_TXN_QUEUE, prometheus_gauge, [], "Txn manager submission queue length"},
+          {?METRICS_TXN_SUBMIT_COUNT, prometheus_counter, [type], "Count of submitted transactions"},
+          {?METRICS_TXN_REJECT_COUNT, prometheus_counter, [type], "Count of rejected transactions"},
+          {?METRICS_TXN_ACCEPT_COUNT, prometheus_counter, [type], "Count of accepted transactions"},
+          {?METRICS_TXN_UPDATE_COUNT, prometheus_counter, [type], "Count of updated transactions"},
+          {?METRICS_TXN_PROCESS_DURATION, prometheus_histogram, [stage], "Transaction manager cache process duration"},
+          {?METRICS_TXN_CACHE_SIZE, prometheus_gauge, [height], "Transaction manager buffer size"},
+          {?METRICS_TXN_BLOCK_TIME, prometheus_gauge, [height], "Block time observed from the transaction mgr"},
+          {?METRICS_TXN_BLOCK_AGE, prometheus_gauge, [height], "Block age observed from the transaction mgr"} ]
+    },
+    grpc_metrics => {
+        [ [grpcbox, server, rpc_begin],
+          [grpcbox, server, rpc_end] ],
+        [ {?METRICS_GRPC_SESSIONS, prometheus_gauge, [method], "Grpc session count"},
+          {?METRICS_GRPC_LATENCY, prometheus_histogram, [method, status], "Grpc session latency"} ]
+    },
+    state_channel_metrics => {
+        [ [blockchain, state_channel, open],
+          [blockchain, state_channel, close] ],
+        [ {?METRICS_SC_COUNT, prometheus_gauge, [version, id], "Active state channel count"} ]
+    },
+    snapshot_metrics => {
+        [ [blockchain, snapshot, generate],
+          [blockchain, snapshot, load] ],
+        [ {?METRICS_SNAP_GEN_SIZE, prometheus_gauge, [blocks, version], "Generated snapshot byte size"},
+          {?METRICS_SNAP_GEN_DURATION, prometheus_gauge, [blocks, version], "Generated snapshot processing duration"},
+          {?METRICS_SNAP_LOAD_SIZE, prometheus_gauge, [height, hash, version, source], "Loaded snapshot byte size"},
+          {?METRICS_SNAP_LOAD_DURATION, prometheus_gauge, [height, hash, version, source], "Loaded snapshot processing duration"} ]
+    }
+}).

rebar.config

@@ -26,6 +26,8 @@
         {http2_client, {git, "https://github.com/helium/http2_client.git",
                                  {branch, "master"}}},
         recon,
+        {telemetry, "1.1.0"},
+        {prometheus, "4.8.2"},
         {elli, "3.3.0"},
         {jsonrpc2, {git, "https://github.com/novalabsxyz/jsonrpc2-erlang",
                  {branch, "master"}}},

rebar.lock

@@ -165,8 +165,8 @@
   2},
  {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.3.1">>},2},
  {<<"procket">>,{pkg,<<"procket">>,<<"0.9.6">>},3},
- {<<"prometheus">>,{pkg,<<"prometheus">>,<<"4.8.1">>},2},
- {<<"quantile_estimator">>,{pkg,<<"quantile_estimator">>,<<"0.2.1">>},3},
+ {<<"prometheus">>,{pkg,<<"prometheus">>,<<"4.8.2">>},0},
+ {<<"quantile_estimator">>,{pkg,<<"quantile_estimator">>,<<"0.2.1">>},1},
  {<<"ranch">>,{pkg,<<"ranch">>,<<"1.5.0">>},2},
  {<<"rand_compat">>,{pkg,<<"rand_compat">>,<<"0.0.3">>},3},
  {<<"recon">>,{pkg,<<"recon">>,<<"2.5.2">>},0},
@@ -186,7 +186,7 @@
  {<<"small_ints">>,{pkg,<<"small_ints">>,<<"0.1.0">>},4},
  {<<"splicer">>,{pkg,<<"splicer">>,<<"0.5.5">>},2},
  {<<"ssl_verify_fun">>,{pkg,<<"ssl_verify_fun">>,<<"1.1.6">>},2},
- {<<"telemetry">>,{pkg,<<"telemetry">>,<<"1.1.0">>},1},
+ {<<"telemetry">>,{pkg,<<"telemetry">>,<<"1.1.0">>},0},
  {<<"throttle">>,{pkg,<<"lambda_throttle">>,<<"0.2.0">>},2},
  {<<"unicode_util_compat">>,{pkg,<<"unicode_util_compat">>,<<"0.7.0">>},2},
  {<<"vincenty">>,
@@ -229,7 +229,7 @@
  {<<"multihash">>, <<"F084F7C6BEC062F0C0E82AE18CFDC8DAEC8F4FAA4C8E1ACE0B9C676A9323162F">>},
  {<<"parse_trans">>, <<"16328AB840CC09919BD10DAB29E431DA3AF9E9E7E7E6F0089DD5A2D2820011D8">>},
  {<<"procket">>, <<"467014637052CAB030DD32D5269591B82763420C9572F61CD6C5657FE4579777">>},
- {<<"prometheus">>, <<"FA76B152555273739C14B06F09F485CF6D5D301FE4E9D31B7FF803D26025D7A0">>},
+ {<<"prometheus">>, <<"B88F24279DD7A1F512CB090595FF6C88B50AAD0A6B394A4C4983725723DCD834">>},
  {<<"quantile_estimator">>, <<"EF50A361F11B5F26B5F16D0696E46A9E4661756492C981F7B2229EF42FF1CD15">>},
  {<<"ranch">>, <<"F04166F456790FEE2AC1AA05A02745CC75783C2BFB26D39FAF6AEFC9A3D3A58A">>},
  {<<"rand_compat">>, <<"011646BC1F0B0C432FE101B816F25B9BBB74A085713CEE1DAFD2D62E9415EAD3">>},
@@ -268,7 +268,7 @@
  {<<"multihash">>, <<"E73AD5D0099DBFFB4EE429A78436B605AFE2530ED684AB36BB86733AB65707C8">>},
  {<<"parse_trans">>, <<"07CD9577885F56362D414E8C4C4E6BDF10D43A8767ABB92D24CBE8B24C54888B">>},
  {<<"procket">>, <<"037287B8D10A747E54A69BDCE56724ECC8BE12BCB8233AE4D6F885F0ED71E895">>},
- {<<"prometheus">>, <<"6EDFBE928D271C7F657A6F2C46258738086584BD6CAE4A000B8B9A6009BA23A5">>},
+ {<<"prometheus">>, <<"C3ABD6521E52CEC4F0D8ECA603CF214DFC84D8A27AA85946639F1424B8554D98">>},
  {<<"quantile_estimator">>, <<"282A8A323CA2A845C9E6F787D166348F776C1D4A41EDE63046D72D422E3DA946">>},
  {<<"ranch">>, <<"86D40FC42AA47BCB6952DDF1DBFD3DA04B5BA69AFB65C322C99845913250B11F">>},
  {<<"rand_compat">>, <<"CDF7BE2B17308EC245B912C45FE55741F93B6E4F1A24BA6074F7137B0CC09BF4">>},

src/handlers/miner_hbbft_handler.erl

@@ -79,7 +79,7 @@ metadata(Version, Meta, Chain) ->
                                         Infos = blockchain_ledger_snapshot_v1:get_infos(Chain),
                                         case blockchain_ledger_snapshot_v1:snapshot(Ledger, Blocks, Infos) of
                                             {ok, Snapshot} ->
-                                                {ok, {_SnapHeight, SnapHash, SnapSize}} = blockchain:add_snapshot(Snapshot, Chain),
+                                                {ok, {_SnapHeight, SnapHash, _SnapSize}} = blockchain:add_snapshot(Snapshot, Chain),
                                                 lager:info("snapshot hash is ~p", [SnapHash]),
                                                 maps:put(snapshot_hash, SnapHash, ChainMeta0);
                                             _Err ->

src/metrics/miner_metrics_exporter.erl

@@ -0,0 +1,19 @@
+-module(miner_metrics_exporter).
+
+-behaviour(elli_handler).
+
+-include_lib("elli/include/elli.hrl").
+
+-export([handle/2, handle_event/3]).
+
+handle(Req, _Args) ->
+    handle(Req#req.method, elli_request:path(Req), Req).
+
+%% Expose /metrics for Prometheus as a scrape target
+handle('GET', [<<"metrics">>], _Req) ->
+    {ok, [], prometheus_text_format:format()};
+handle(_Verb, _Path, _Req) ->
+    ignore.
+
+handle_event(_Event, _Data, _Args) ->
+    ok.