, {dst_host , "localhost"}
, {dst_port , 8125}
, {src_port , 8124}
+ , {num_msgs_per_packet , 10}
]}
, {beam_stats_consumer_graphite,
[ {consumption_interval , 60000}
, context_switches :: non_neg_integer()
, reductions :: non_neg_integer()
, run_queue :: non_neg_integer()
+ , ets :: beam_stats_ets:t()
+
%, statistics :: [{atom() , term()}]
%, system :: [{atom() , term()}]
%, process :: [{atom() , term()}]
%, port :: [{atom() , term()}]
- %, ets :: [{atom() , term()}]
%, dets :: [{atom() , term()}]
}).
--- /dev/null
+-record(beam_stats_ets_table,
+ { id :: beam_stats_ets_table:id()
+ , name :: atom()
+ , size :: non_neg_integer()
+ , memory :: non_neg_integer()
+ }).
{ deps
-, [ {hope, ".*", {git, "https://github.com/ibnfirnas/hope.git", {tag, "3.7.0"}}}
+, [ {hope, ".*", {git, "https://github.com/ibnfirnas/hope.git", {tag, "3.8.0"}}}
]
}.
{application, beam_stats,
[
{description, "Periodic VM stats production and consumption."},
- {vsn, "0.5.2"},
+ {vsn, "0.6.0"},
{registered, []},
{applications,
[ kernel
-module(beam_stats_consumer_statsd).
-include("include/beam_stats.hrl").
+-include("include/beam_stats_ets_table.hrl").
-include("beam_stats_logging.hrl").
-behaviour(beam_stats_consumer).
| {dst_host , inet:ip_address() | inet:hostname()}
| {dst_port , inet:port_number()}
| {src_port , inet:port_number()}
+ | {num_msgs_per_packet , non_neg_integer()}
.
-define(DEFAULT_DST_HOST, "localhost").
, dst_host :: inet:ip_address() | inet:hostname()
, dst_port :: inet:port_number()
, src_port :: inet:port_number()
+ , num_msgs_per_packet :: non_neg_integer()
}).
-type state() ::
DstHost = hope_kv_list:get(Options, dst_host, ?DEFAULT_DST_HOST),
DstPort = hope_kv_list:get(Options, dst_port, ?DEFAULT_DST_PORT),
SrcPort = hope_kv_list:get(Options, src_port, ?DEFAULT_SRC_PORT),
+ NumMsgsPerPacket = hope_kv_list:get(Options, num_msgs_per_packet, 10),
State = #state
{ sock = none
, dst_host = DstHost
, dst_port = DstPort
, src_port = SrcPort
+ , num_msgs_per_packet = NumMsgsPerPacket
},
{ConsumptionInterval, State}.
-spec consume(beam_stats_consumer:queue(), state()) ->
state().
-consume(Q, #state{}=State1) ->
- Payload = beam_stats_queue_to_binary(Q),
- State2 = try_to_connect_if_no_socket(State1),
- try_to_send(State2, Payload).
+consume(Q, #state{num_msgs_per_packet=NumMsgsPerPacket}=State) ->
+ Packets = beam_stats_queue_to_packets(Q, NumMsgsPerPacket),
+ lists:foldl(fun try_to_connect_and_send/2, State, Packets).
-spec terminate(state()) ->
{}.
%% Transport
%% ============================================================================
+-spec try_to_connect_and_send(binary(), state()) ->
+ state().
+try_to_connect_and_send(<<Payload/binary>>, #state{}=State1) ->
+ State2 = try_to_connect_if_no_socket(State1),
+ try_to_send(State2, Payload).
+
-spec try_to_send(state(), binary()) ->
state().
try_to_send(#state{sock=none}=State, _) ->
%% Serialization
%% ============================================================================
--spec beam_stats_queue_to_binary(beam_stats_consumer:queue()) ->
- binary().
-beam_stats_queue_to_binary(Q) ->
- iolist_to_binary([beam_stats_to_bins(B) || B <- queue:to_list(Q)]).
+-spec beam_stats_queue_to_packets(beam_stats_consumer:queue(), non_neg_integer()) ->
+ [binary()].
+beam_stats_queue_to_packets(Q, NumMsgsPerPacket) ->
+ MsgBins = lists:append([beam_stats_to_bins(B) || B <- queue:to_list(Q)]),
+ MsgBinsChucks = hope_list:divide(MsgBins, NumMsgsPerPacket),
+ lists:map(fun erlang:iolist_to_binary/1, MsgBinsChucks).
-spec beam_stats_to_bins(beam_stats:t()) ->
[binary()].
, context_switches = ContextSwitches
, reductions = Reductions
, run_queue = RunQueue
+ , ets = ETS
}
) ->
NodeIDBin = node_id_to_bin(NodeID),
, reductions_to_msg(Reductions)
, run_queue_to_msg(RunQueue)
| memory_to_msgs(Memory)
- ],
+ ]
+ ++ ets_to_msgs(ETS),
Msgs2 = [statsd_msg_add_name_prefix(M, NodeIDBin) || M <- Msgs1],
[statsd_msg_to_bin(M) || M <- Msgs2].
, type = gauge
}.
+-spec ets_to_msgs(beam_stats_ets:t()) ->
+ [statsd_msg()].
+ets_to_msgs(PerTableStats) ->
+ NestedMsgs = lists:map(fun ets_table_to_msgs/1, PerTableStats),
+ lists:append(NestedMsgs).
+
+-spec ets_table_to_msgs(beam_stats_ets_table:t()) ->
+ [statsd_msg()].
+ets_table_to_msgs(#beam_stats_ets_table
+ { id = ID
+ , name = Name
+ , size = Size
+ , memory = Memory
+ }
+) ->
+ IDBin = beam_stats_ets_table:id_to_bin(ID),
+ NameBin = atom_to_binary(Name, latin1),
+ NameAndID = <<NameBin/binary, ".", IDBin/binary>>,
+ SizeMsg =
+ #statsd_msg
+ { name = <<"ets_table.size.", NameAndID/binary>>
+ , value = Size
+ , type = gauge
+ },
+ MemoryMsg =
+ #statsd_msg
+ { name = <<"ets_table.memory.", NameAndID/binary>>
+ , value = Memory
+ , type = gauge
+ },
+ [SizeMsg, MemoryMsg].
+
-spec memory_to_msgs([{atom(), non_neg_integer()}]) ->
[statsd_msg()].
memory_to_msgs(Memory) ->
--- /dev/null
+-module(beam_stats_ets).
+
+-export_type(
+ [ t/0
+ ]).
+
+-export(
+ [ collect/0
+ ]).
+
+-type t() ::
+ [beam_stats_ets_table:t()].
+
+-spec collect() ->
+ t().
+collect() ->
+ lists:map(fun beam_stats_ets_table:of_id/1, ets:all()).
--- /dev/null
+-module(beam_stats_ets_table).
+
+-include("include/beam_stats_ets_table.hrl").
+
+-export_type(
+ [ t/0
+ , id/0
+ ]).
+
+-export(
+ [ of_id/1
+ , id_to_bin/1
+ ]).
+
+-type id() ::
+ atom()
+ | ets:tid()
+ % integer() is just a workaround, to let us mock ets:tid(), which is
+ % opaque, but represented as an integer, without Dialyzer complaining.
+ | integer()
+ .
+
+-type t() ::
+ #?MODULE{}.
+
+-spec of_id(id()) ->
+ t().
+of_id(ID) ->
+ WordSize = erlang:system_info(wordsize),
+ NumberOfWords = ets:info(ID, memory),
+ NumberOfBytes = NumberOfWords * WordSize,
+ #?MODULE
+ { id = ID
+ , name = ets:info(ID, name)
+ , size = ets:info(ID, size)
+ , memory = NumberOfBytes
+ }.
+
+-spec id_to_bin(atom() | ets:tid()) ->
+ binary().
+id_to_bin(ID) when is_atom(ID) ->
+ atom_to_binary(ID, latin1);
+id_to_bin(ID) when is_integer(ID) ->
+ integer_to_binary(ID).
-record(snapshots,
{ memory :: [{atom(), non_neg_integer()}]
, run_queue :: non_neg_integer()
+ , ets :: beam_stats_ets:t()
}).
-type snapshots() ::
#snapshots
{ memory = Memory
, run_queue = RunQueue
+ , ets = ETS
}
, deltas =
#deltas
, context_switches = CurrentContextSwitches - PreviousContextSwitches
, reductions = Reductions
, run_queue = RunQueue
+ , ets = ETS
}.
snapshots_new() ->
#snapshots
{ memory = erlang:memory()
, run_queue = erlang:statistics(run_queue)
+ , ets = beam_stats_ets:collect()
}.
deltas_new() ->
-module(beam_stats_consumer_statsd_SUITE).
-include_lib("beam_stats/include/beam_stats.hrl").
+-include_lib("beam_stats/include/beam_stats_ets_table.hrl").
-export(
[ all/0
%% =============================================================================
t_send(_Cfg) ->
+ ETSTableStatsFoo =
+ #beam_stats_ets_table
+ { id = foo
+ , name = foo
+ , size = 5
+ , memory = 25
+ },
+ ETSTableStatsBar =
+ #beam_stats_ets_table
+ { id = 37
+ , name = bar
+ , size = 8
+ , memory = 38
+ },
+ % TODO: Indent #beam_stats as #beam_stats_ets_table
BEAMStats = #beam_stats
{ timestamp = {1, 2, 3}
, node_id = 'node_foo@host_bar'
- , memory = [{mem_type_foo, 1}]
+ , memory = [{mem_type_foo, 1}, {mem_type_bar, 2}, {mem_type_baz, 3}]
, io_bytes_in = 3
, io_bytes_out = 7
, context_switches = 5
, reductions = 9
, run_queue = 17
+ , ets = [ETSTableStatsFoo, ETSTableStatsBar]
},
ServerPort = 8125,
{ok, ServerSocket} = gen_udp:open(ServerPort, [binary, {active, false}]),
{_, State1} = beam_stats_consumer_statsd:init(Options),
State2 = beam_stats_consumer_statsd:consume(BEAMStatsQ, State1),
{} = beam_stats_consumer_statsd:terminate(State2),
- ResultOfReceive = gen_udp:recv(ServerSocket, 0),
+ ResultOfReceive1 = gen_udp:recv(ServerSocket, 0),
+ {ok, {_, _, PacketReceived1}} = ResultOfReceive1,
+ ResultOfReceive2 = gen_udp:recv(ServerSocket, 0),
+ {ok, {_, _, PacketReceived2}} = ResultOfReceive2,
ok = gen_udp:close(ServerSocket),
- {ok, {_, _, Data}} = ResultOfReceive,
- ct:log("Packet: ~n~s~n", [Data]),
- << "beam_stats.node_foo_host_bar.io.bytes_in:3|g\n"
- , "beam_stats.node_foo_host_bar.io.bytes_out:7|g\n"
- , "beam_stats.node_foo_host_bar.context_switches:5|g\n"
- , "beam_stats.node_foo_host_bar.reductions:9|g\n"
- , "beam_stats.node_foo_host_bar.run_queue:17|g\n"
- , "beam_stats.node_foo_host_bar.memory.mem_type_foo:1|g\n"
- >> = Data.
+ ct:log("PacketReceived1: ~n~s~n", [PacketReceived1]),
+ ct:log("PacketReceived2: ~n~s~n", [PacketReceived2]),
+ PacketsCombined = <<PacketReceived1/binary, PacketReceived2/binary>>,
+ ct:log("PacketsCombined: ~n~s~n", [PacketsCombined]),
+ MsgsExpected =
+ [ <<"beam_stats.node_foo_host_bar.io.bytes_in:3|g">>
+ , <<"beam_stats.node_foo_host_bar.io.bytes_out:7|g">>
+ , <<"beam_stats.node_foo_host_bar.context_switches:5|g">>
+ , <<"beam_stats.node_foo_host_bar.reductions:9|g">>
+ , <<"beam_stats.node_foo_host_bar.run_queue:17|g">>
+ , <<"beam_stats.node_foo_host_bar.memory.mem_type_foo:1|g">>
+ , <<"beam_stats.node_foo_host_bar.memory.mem_type_bar:2|g">>
+ , <<"beam_stats.node_foo_host_bar.memory.mem_type_baz:3|g">>
+ , <<"beam_stats.node_foo_host_bar.ets_table.size.foo.foo:5|g">>
+ , <<"beam_stats.node_foo_host_bar.ets_table.memory.foo.foo:25|g">>
+ , <<"beam_stats.node_foo_host_bar.ets_table.size.bar.37:8|g">>
+ , <<"beam_stats.node_foo_host_bar.ets_table.memory.bar.37:38|g">>
+ ],
+ MsgsReceived = binary:split(PacketsCombined, <<"\n">>, [global, trim]),
+ RemoveExpectedFromReceived =
+ fun (Expected, Received) ->
+ ct:log(
+ "Looking for expected msg ~p in remaining received ~p~n",
+ [Expected, Received]
+ ),
+ true = lists:member(Expected, Received),
+ Received -- [Expected]
+ end,
+ [] = lists:foldl(RemoveExpectedFromReceived, MsgsReceived, MsgsExpected).
home / code / beam_stats.git / commitdiff
