test/beam_stats_consumer_statsd_SUITE.erl

   1 -module(beam_stats_consumer_statsd_SUITE).
   2
   3 -include_lib("beam_stats/include/beam_stats.hrl").
   4 -include_lib("beam_stats/include/beam_stats_ets_table.hrl").
   5 -include_lib("beam_stats/include/beam_stats_process.hrl").
   6 -include_lib("beam_stats/include/beam_stats_process_ancestry.hrl").
   7 -include_lib("beam_stats/include/beam_stats_processes.hrl").
   8
   9 -export(
  10     [ all/0
  11     , groups/0
  12     ]).
  13
  14 %% Test cases
  15 -export(
  16     [ t_full_cycle/1
  17     ]).
  18
  19 -define(GROUP, beam_stats_consumer_statsd).
  20
  21 %% ============================================================================
  22 %% Common Test callbacks
  23 %% ============================================================================
  24
  25 all() ->
  26     [{group, ?GROUP}].
  27
  28 groups() ->
  29     Tests =
  30         [ t_full_cycle
  31         ],
  32     Properties = [],
  33     [{?GROUP, Properties, Tests}].
  34
  35 %% ============================================================================
  36 %%  Test cases
  37 %% ============================================================================
  38
  39 t_full_cycle(_Cfg) ->
  40     meck:new(beam_stats_source),
  41     _BEAMStatsExpected = meck_expect_beam_stats(),
  42
  43     {ok,[hope,beam_stats]} = application:ensure_all_started(beam_stats),
  44     ct:log("beam_stats started~n"),
  45     ServerPort = 8125,
  46     {ok, ServerSocket} = gen_udp:open(ServerPort, [binary, {active, false}]),
  47     ct:log("UDP server started started~n"),
  48     {ok, _} = beam_stats_consumer:add(beam_stats_consumer_statsd,
  49         [ {consumption_interval , 60000}
  50         , {dst_host             , "localhost"}
  51         , {dst_port             , ServerPort}
  52         , {src_port             , 8124}
  53         , {num_msgs_per_packet  , 10}
  54         ]
  55     ),
  56     ct:log("consumer added~n"),
  57     _ = meck_expect_beam_stats(
  58         % Double the original values, so that deltas will equal originals after
  59         % 1 update of new beam_stats_state:t()
  60         [ {io_bytes_in      , 6}
  61         , {io_bytes_out     , 14}
  62         , {context_switches , 10}
  63         , {reductions       , 18}
  64         ]
  65     ),
  66     ct:log("meck_expect_beam_stats ok~n"),
  67     {} = beam_stats_producer:sync_produce_consume(),
  68     ct:log("produced and consumed~n"),
  69     ok = application:stop(beam_stats),
  70     ct:log("beam_stats stopped~n"),
  71
  72     ResultOfReceive1 = gen_udp:recv(ServerSocket, 0),
  73     ResultOfReceive2 = gen_udp:recv(ServerSocket, 0),
  74     ResultOfReceive3 = gen_udp:recv(ServerSocket, 0),
  75     ResultOfReceive4 = gen_udp:recv(ServerSocket, 0),
  76     ok = gen_udp:close(ServerSocket),
  77     {ok, {_, _, PacketReceived1}} = ResultOfReceive1,
  78     {ok, {_, _, PacketReceived2}} = ResultOfReceive2,
  79     {ok, {_, _, PacketReceived3}} = ResultOfReceive3,
  80     {ok, {_, _, PacketReceived4}} = ResultOfReceive4,
  81     ct:log("PacketReceived1: ~n~s~n", [PacketReceived1]),
  82     ct:log("PacketReceived2: ~n~s~n", [PacketReceived2]),
  83     ct:log("PacketReceived3: ~n~s~n", [PacketReceived3]),
  84     ct:log("PacketReceived4: ~n~s~n", [PacketReceived4]),
  85     PacketsCombined =
  86         << PacketReceived1/binary
  87          , PacketReceived2/binary
  88          , PacketReceived3/binary
  89          , PacketReceived4/binary
  90         >>,
  91     ct:log("PacketsCombined: ~n~s~n", [PacketsCombined]),
  92     MsgsExpected =
  93         [ <<"beam_stats_v0.node_foo_host_bar.io.bytes_in:3|g">>
  94         , <<"beam_stats_v0.node_foo_host_bar.io.bytes_out:7|g">>
  95         , <<"beam_stats_v0.node_foo_host_bar.context_switches:5|g">>
  96         , <<"beam_stats_v0.node_foo_host_bar.reductions:9|g">>
  97         , <<"beam_stats_v0.node_foo_host_bar.run_queue:17|g">>
  98         , <<"beam_stats_v0.node_foo_host_bar.memory.mem_type_foo:1|g">>
  99         , <<"beam_stats_v0.node_foo_host_bar.memory.mem_type_bar:2|g">>
 100         , <<"beam_stats_v0.node_foo_host_bar.memory.mem_type_baz:3|g">>
 101         , <<"beam_stats_v0.node_foo_host_bar.ets_table.size.foo.NAMED:5|g">>
 102         , <<"beam_stats_v0.node_foo_host_bar.ets_table.memory.foo.NAMED:40|g">>
 103         , <<"beam_stats_v0.node_foo_host_bar.ets_table.size.bar.TID:16|g">>
 104         , <<"beam_stats_v0.node_foo_host_bar.ets_table.memory.bar.TID:128|g">>
 105
 106         % Processes totals
 107         , <<"beam_stats_v0.node_foo_host_bar.processes_count_all:4|g">>
 108         , <<"beam_stats_v0.node_foo_host_bar.processes_count_exiting:0|g">>
 109         , <<"beam_stats_v0.node_foo_host_bar.processes_count_garbage_collecting:0|g">>
 110         , <<"beam_stats_v0.node_foo_host_bar.processes_count_registered:1|g">>
 111         , <<"beam_stats_v0.node_foo_host_bar.processes_count_runnable:0|g">>
 112         , <<"beam_stats_v0.node_foo_host_bar.processes_count_running:3|g">>
 113         , <<"beam_stats_v0.node_foo_host_bar.processes_count_suspended:0|g">>
 114         , <<"beam_stats_v0.node_foo_host_bar.processes_count_waiting:1|g">>
 115
 116         % Process 1
 117         , <<"beam_stats_v0.node_foo_host_bar.process_memory.named--reg_name_foo:15|g">>
 118         , <<"beam_stats_v0.node_foo_host_bar.process_total_heap_size.named--reg_name_foo:25|g">>
 119         , <<"beam_stats_v0.node_foo_host_bar.process_stack_size.named--reg_name_foo:10|g">>
 120         , <<"beam_stats_v0.node_foo_host_bar.process_message_queue_len.named--reg_name_foo:0|g">>
 121         , <<"beam_stats_v0.node_foo_host_bar.process_reductions.named--reg_name_foo:0|g">>
 122
 123         % Process 2
 124         , <<"beam_stats_v0.node_foo_host_bar.process_memory.spawned-via--bar_mod-bar_fun-1--NONE--NONE:25|g">>
 125         , <<"beam_stats_v0.node_foo_host_bar.process_total_heap_size.spawned-via--bar_mod-bar_fun-1--NONE--NONE:35|g">>
 126         , <<"beam_stats_v0.node_foo_host_bar.process_stack_size.spawned-via--bar_mod-bar_fun-1--NONE--NONE:40|g">>
 127         , <<"beam_stats_v0.node_foo_host_bar.process_message_queue_len.spawned-via--bar_mod-bar_fun-1--NONE--NONE:5|g">>
 128         , <<"beam_stats_v0.node_foo_host_bar.process_reductions.spawned-via--bar_mod-bar_fun-1--NONE--NONE:0|g">>
 129
 130         % Process 3 and 4, aggregated by origin
 131         , <<"beam_stats_v0.node_foo_host_bar.process_memory.spawned-via--baz_mod-baz_fun-3--baz_otp_mod-baz_otp_fun-2--PID-PID:30|g">>
 132         , <<"beam_stats_v0.node_foo_host_bar.process_total_heap_size.spawned-via--baz_mod-baz_fun-3--baz_otp_mod-baz_otp_fun-2--PID-PID:45|g">>
 133         , <<"beam_stats_v0.node_foo_host_bar.process_stack_size.spawned-via--baz_mod-baz_fun-3--baz_otp_mod-baz_otp_fun-2--PID-PID:55|g">>
 134         , <<"beam_stats_v0.node_foo_host_bar.process_message_queue_len.spawned-via--baz_mod-baz_fun-3--baz_otp_mod-baz_otp_fun-2--PID-PID:1|g">>
 135         , <<"beam_stats_v0.node_foo_host_bar.process_reductions.spawned-via--baz_mod-baz_fun-3--baz_otp_mod-baz_otp_fun-2--PID-PID:0|g">>
 136         ],
 137     MsgsReceived = binary:split(PacketsCombined, <<"\n">>, [global, trim]),
 138     RemoveExpectedFromReceived =
 139         fun (Expected, Received) ->
 140             ct:log(
 141                 "Looking for expected msg ~p in remaining received ~p~n",
 142                 [Expected, Received]
 143             ),
 144             true = lists:member(Expected, Received),
 145             Received -- [Expected]
 146         end,
 147     MsgsRemaining = lists:foldl(RemoveExpectedFromReceived, MsgsReceived, MsgsExpected),
 148     ct:log("MsgsRemaining: ~p", [MsgsRemaining]),
 149     [] = MsgsRemaining,
 150     meck:unload(beam_stats_source).
 151
 152 meck_expect_beam_stats() ->
 153     meck_expect_beam_stats([]).
 154
 155 meck_expect_beam_stats(Overrides) ->
 156     IOBytesIn       = hope_kv_list:get(Overrides, io_bytes_in , 3),
 157     IOBytesOut      = hope_kv_list:get(Overrides, io_bytes_out, 7),
 158     ContextSwitches = hope_kv_list:get(Overrides, context_switches, 5),
 159     Reductions      = hope_kv_list:get(Overrides, reductions, 9),
 160     Pid0 = list_to_pid("<0.0.0>"),
 161     Pid1 = list_to_pid("<0.1.0>"),
 162     Pid2 = list_to_pid("<0.2.0>"),
 163     Pid3 = list_to_pid("<0.3.0>"),
 164     Pid4 = list_to_pid("<0.4.0>"),
 165     Process1 =
 166         #beam_stats_process
 167         { pid               = Pid1
 168         , registered_name   = {some, reg_name_foo}
 169         , ancestry =
 170               #beam_stats_process_ancestry
 171               { raw_initial_call  = {foo_mod, foo_fun, 2}
 172               , otp_initial_call  = none
 173               , otp_ancestors     = none
 174               }
 175         , status            = running
 176         , memory            = 15
 177         , total_heap_size   = 25
 178         , stack_size        = 10
 179         , message_queue_len = 0
 180         },
 181     Process2 =
 182         #beam_stats_process
 183         { pid               = Pid2
 184         , registered_name   = none
 185         , ancestry =
 186               #beam_stats_process_ancestry
 187               { raw_initial_call  = {bar_mod, bar_fun, 1}
 188               , otp_initial_call  = none
 189               , otp_ancestors     = none
 190               }
 191         , status            = running
 192         , memory            = 25
 193         , total_heap_size   = 35
 194         , stack_size        = 40
 195         , message_queue_len = 5
 196         },
 197     Process3 =
 198         #beam_stats_process
 199         { pid               = Pid3
 200         , registered_name   = none
 201         , ancestry =
 202               #beam_stats_process_ancestry
 203               { raw_initial_call  = {baz_mod, baz_fun, 3}
 204               , otp_initial_call  = {some, {baz_otp_mod, baz_otp_fun, 2}}
 205               , otp_ancestors     = {some, [Pid0, Pid1]}
 206               }
 207         , status            = running
 208         , memory            = 25
 209         , total_heap_size   = 35
 210         , stack_size        = 40
 211         , message_queue_len = 1
 212         },
 213     Process4 =
 214         #beam_stats_process
 215         { pid               = Pid4
 216         , registered_name   = none
 217         , ancestry =
 218               #beam_stats_process_ancestry
 219               { raw_initial_call  = {baz_mod, baz_fun, 3}
 220               , otp_initial_call  = {some, {baz_otp_mod, baz_otp_fun, 2}}
 221               , otp_ancestors     = {some, [Pid0, Pid1]}
 222               }
 223         , status            = waiting
 224         , memory            = 5
 225         , total_heap_size   = 10
 226         , stack_size        = 15
 227         , message_queue_len = 0
 228         },
 229     Processes =
 230         #beam_stats_processes
 231         { individual_stats =
 232             [ Process1
 233             , Process2
 234             , Process3
 235             , Process4
 236             ]
 237         , count_all                = 4
 238         , count_exiting            = 0
 239         , count_garbage_collecting = 0
 240         , count_registered         = 1
 241         , count_runnable           = 0
 242         , count_running            = 3
 243         , count_suspended          = 0
 244         , count_waiting            = 1
 245         },
 246     ETSTableStatsFoo =
 247         #beam_stats_ets_table
 248         { id     = foo
 249         , name   = foo
 250         , size   = 5
 251         , memory = 40
 252         },
 253     ETSTableStatsBarA =
 254         #beam_stats_ets_table
 255         { id     = 37
 256         , name   = bar
 257         , size   = 8
 258         , memory = 64
 259         },
 260     ETSTableStatsBarB =
 261         #beam_stats_ets_table
 262         { id     = 38
 263         , name   = bar
 264         , size   = 8
 265         , memory = 64
 266         },
 267     meck:expect(beam_stats_source, erlang_is_process_alive,
 268         fun (_) -> true end),
 269     meck:expect(beam_stats_source, erlang_memory,
 270         fun () -> [{mem_type_foo, 1}, {mem_type_bar, 2}, {mem_type_baz, 3}] end),
 271     meck:expect(beam_stats_source, erlang_node,
 272         fun () -> 'node_foo@host_bar' end),
 273     meck:expect(beam_stats_source, erlang_registered,
 274         fun () -> [reg_name_foo] end),
 275     meck:expect(beam_stats_source, erlang_statistics,
 276         fun (io              ) -> {{input, IOBytesIn}, {output, IOBytesOut}}
 277         ;   (context_switches) -> {ContextSwitches, 0}
 278         ;   (reductions      ) -> {Reductions, undefined} % 2nd element is unused
 279         ;   (run_queue       ) -> 17
 280         end
 281     ),
 282     meck:expect(beam_stats_source, ets_all,
 283         fun () -> [foo, 37, 38] end),
 284     meck:expect(beam_stats_source, erlang_system_info,
 285         fun (wordsize) -> 8 end),
 286     meck:expect(beam_stats_source, ets_info,
 287         fun (foo, memory) -> 5
 288         ;   (foo, name  ) -> foo
 289         ;   (foo, size  ) -> 5
 290
 291         ;   (37 , memory) -> 8
 292         ;   (37 , name  ) -> bar
 293         ;   (37 , size  ) -> 8
 294
 295         ;   (38 , memory) -> 8
 296         ;   (38 , name  ) -> bar
 297         ;   (38 , size  ) -> 8
 298         end
 299     ),
 300     meck:expect(beam_stats_source, erlang_processes,
 301         fun () -> [Pid1, Pid2, Pid3, Pid4] end),
 302     meck:expect(beam_stats_source, os_timestamp,
 303         fun () -> {1, 2, 3} end),
 304     meck:expect(beam_stats_source, erlang_process_info,
 305         fun (P, K) when P == Pid1 ->
 306                 case K
 307                 of  dictionary        -> {K, []}
 308                 ;   initial_call      -> {K, {foo_mod, foo_fun, 2}}
 309                 ;   registered_name   -> {K, reg_name_foo}
 310                 ;   status            -> {K, running}
 311                 ;   memory            -> {K, 15}
 312                 ;   total_heap_size   -> {K, 25}
 313                 ;   stack_size        -> {K, 10}
 314                 ;   message_queue_len -> {K, 0}
 315                 ;   reductions        -> {K, 1}
 316                 end
 317         ;   (P, K) when P == Pid2 ->
 318                 case K
 319                 of  dictionary        -> {K, []}
 320                 ;   initial_call      -> {K, {bar_mod, bar_fun, 1}}
 321                 ;   registered_name   -> []
 322                 ;   status            -> {K, running}
 323                 ;   memory            -> {K, 25}
 324                 ;   total_heap_size   -> {K, 35}
 325                 ;   stack_size        -> {K, 40}
 326                 ;   message_queue_len -> {K, 5}
 327                 ;   reductions        -> {K, 2}
 328                 end
 329         ;   (P, K) when P == Pid3 ->
 330                 Dict =
 331                     [ {'$initial_call', {baz_otp_mod, baz_otp_fun, 2}}
 332                     , {'$ancestors'   , [Pid0, Pid1]}
 333                     ],
 334                 case K
 335                 of  dictionary        -> {K, Dict}
 336                 ;   initial_call      -> {K, {baz_mod, baz_fun, 3}}
 337                 ;   registered_name   -> []
 338                 ;   status            -> {K, running}
 339                 ;   memory            -> {K, 25}
 340                 ;   total_heap_size   -> {K, 35}
 341                 ;   stack_size        -> {K, 40}
 342                 ;   message_queue_len -> {K, 1}
 343                 ;   reductions        -> {K, 3}
 344                 end
 345         ;   (P, K) when P == Pid4 ->
 346                 Dict =
 347                     [ {'$initial_call', {baz_otp_mod, baz_otp_fun, 2}}
 348                     , {'$ancestors'   , [Pid0, Pid1]}
 349                     ],
 350                 case K
 351                 of  dictionary        -> {K, Dict}
 352                 ;   initial_call      -> {K, {baz_mod, baz_fun, 3}}
 353                 ;   registered_name   -> []
 354                 ;   status            -> {K, waiting}
 355                 ;   memory            -> {K, 5}
 356                 ;   total_heap_size   -> {K, 10}
 357                 ;   stack_size        -> {K, 15}
 358                 ;   message_queue_len -> {K, 0}
 359                 ;   reductions        -> {K, 4}
 360                 end
 361         end
 362     ),
 363     #beam_stats
 364     { timestamp = {1, 2, 3}
 365     , node_id   = 'node_foo@host_bar'
 366     , memory    = [{mem_type_foo, 1}, {mem_type_bar, 2}, {mem_type_baz, 3}]
 367     , io_bytes_in  = IOBytesIn
 368     , io_bytes_out = IOBytesOut
 369     , context_switches = ContextSwitches
 370     , reductions       = 9
 371     , run_queue        = 17
 372     , ets              = [ETSTableStatsFoo, ETSTableStatsBarA, ETSTableStatsBarB]
 373     , processes        = Processes
 374     }.