From: Siraaj Khandkar Date: Sun, 20 Sep 2015 00:50:27 +0000 (-0400) Subject: Merge pull request #9 from ibnfirnas/test-full-produce-consume-cycle X-Git-Tag: 0.8.1^0 X-Git-Url: https://git.xandkar.net/?p=beam_stats.git;a=commitdiff_plain;h=c2d1e9edd58faa8bd5668a08d3cf9c2ff8f22ef4;hp=9ddb08ce8505f0cb3eadb9176cce1cf56da9e692 Merge pull request #9 from ibnfirnas/test-full-produce-consume-cycle Test full produce consume cycle --- diff --git a/rebar.config b/rebar.config index f1816dc..5eb75d1 100644 --- a/rebar.config +++ b/rebar.config @@ -1,5 +1,6 @@ { deps , [ {hope, ".*", {git, "https://github.com/ibnfirnas/hope.git", {tag, "3.8.0"}}} + , {meck, ".*", {git, "https://github.com/eproxus/meck.git" , {tag, "0.8.3"}}} ] }. diff --git a/src/beam_stats.app.src b/src/beam_stats.app.src index 5f7b4cf..1597668 100644 --- a/src/beam_stats.app.src +++ b/src/beam_stats.app.src @@ -1,7 +1,7 @@ {application, beam_stats, [ {description, "Periodic VM stats production and consumption."}, - {vsn, "0.8.0"}, + {vsn, "0.8.1"}, {registered, []}, {applications, [ kernel diff --git a/src/beam_stats_ets.erl b/src/beam_stats_ets.erl index 3965765..cfc0606 100644 --- a/src/beam_stats_ets.erl +++ b/src/beam_stats_ets.erl @@ -14,4 +14,5 @@ -spec collect() -> t(). collect() -> - lists:map(fun beam_stats_ets_table:of_id/1, ets:all()). + TableIDs = beam_stats_source:ets_all(), + lists:map(fun beam_stats_ets_table:of_id/1, TableIDs). diff --git a/src/beam_stats_ets_table.erl b/src/beam_stats_ets_table.erl index b603656..622c0f2 100644 --- a/src/beam_stats_ets_table.erl +++ b/src/beam_stats_ets_table.erl @@ -26,13 +26,13 @@ -spec of_id(id()) -> t(). of_id(ID) -> - WordSize = erlang:system_info(wordsize), - NumberOfWords = ets:info(ID, memory), + WordSize = beam_stats_source:erlang_system_info(wordsize), + NumberOfWords = beam_stats_source:ets_info(ID, memory), NumberOfBytes = NumberOfWords * WordSize, #?MODULE { id = ID - , name = ets:info(ID, name) - , size = ets:info(ID, size) + , name = beam_stats_source:ets_info(ID, name) + , size = beam_stats_source:ets_info(ID, size) , memory = NumberOfBytes }. diff --git a/src/beam_stats_process.erl b/src/beam_stats_process.erl index ab6d1c2..e2751ea 100644 --- a/src/beam_stats_process.erl +++ b/src/beam_stats_process.erl @@ -119,7 +119,7 @@ pid_info_exn(Pid, Key) -> Value. pid_info_opt(Pid, Key) -> - case {Key, erlang:process_info(Pid, Key)} + case {Key, beam_stats_source:erlang_process_info(Pid, Key)} of {registered_name, []} -> none ; {_ , {Key, Value}} -> {some, Value} end. diff --git a/src/beam_stats_processes.erl b/src/beam_stats_processes.erl index 2505178..90c3886 100644 --- a/src/beam_stats_processes.erl +++ b/src/beam_stats_processes.erl @@ -21,7 +21,8 @@ -spec collect() -> t(). collect() -> - Ps = [beam_stats_process:of_pid(P) || P <- erlang:processes()], + Pids = beam_stats_source:erlang_processes(), + Ps = [beam_stats_process:of_pid(P) || P <- Pids], ?T { individual_stats = Ps @@ -32,7 +33,7 @@ collect() -> , count_garbage_collecting = length([P || P <- Ps, P#beam_stats_process.status =:= garbage_collecting]) , count_registered - = length(registered()) + = length(beam_stats_source:erlang_registered()) , count_runnable = length([P || P <- Ps, P#beam_stats_process.status =:= runnable]) , count_running diff --git a/src/beam_stats_source.erl b/src/beam_stats_source.erl new file mode 100644 index 0000000..eb0d7d1 --- /dev/null +++ b/src/beam_stats_source.erl @@ -0,0 +1,44 @@ +-module(beam_stats_source). + +-export( + [ erlang_memory/0 + , erlang_node/0 + , erlang_process_info/2 + , erlang_processes/0 + , erlang_registered/0 + , erlang_statistics/1 + , erlang_system_info/1 + , ets_all/0 + , ets_info/2 + , os_timestamp/0 + ]). + +erlang_memory() -> + erlang:memory(). + +erlang_node() -> + erlang:node(). + +erlang_process_info(Pid, Key) -> + erlang:process_info(Pid, Key). + +erlang_processes() -> + erlang:processes(). + +erlang_registered() -> + erlang:registered(). + +erlang_statistics(Key) -> + erlang:statistics(Key). + +erlang_system_info(Key) -> + erlang:system_info(Key). + +ets_all() -> + ets:all(). + +ets_info(Table, Key) -> + ets:info(Table, Key). + +os_timestamp() -> + os:timestamp(). diff --git a/src/beam_stats_state.erl b/src/beam_stats_state.erl index ad424d5..d514f95 100644 --- a/src/beam_stats_state.erl +++ b/src/beam_stats_state.erl @@ -64,8 +64,8 @@ new() -> t(). new(#totals{}=TotalsPrevious) -> ?T - { timestamp = os:timestamp() - , node_id = erlang:node() + { timestamp = beam_stats_source:os_timestamp() + , node_id = beam_stats_source:erlang_node() , snapshots = snapshots_new() , deltas = deltas_new() , totals_previous = TotalsPrevious @@ -123,14 +123,14 @@ export( snapshots_new() -> #snapshots - { memory = erlang:memory() + { memory = beam_stats_source:erlang_memory() , processes = beam_stats_processes:collect() - , run_queue = erlang:statistics(run_queue) + , run_queue = beam_stats_source:erlang_statistics(run_queue) , ets = beam_stats_ets:collect() }. deltas_new() -> - {_ReductionsTotal, ReductionsDelta} = erlang:statistics(reductions), + {_ReductionsTotal, ReductionsDelta} = beam_stats_source:erlang_statistics(reductions), #deltas { reductions = ReductionsDelta }. @@ -138,8 +138,8 @@ deltas_new() -> totals_new() -> { {input , IOBytesIn} , {output , IOBytesOut} - } = erlang:statistics(io), - {ContextSwitches, 0} = erlang:statistics(context_switches), + } = beam_stats_source:erlang_statistics(io), + {ContextSwitches, 0} = beam_stats_source:erlang_statistics(context_switches), #totals { io_bytes_in = IOBytesIn , io_bytes_out = IOBytesOut diff --git a/test/beam_stats_consumer_statsd_SUITE.erl b/test/beam_stats_consumer_statsd_SUITE.erl index ffd0c46..33d78c9 100644 --- a/test/beam_stats_consumer_statsd_SUITE.erl +++ b/test/beam_stats_consumer_statsd_SUITE.erl @@ -13,7 +13,7 @@ %% Test cases -export( - [ t_send/1 + [ t_full_cycle/1 ]). -define(GROUP, beam_stats_consumer_statsd). @@ -27,7 +27,7 @@ all() -> groups() -> Tests = - [ t_send + [ t_full_cycle ], Properties = [], [{?GROUP, Properties, Tests}]. @@ -36,7 +36,124 @@ groups() -> %% Test cases %% ============================================================================ -t_send(_Cfg) -> +t_full_cycle(_Cfg) -> + meck:new(beam_stats_source), + BEAMStatsExpected = meck_expect_beam_stats(), + BEAMStatsComputed = beam_stats_state:export(beam_stats_state:new()), + ct:log("BEAMStatsExpected: ~p~n", [BEAMStatsExpected]), + ct:log("BEAMStatsComputed: ~p~n", [BEAMStatsComputed]), + BEAMStatsExpected = BEAMStatsComputed, + + {ok,[hope,beam_stats]} = application:ensure_all_started(beam_stats), + ct:log("beam_stats started~n"), + ServerPort = 8125, + {ok, ServerSocket} = gen_udp:open(ServerPort, [binary, {active, false}]), + ct:log("UDP server started started~n"), + {ok, _} = beam_stats_consumer:add(beam_stats_consumer_statsd, + [ {consumption_interval , 60000} + , {dst_host , "localhost"} + , {dst_port , ServerPort} + , {src_port , 8124} + , {num_msgs_per_packet , 10} + ] + ), + ct:log("consumer added~n"), + _ = meck_expect_beam_stats( + % Double the original values, so that deltas will equal originals after + % 1 update of new beam_stats_state:t() + [ {io_bytes_in , 6} + , {io_bytes_out , 14} + , {context_switches , 10} + ] + ), + ct:log("meck_expect_beam_stats ok~n"), + {} = beam_stats_producer:sync_produce_consume(), + ct:log("produced and consumed~n"), + ok = application:stop(beam_stats), + ct:log("beam_stats stopped~n"), + + ResultOfReceive1 = gen_udp:recv(ServerSocket, 0), + ResultOfReceive2 = gen_udp:recv(ServerSocket, 0), + ResultOfReceive3 = gen_udp:recv(ServerSocket, 0), + ResultOfReceive4 = gen_udp:recv(ServerSocket, 0), + ok = gen_udp:close(ServerSocket), + {ok, {_, _, PacketReceived1}} = ResultOfReceive1, + {ok, {_, _, PacketReceived2}} = ResultOfReceive2, + {ok, {_, _, PacketReceived3}} = ResultOfReceive3, + {ok, {_, _, PacketReceived4}} = ResultOfReceive4, + ct:log("PacketReceived1: ~n~s~n", [PacketReceived1]), + ct:log("PacketReceived2: ~n~s~n", [PacketReceived2]), + ct:log("PacketReceived3: ~n~s~n", [PacketReceived3]), + ct:log("PacketReceived4: ~n~s~n", [PacketReceived4]), + PacketsCombined = + << PacketReceived1/binary + , PacketReceived2/binary + , PacketReceived3/binary + , PacketReceived4/binary + >>, + ct:log("PacketsCombined: ~n~s~n", [PacketsCombined]), + MsgsExpected = + [ <<"beam_stats.node_foo_host_bar.io.bytes_in:3|g">> + , <<"beam_stats.node_foo_host_bar.io.bytes_out:7|g">> + , <<"beam_stats.node_foo_host_bar.context_switches:5|g">> + , <<"beam_stats.node_foo_host_bar.reductions:9|g">> + , <<"beam_stats.node_foo_host_bar.run_queue:17|g">> + , <<"beam_stats.node_foo_host_bar.memory.mem_type_foo:1|g">> + , <<"beam_stats.node_foo_host_bar.memory.mem_type_bar:2|g">> + , <<"beam_stats.node_foo_host_bar.memory.mem_type_baz:3|g">> + , <<"beam_stats.node_foo_host_bar.ets_table.size.foo.foo:5|g">> + , <<"beam_stats.node_foo_host_bar.ets_table.memory.foo.foo:40|g">> + , <<"beam_stats.node_foo_host_bar.ets_table.size.bar.37:8|g">> + , <<"beam_stats.node_foo_host_bar.ets_table.memory.bar.37:64|g">> + + % Processes totals + , <<"beam_stats.node_foo_host_bar.processes_count_all:3|g">> + , <<"beam_stats.node_foo_host_bar.processes_count_exiting:0|g">> + , <<"beam_stats.node_foo_host_bar.processes_count_garbage_collecting:0|g">> + , <<"beam_stats.node_foo_host_bar.processes_count_registered:1|g">> + , <<"beam_stats.node_foo_host_bar.processes_count_runnable:0|g">> + , <<"beam_stats.node_foo_host_bar.processes_count_running:3|g">> + , <<"beam_stats.node_foo_host_bar.processes_count_suspended:0|g">> + , <<"beam_stats.node_foo_host_bar.processes_count_waiting:0|g">> + + % Process 1 + , <<"beam_stats.node_foo_host_bar.process_memory.reg_name_foo.0_1_0:15|g">> + , <<"beam_stats.node_foo_host_bar.process_total_heap_size.reg_name_foo.0_1_0:25|g">> + , <<"beam_stats.node_foo_host_bar.process_stack_size.reg_name_foo.0_1_0:10|g">> + , <<"beam_stats.node_foo_host_bar.process_message_queue_len.reg_name_foo.0_1_0:0|g">> + + % Process 2 + , <<"beam_stats.node_foo_host_bar.process_memory.bar_mod-bar_fun-1--NONE--NONE.0_2_0:25|g">> + , <<"beam_stats.node_foo_host_bar.process_total_heap_size.bar_mod-bar_fun-1--NONE--NONE.0_2_0:35|g">> + , <<"beam_stats.node_foo_host_bar.process_stack_size.bar_mod-bar_fun-1--NONE--NONE.0_2_0:40|g">> + , <<"beam_stats.node_foo_host_bar.process_message_queue_len.bar_mod-bar_fun-1--NONE--NONE.0_2_0:5|g">> + + % Process 3 + , <<"beam_stats.node_foo_host_bar.process_memory.baz_mod-baz_fun-3--baz_otp_mod-baz_otp_fun-2--0_0_0-0_1_0.0_3_0:25|g">> + , <<"beam_stats.node_foo_host_bar.process_total_heap_size.baz_mod-baz_fun-3--baz_otp_mod-baz_otp_fun-2--0_0_0-0_1_0.0_3_0:35|g">> + , <<"beam_stats.node_foo_host_bar.process_stack_size.baz_mod-baz_fun-3--baz_otp_mod-baz_otp_fun-2--0_0_0-0_1_0.0_3_0:40|g">> + , <<"beam_stats.node_foo_host_bar.process_message_queue_len.baz_mod-baz_fun-3--baz_otp_mod-baz_otp_fun-2--0_0_0-0_1_0.0_3_0:1|g">> + ], + MsgsReceived = binary:split(PacketsCombined, <<"\n">>, [global, trim]), + RemoveExpectedFromReceived = + fun (Expected, Received) -> + ct:log( + "Looking for expected msg ~p in remaining received ~p~n", + [Expected, Received] + ), + true = lists:member(Expected, Received), + Received -- [Expected] + end, + [] = lists:foldl(RemoveExpectedFromReceived, MsgsReceived, MsgsExpected), + meck:unload(beam_stats_source). + +meck_expect_beam_stats() -> + meck_expect_beam_stats([]). + +meck_expect_beam_stats(Overrides) -> + IOBytesIn = hope_kv_list:get(Overrides, io_bytes_in , 3), + IOBytesOut = hope_kv_list:get(Overrides, io_bytes_out, 7), + ContextSwitches = hope_kv_list:get(Overrides, context_switches, 5), Pid0 = list_to_pid("<0.0.0>"), Pid1 = list_to_pid("<0.1.0>"), Pid2 = list_to_pid("<0.2.0>"), @@ -110,105 +227,94 @@ t_send(_Cfg) -> { id = foo , name = foo , size = 5 - , memory = 25 + , memory = 40 }, ETSTableStatsBar = #beam_stats_ets_table { id = 37 , name = bar , size = 8 - , memory = 38 + , memory = 64 }, - % TODO: Indent #beam_stats as #beam_stats_ets_table - BEAMStats = #beam_stats + meck:expect(beam_stats_source, erlang_memory, + fun () -> [{mem_type_foo, 1}, {mem_type_bar, 2}, {mem_type_baz, 3}] end), + meck:expect(beam_stats_source, erlang_node, + fun () -> 'node_foo@host_bar' end), + meck:expect(beam_stats_source, erlang_registered, + fun () -> [reg_name_foo] end), + meck:expect(beam_stats_source, erlang_statistics, + fun (io ) -> {{input, IOBytesIn}, {output, IOBytesOut}} + ; (context_switches) -> {ContextSwitches, 0} + ; (reductions ) -> {0, 9} % 1st element is unused + ; (run_queue ) -> 17 + end + ), + meck:expect(beam_stats_source, ets_all, + fun () -> [foo, 37] end), + meck:expect(beam_stats_source, erlang_system_info, + fun (wordsize) -> 8 end), + meck:expect(beam_stats_source, ets_info, + fun (foo, memory) -> 5 + ; (foo, name ) -> foo + ; (foo, size ) -> 5 + ; (37 , memory) -> 8 + ; (37 , name ) -> bar + ; (37 , size ) -> 8 + end + ), + meck:expect(beam_stats_source, erlang_processes, + fun () -> [Pid1, Pid2, Pid3] end), + meck:expect(beam_stats_source, os_timestamp, + fun () -> {1, 2, 3} end), + meck:expect(beam_stats_source, erlang_process_info, + fun (P, K) when P == Pid1 -> + case K + of dictionary -> {K, []} + ; initial_call -> {K, {foo_mod, foo_fun, 2}} + ; registered_name -> {K, reg_name_foo} + ; status -> {K, running} + ; memory -> {K, 15} + ; total_heap_size -> {K, 25} + ; stack_size -> {K, 10} + ; message_queue_len -> {K, 0} + end + ; (P, K) when P == Pid2 -> + case K + of dictionary -> {K, []} + ; initial_call -> {K, {bar_mod, bar_fun, 1}} + ; registered_name -> [] + ; status -> {K, running} + ; memory -> {K, 25} + ; total_heap_size -> {K, 35} + ; stack_size -> {K, 40} + ; message_queue_len -> {K, 5} + end + ; (P, K) when P == Pid3 -> + Dict = + [ {'$initial_call', {baz_otp_mod, baz_otp_fun, 2}} + , {'$ancestors' , [Pid0, Pid1]} + ], + case K + of dictionary -> {K, Dict} + ; initial_call -> {K, {baz_mod, baz_fun, 3}} + ; registered_name -> [] + ; status -> {K, running} + ; memory -> {K, 25} + ; total_heap_size -> {K, 35} + ; stack_size -> {K, 40} + ; message_queue_len -> {K, 1} + end + end + ), + #beam_stats { timestamp = {1, 2, 3} , node_id = 'node_foo@host_bar' , memory = [{mem_type_foo, 1}, {mem_type_bar, 2}, {mem_type_baz, 3}] - , io_bytes_in = 3 - , io_bytes_out = 7 - , context_switches = 5 + , io_bytes_in = IOBytesIn + , io_bytes_out = IOBytesOut + , context_switches = ContextSwitches , reductions = 9 , run_queue = 17 , ets = [ETSTableStatsFoo, ETSTableStatsBar] , processes = Processes - }, - ServerPort = 8125, - {ok, ServerSocket} = gen_udp:open(ServerPort, [binary, {active, false}]), - BEAMStatsQ = queue:in(BEAMStats, queue:new()), - Options = [{dst_port, ServerPort}], - {_, State1} = beam_stats_consumer_statsd:init(Options), - State2 = beam_stats_consumer_statsd:consume(BEAMStatsQ, State1), - {} = beam_stats_consumer_statsd:terminate(State2), - ResultOfReceive1 = gen_udp:recv(ServerSocket, 0), - ResultOfReceive2 = gen_udp:recv(ServerSocket, 0), - ResultOfReceive3 = gen_udp:recv(ServerSocket, 0), - ResultOfReceive4 = gen_udp:recv(ServerSocket, 0), - ok = gen_udp:close(ServerSocket), - {ok, {_, _, PacketReceived1}} = ResultOfReceive1, - {ok, {_, _, PacketReceived2}} = ResultOfReceive2, - {ok, {_, _, PacketReceived3}} = ResultOfReceive3, - {ok, {_, _, PacketReceived4}} = ResultOfReceive4, - ct:log("PacketReceived1: ~n~s~n", [PacketReceived1]), - ct:log("PacketReceived2: ~n~s~n", [PacketReceived2]), - ct:log("PacketReceived3: ~n~s~n", [PacketReceived3]), - ct:log("PacketReceived4: ~n~s~n", [PacketReceived4]), - PacketsCombined = - << PacketReceived1/binary - , PacketReceived2/binary - , PacketReceived3/binary - , PacketReceived4/binary - >>, - ct:log("PacketsCombined: ~n~s~n", [PacketsCombined]), - MsgsExpected = - [ <<"beam_stats.node_foo_host_bar.io.bytes_in:3|g">> - , <<"beam_stats.node_foo_host_bar.io.bytes_out:7|g">> - , <<"beam_stats.node_foo_host_bar.context_switches:5|g">> - , <<"beam_stats.node_foo_host_bar.reductions:9|g">> - , <<"beam_stats.node_foo_host_bar.run_queue:17|g">> - , <<"beam_stats.node_foo_host_bar.memory.mem_type_foo:1|g">> - , <<"beam_stats.node_foo_host_bar.memory.mem_type_bar:2|g">> - , <<"beam_stats.node_foo_host_bar.memory.mem_type_baz:3|g">> - , <<"beam_stats.node_foo_host_bar.ets_table.size.foo.foo:5|g">> - , <<"beam_stats.node_foo_host_bar.ets_table.memory.foo.foo:25|g">> - , <<"beam_stats.node_foo_host_bar.ets_table.size.bar.37:8|g">> - , <<"beam_stats.node_foo_host_bar.ets_table.memory.bar.37:38|g">> - - % Processes totals - , <<"beam_stats.node_foo_host_bar.processes_count_all:3|g">> - , <<"beam_stats.node_foo_host_bar.processes_count_exiting:0|g">> - , <<"beam_stats.node_foo_host_bar.processes_count_garbage_collecting:0|g">> - , <<"beam_stats.node_foo_host_bar.processes_count_registered:1|g">> - , <<"beam_stats.node_foo_host_bar.processes_count_runnable:0|g">> - , <<"beam_stats.node_foo_host_bar.processes_count_running:3|g">> - , <<"beam_stats.node_foo_host_bar.processes_count_suspended:0|g">> - , <<"beam_stats.node_foo_host_bar.processes_count_waiting:0|g">> - - % Process 1 - , <<"beam_stats.node_foo_host_bar.process_memory.reg_name_foo.0_1_0:15|g">> - , <<"beam_stats.node_foo_host_bar.process_total_heap_size.reg_name_foo.0_1_0:25|g">> - , <<"beam_stats.node_foo_host_bar.process_stack_size.reg_name_foo.0_1_0:10|g">> - , <<"beam_stats.node_foo_host_bar.process_message_queue_len.reg_name_foo.0_1_0:0|g">> - - % Process 2 - , <<"beam_stats.node_foo_host_bar.process_memory.bar_mod-bar_fun-1--NONE--NONE.0_2_0:25|g">> - , <<"beam_stats.node_foo_host_bar.process_total_heap_size.bar_mod-bar_fun-1--NONE--NONE.0_2_0:35|g">> - , <<"beam_stats.node_foo_host_bar.process_stack_size.bar_mod-bar_fun-1--NONE--NONE.0_2_0:40|g">> - , <<"beam_stats.node_foo_host_bar.process_message_queue_len.bar_mod-bar_fun-1--NONE--NONE.0_2_0:5|g">> - - % Process 3 - , <<"beam_stats.node_foo_host_bar.process_memory.baz_mod-baz_fun-3--baz_otp_mod-baz_otp_fun-2--0_0_0-0_1_0.0_3_0:25|g">> - , <<"beam_stats.node_foo_host_bar.process_total_heap_size.baz_mod-baz_fun-3--baz_otp_mod-baz_otp_fun-2--0_0_0-0_1_0.0_3_0:35|g">> - , <<"beam_stats.node_foo_host_bar.process_stack_size.baz_mod-baz_fun-3--baz_otp_mod-baz_otp_fun-2--0_0_0-0_1_0.0_3_0:40|g">> - , <<"beam_stats.node_foo_host_bar.process_message_queue_len.baz_mod-baz_fun-3--baz_otp_mod-baz_otp_fun-2--0_0_0-0_1_0.0_3_0:1|g">> - ], - MsgsReceived = binary:split(PacketsCombined, <<"\n">>, [global, trim]), - RemoveExpectedFromReceived = - fun (Expected, Received) -> - ct:log( - "Looking for expected msg ~p in remaining received ~p~n", - [Expected, Received] - ), - true = lists:member(Expected, Received), - Received -- [Expected] - end, - [] = lists:foldl(RemoveExpectedFromReceived, MsgsReceived, MsgsExpected). + }.