Support code coverage

Previously it was not possible to see code coverage for the majority of
test cases: integration tests that create RabbitMQ nodes.
It was only possible to see code coverage for unit tests.
This commit allows to see code coverage for tests that create RabbitMQ
nodes.

The only thing you need to do is setting the `COVER` variable, for example
```
make -C deps/rabbitmq_mqtt ct COVER=1
```
will show you coverage across all tests in the MQTT plugin.

This commit fixes the erlang.mk file that did not append the `-cover
<sepc_path>` flag to the `ct_run` command.

Whenever a RabbitMQ node is started `ct_cover:add_nodes/1` is called.
Contrary to the documentation which states

> To have effect, this function is to be called from init_per_suite/1 (see common_test) before any tests are performed.

I found that it also works in init_per_group/1 or even within the test cases themselves.

Whenever a RabbitMQ node is stopped or killed `ct_cover:remove_nodes/1`
is called to transfer results from the RabbitMQ node to the CT node.

Since the erlang.mk file writes a file called `test/ct.cover.spec`
including the line:
```
{export,".../rabbitmq-server/deps/rabbitmq_mqtt/cover/ct.coverdata"}.
```
results across all test suites will be accumulated in that file.

The accumulated result can be seen through the link `Coverage log` on the test suite result pages.

Author: ansd

.gitignore

@@ -61,6 +61,7 @@
 /plugins.lock
 /sbin/
 /sbin.lock
+ct.cover.spec
 erl_crash.dump
 rebar3.crashdump
 .envrc

deps/rabbit/test/direct_exchange_routing_v2_SUITE.erl

@@ -321,8 +321,7 @@ recover_bindings(Config) ->
     assert_index_table_empty(Config),
     rabbit_ct_broker_helpers:rabbitmqctl(Config, Server, ["import_definitions", Path], 10_000),
     ?assertEqual(?NUM_BINDINGS_TO_DIRECT_ECHANGE, table_size(Config, ?INDEX_TABLE_NAME)),
-    ok = rabbit_ct_broker_helpers:stop_node(Config, 0),
-    ok = rabbit_ct_broker_helpers:start_node(Config, 0),
+    ok = rabbit_ct_broker_helpers:restart_node(Config, 0),
     ?assertEqual(?NUM_BINDINGS_TO_DIRECT_ECHANGE_DURABLE, table_size(Config, ?INDEX_TABLE_NAME)),
 
     %% cleanup

deps/rabbitmq_ct_helpers/src/rabbit_ct_broker_helpers.erl

@@ -805,6 +805,7 @@ query_node(Config, NodeConfig) ->
                    ct:pal("NO RABBITMQ_FEATURE_FLAGS_FILE"),
                    Vars0
            end,
+    cover_add_node(Nodename),
     rabbit_ct_helpers:set_config(NodeConfig, Vars).
 
 maybe_cluster_nodes(Config) ->
@@ -1002,9 +1003,10 @@ stop_rabbitmq_nodes(Config) ->
     proplists:delete(rmq_nodes, Config).
 
 stop_rabbitmq_node(Config, NodeConfig) ->
+    Nodename = ?config(nodename, NodeConfig),
+    cover_remove_node(Nodename),
     SrcDir = ?config(effective_srcdir, NodeConfig),
     InitialMakeVars = ?config(make_vars_for_node_startup, NodeConfig),
-    Nodename = ?config(nodename, NodeConfig),
     InitialNodename = ?config(initial_nodename, NodeConfig),
     MakeVars = InitialMakeVars ++ [
       {"RABBITMQ_NODENAME=~ts", [Nodename]},
@@ -1626,7 +1628,7 @@ stop_node_after(Config, Node, Sleep) ->
 
 kill_node(Config, Node) ->
     Pid = rpc(Config, Node, os, getpid, []),
-    %% FIXME maybe_flush_cover(Cfg),
+    cover_remove_node(Node),
     Cmd = case os:type() of
               {win32, _} ->
                   case os:find_executable("taskkill.exe") of
@@ -1917,3 +1919,23 @@ user(Username) ->
     #user{username       = Username,
           tags           = [administrator],
           authz_backends = [{rabbit_auth_backend_internal, none}]}.
+
+cover_add_node(Node) ->
+    if_cover(
+      fun() ->
+              {ok, [Node]} = ct_cover:add_nodes([Node])
+      end).
+
+cover_remove_node(Node) ->
+    if_cover(
+      fun() ->
+              ok = ct_cover:remove_nodes([Node])
+      end).
+
+if_cover(F) ->
+    case os:getenv("COVER") of
+        false ->
+            ok;
+        _ ->
+            F()
+    end.

deps/rabbitmq_mqtt/.gitignore

@@ -1,6 +1,7 @@
 .sw?
 .*.sw?
 *.beam
+*.coverdata
 .idea/*
 /.erlang.mk/
 /cover/
@@ -15,6 +16,7 @@
 /plugins.lock
 /sbin/
 /sbin.lock
+/test/ct.cover.spec
 /xrefr
 debug/*
 *.plt

erlang.mk

@@ -6669,6 +6669,12 @@ CT_RUN = ct_run \
 	-dir $(TEST_DIR) \
 	-logdir $(CT_LOGS_DIR)
 
+ifdef COVER
+ifneq ($(wildcard $(TEST_DIR)),)
+CT_RUN += -cover $(TEST_DIR)/ct.cover.spec
+endif
+endif
+
 ifeq ($(CT_SUITES),)
 ct: $(if $(IS_APP)$(ROOT_DIR),,apps-ct)
 else
@@ -8011,7 +8017,6 @@ endif
 # Code coverage for Common Test.
 
 ifdef COVER
-ifdef CT_RUN
 ifneq ($(wildcard $(TEST_DIR)),)
 test-build:: $(TEST_DIR)/ct.cover.spec
 
@@ -8022,9 +8027,6 @@ $(TEST_DIR)/ct.cover.spec: cover-data-dir
 			$(foreach a,$(COVER_APPS),$(comma) \"$(call core_native_path,$(APPS_DIR)/$a/ebin)\") \
 			$(foreach d,$(COVER_DEPS),$(comma) \"$(call core_native_path,$(DEPS_DIR)/$d/ebin)\")]}." \
 		'{export,"$(call core_native_path,$(abspath $(COVER_DATA_DIR))/ct.coverdata)"}.' > $@
-
-CT_RUN += -cover $(TEST_DIR)/ct.cover.spec
-endif
 endif
 endif