testing add/remove users REST API in integration tests

Author: DenysGonchar

ci/run.sh

@@ -15,4 +15,5 @@ run ./ci/test_distribute_scenario.sh
 run ./ci/test_run_scenario.sh
 run ./ci/test_add_new_node.sh
 run ./ci/test_update_settings.sh
+run ./ci/test_add_and_remove_users.sh
 run ./ci/stop_demo_cluster.sh

ci/test_add_and_remove_users.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+
+source "$(dirname "$0")/helper.sh"
+enable_strict_mode
+
+#############################
+## amoc REST API functions ##
+#############################
+
+function node_list() {
+    local nodes=( "$@" ) nodes_json="" node=
+    if [ "${#nodes[@]}" -gt "0" ]; then
+        nodes_json+='"nodes": ["'"${nodes[0]}"'"'
+        for node in "${nodes[@]:1}"; do
+            nodes_json+=', "'"$node"'"'
+        done
+        nodes_json+='], '
+    fi
+    echo -n "$nodes_json"
+}
+
+function add_users() {
+    local port="$(amoc_container_port "$1")" # $1 - where to send request
+    local n="$2" # $2 - number of users to add
+    shift 2
+    local nodes="$(node_list "$@")"
+    local json_body="{ ${nodes} \"users\": $n}"
+    curl -X PATCH --header 'Content-Type: application/json' --header 'Accept: application/json' \
+         -s  -w " %{http_code}" -d "$json_body" "http://localhost:${port}/execution/add_users"
+}
+
+function remove_users() {
+    local port="$(amoc_container_port "$1")" # $1 - where to send request
+    local n="$2" # $2 - number of users to add
+    shift 2
+    local nodes="$(node_list "$@")"
+    local json_body="{ ${nodes} \"users\": $n}"
+    curl -X PATCH --header 'Content-Type: application/json' --header 'Accept: application/json' \
+         -s  -w " %{http_code}" -d "$json_body" "http://localhost:${port}/execution/remove_users"
+}
+
+
+echo "adding users to nodes"
+add_users amoc-master 6 "amoc_arsenal@amoc-worker-3" | contains 200
+add_users amoc-master 2 "amoc_arsenal@"{"amoc-worker-1","amoc-worker-2"} | contains 200
+add_users amoc-master 15 | contains 200 ## add 5 users on each node
+add_users amoc-worker-3 15 | contains 500 '"error":"not_a_master"'
+
+echo "checking status of the nodes"
+sleep 1 ## 1 second is enough to start 11
+get_status amoc-master | contains '"amoc_status":"up"' \
+                                  '"status":"disabled"'
+worker_status=( '"amoc_status":"up"'
+                '"status":"running"'
+                '"scenario":"dummy_scenario"'
+                '"test":"<<\\"test_value\\">>"'
+                '"interarrival":"50"' )
+get_status amoc-worker-1 | contains "${worker_status[@]}" '"number_of_users":11'
+get_status amoc-worker-2 | contains "${worker_status[@]}" '"number_of_users":11'
+get_status amoc-worker-3 | contains "${worker_status[@]}" '"number_of_users":11'
+
+echo "removing users from nodes"
+ ## remove all the users from amoc-worker-3
+remove_users amoc-master 11 "amoc_arsenal@amoc-worker-3" | contains 200
+remove_users amoc-master 2 "amoc_arsenal@"{"amoc-worker-1","amoc-worker-2"} | contains 200
+sleep 3 ## two seconds is a shutdown period for a forced users removal.
+        ## so we must wait before removing the next chunk of users,
+        ## otherwise the same users can be selected for removal twice.
+remove_users amoc-master 15 | contains 200 ## remove 5 users from each node
+remove_users amoc-worker-3 15 | contains 500 '"error":"not_a_master"'
+
+echo "checking status of the nodes"
+sleep 3 ## 2 seconds is a shutdown period for a forced users removal.
+get_status amoc-master | contains '"amoc_status":"up"' \
+                                  '"status":"disabled"'
+get_status amoc-worker-1 | contains "${worker_status[@]}" '"number_of_users":5'
+get_status amoc-worker-2 | contains "${worker_status[@]}" '"number_of_users":5'
+get_status amoc-worker-3 | contains "${worker_status[@]}" '"number_of_users":0'

src/rest_api/amoc_api_helpers_execution.erl

@@ -65,9 +65,9 @@ add_users(_) ->
 
 -spec remove_users(body()) -> ret_value().
 remove_users(#{<<"users">> := Users, <<"nodes">> := Nodes}) ->
-    amoc_dist:remove(Users, false, read_nodes(Nodes));
+    amoc_dist:remove(Users, true, read_nodes(Nodes));
 remove_users(#{<<"users">> := Users}) ->
-    amoc_dist:remove(Users, false);
+    amoc_dist:remove(Users, true);
 remove_users(_) ->
     {error, invalid_body}.
 

test/amoc_api_execution_handler_SUITE.erl

@@ -92,13 +92,13 @@ setup_meck(add_users_on_nodes) ->
     meck:expect(amoc_dist, add, fun(10, [node1@host1, node2@host2]) -> {ok, mocked} end);
 setup_meck(remove_users) ->
     meck:expect(amoc_dist, get_state, fun() -> running end),
-    meck:expect(amoc_dist, remove, fun(10, false) -> {ok, mocked} end);
+    meck:expect(amoc_dist, remove, fun(10, true) -> {ok, mocked} end);
 setup_meck(fail_to_remove_users_when_amoc_dist_fails) ->
     meck:expect(amoc_dist, get_state, fun() -> running end),
-    meck:expect(amoc_dist, remove, fun(10, false) -> {error, mocked} end);
+    meck:expect(amoc_dist, remove, fun(10, true) -> {error, mocked} end);
 setup_meck(remove_users_on_nodes) ->
     meck:expect(amoc_dist, get_state, fun() -> running end),
-    meck:expect(amoc_dist, remove, fun(10, false, [node1@host1, node2@host2]) -> {ok, mocked} end);
+    meck:expect(amoc_dist, remove, fun(10, true, [node1@host1, node2@host2]) -> {ok, mocked} end);
 setup_meck(update_settings) ->
     meck:expect(amoc_dist, get_state, fun() -> running end),
     meck:expect(amoc_dist, update_settings,