Merge pull request #530 from anhanhnguyen/force-shrink-to-non-voter

Allow force shrink to non-voter member

Author: kjnilsson

src/ra_server.erl

@@ -1495,13 +1495,15 @@ handle_follower({register_external_log_reader, Pid}, #{log := Log0} = State) ->
     {follower, State#{log => Log}, Effs};
 handle_follower(force_member_change,
                 #{cfg := #cfg{id = Id,
+                              uid = Uid,
                               log_id = LogId}} = State0) ->
-    Cluster = #{Id => new_peer()},
+    Cluster = #{Id => new_peer_with(#{voter_status => #{uid => Uid, 
+                                                        membership => voter}})},
     ?WARN("~ts: Forcing cluster change. New cluster ~w",
           [LogId, Cluster]),
     {ok, _, _, State, Effects} =
         append_cluster_change(Cluster, undefined, no_reply, State0, []),
-    call_for_election(pre_vote, State, [{reply, ok} | Effects]);
+    call_for_election(pre_vote, State#{membership => voter}, [{reply, ok} | Effects]);
 handle_follower(#info_rpc{term = Term} = Msg,
                 #{current_term := CurTerm} = State)
   when CurTerm < Term ->

test/ra_2_SUITE.erl

@@ -48,6 +48,7 @@ all_tests() ->
      add_member_without_quorum,
      force_start_follower_as_single_member,
      force_start_follower_as_single_member_nonvoter,
+     force_start_nonvoter_as_single_member,
      initial_members_query
     ].
 
@@ -784,30 +785,80 @@ force_start_follower_as_single_member_nonvoter(Config) ->
     ServerId4 = ?config(server_id4, Config),
     UId4 = ?config(uid4, Config),
     Conf4 = conf(ClusterName, UId4, ServerId4, PrivDir, [ServerId3]),
-    {ok, _, _} = ra:add_member(ServerId3, #{id => ServerId4, membership => promotable, uid => <<"test4">>}),
+    {ok, _, _} = ra:add_member(ServerId3, #{id => ServerId4, membership => promotable, uid => UId4}),
     %% the membership has changed but member not running yet
     %% it is nonvoter and does not affect quorum size
     {ok, _, _} = ra:process_command(ServerId3, {enq, banana}),
     %% start new member
-    ok = ra:start_server(?SYS, Conf4#{membership => promotable, uid => <<"test4">>}),
+    ok = ra:start_server(?SYS, Conf4#{membership => promotable, uid => UId4}),
     {ok, _, ServerId3} = ra:members(ServerId4),
     ok = enqueue(ServerId3, msg3),
 
     %% add a non-voter member
     ServerId5 = ?config(server_id5, Config),
     UId5 = ?config(uid5, Config),
     Conf5 = conf(ClusterName, UId5, ServerId5, PrivDir, [ServerId3]),
-    {ok, _, _} = ra:add_member(ServerId3, #{id => ServerId5, membership => non_voter, uid => <<"test5">>}),
+    {ok, _, _} = ra:add_member(ServerId3, #{id => ServerId5, membership => non_voter, uid => UId5}),
     %% the membership has changed but member not running yet
     %% it is nonvoter and does not affect quorum size
     {ok, _, _} = ra:process_command(ServerId3, {enq, banana}),
     %% start new member
-    ok = ra:start_server(?SYS, Conf5#{membership => non_voter, uid => <<"test5">>}),
+    ok = ra:start_server(?SYS, Conf5#{membership => non_voter, uid => UId5}),
     {ok, _, ServerId3} = ra:members(ServerId5),
     ok = enqueue(ServerId3, msg4),
 
     ok.
 
+force_start_nonvoter_as_single_member(Config) ->
+    ok = logger:set_primary_config(level, all),
+    ClusterName = ?config(cluster_name, Config),
+    PrivDir = ?config(priv_dir, Config),
+    ServerId1 = ?config(server_id, Config),
+    ServerId2 = ?config(server_id2, Config),
+    ServerId3 = ?config(server_id3, Config),
+    InitialCluster = [ServerId1, ServerId2, ServerId3],
+    ok = start_cluster(ClusterName, InitialCluster),
+    timer:sleep(100),
+
+    ServerId4 = ?config(server_id4, Config),
+    UId4 = ?config(uid4, Config),
+    Conf4 = conf(ClusterName, UId4, ServerId4, PrivDir, InitialCluster),
+    {ok, _, _} = ra:add_member(ServerId3, #{id => ServerId4, membership => non_voter, uid => UId4}),
+    ok = ra:start_server(?SYS, Conf4#{membership => non_voter, uid => UId4}),
+
+    %% stop voters to simulate permanent outage
+    ok = ra:stop_server(?SYS, ServerId1),
+    ok = ra:stop_server(?SYS, ServerId2),
+    ok = ra:stop_server(?SYS, ServerId3),
+    timer:sleep(100),
+
+    %% force the remaining node to change it's membership
+    ok = ra_server_proc:force_shrink_members_to_current_member(ServerId4),
+    {ok, [_], ServerId4} = ra:members(ServerId4),
+    ok = enqueue(ServerId4, msg1),
+
+    %% test that it works after restart
+    ok = ra:stop_server(?SYS, ServerId4),
+    ok = ra:restart_server(?SYS, ServerId4),
+    {ok, [_], ServerId4} = ra:members(ServerId4),
+    ok = enqueue(ServerId4, msg2),
+
+
+    %% add a non-voter member
+    ServerId5 = ?config(server_id5, Config),
+    UId5 = ?config(uid5, Config),
+    Conf5 = conf(ClusterName, UId5, ServerId5, PrivDir, [ServerId4]),
+    {ok, _, _} = ra:add_member(ServerId4, #{id => ServerId5, membership => non_voter, uid => UId5}),
+    %% the membership has changed but member not running yet
+    %% it is nonvoter and does not affect quorum size
+    ok = enqueue(ServerId4, msg3),
+    %% start new member
+    ok = ra:start_server(?SYS, Conf5#{membership => non_voter, uid => UId5}),
+    {ok, _, ServerId4} = ra:members(ServerId5),
+    ok = enqueue(ServerId4, msg4),
+
+    ok.
+
 initial_members_query(Config) ->
     ok = logger:set_primary_config(level, all),
     %% ra:start_server should fail if the node already exists