diff options
Diffstat (limited to 'examples/redis-unstable/tests/sentinel')
23 files changed, 1515 insertions, 0 deletions
diff --git a/examples/redis-unstable/tests/sentinel/run.tcl b/examples/redis-unstable/tests/sentinel/run.tcl new file mode 100644 index 0000000..4f2a656 --- /dev/null +++ b/examples/redis-unstable/tests/sentinel/run.tcl | |||
| @@ -0,0 +1,41 @@ | |||
| 1 | # Sentinel test suite. | ||
| 2 | # | ||
| 3 | # Copyright (C) 2014-Present, Redis Ltd. | ||
| 4 | # All Rights reserved. | ||
| 5 | # | ||
| 6 | # Licensed under your choice of (a) the Redis Source Available License 2.0 | ||
| 7 | # (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the | ||
| 8 | # GNU Affero General Public License v3 (AGPLv3). | ||
| 9 | |||
| 10 | cd tests/sentinel | ||
| 11 | source ../instances.tcl | ||
| 12 | |||
| 13 | set ::instances_count 5 ; # How many instances we use at max. | ||
| 14 | set ::tlsdir "../../tls" | ||
| 15 | |||
| 16 | proc main {} { | ||
| 17 | parse_options | ||
| 18 | if {$::leaked_fds_file != ""} { | ||
| 19 | set ::env(LEAKED_FDS_FILE) $::leaked_fds_file | ||
| 20 | } | ||
| 21 | spawn_instance sentinel $::sentinel_base_port $::instances_count { | ||
| 22 | "sentinel deny-scripts-reconfig no" | ||
| 23 | "enable-protected-configs yes" | ||
| 24 | "enable-debug-command yes" | ||
| 25 | } "../tests/includes/sentinel.conf" | ||
| 26 | |||
| 27 | spawn_instance redis $::redis_base_port $::instances_count { | ||
| 28 | "enable-protected-configs yes" | ||
| 29 | "enable-debug-command yes" | ||
| 30 | "save ''" | ||
| 31 | } | ||
| 32 | run_tests | ||
| 33 | cleanup | ||
| 34 | end_tests | ||
| 35 | } | ||
| 36 | |||
| 37 | if {[catch main e]} { | ||
| 38 | puts $::errorInfo | ||
| 39 | cleanup | ||
| 40 | exit 1 | ||
| 41 | } | ||
diff --git a/examples/redis-unstable/tests/sentinel/tests/00-base.tcl b/examples/redis-unstable/tests/sentinel/tests/00-base.tcl new file mode 100644 index 0000000..7b64395 --- /dev/null +++ b/examples/redis-unstable/tests/sentinel/tests/00-base.tcl | |||
| @@ -0,0 +1,210 @@ | |||
| 1 | # Check the basic monitoring and failover capabilities. | ||
| 2 | source "../tests/includes/start-init-tests.tcl" | ||
| 3 | source "../tests/includes/init-tests.tcl" | ||
| 4 | |||
| 5 | foreach_sentinel_id id { | ||
| 6 | S $id sentinel debug default-down-after 1000 | ||
| 7 | } | ||
| 8 | |||
| 9 | if {$::simulate_error} { | ||
| 10 | test "This test will fail" { | ||
| 11 | fail "Simulated error" | ||
| 12 | } | ||
| 13 | } | ||
| 14 | |||
| 15 | test "Sentinel command flag infrastructure works correctly" { | ||
| 16 | foreach_sentinel_id id { | ||
| 17 | set command_list [S $id command list] | ||
| 18 | |||
| 19 | foreach cmd {ping info subscribe client|setinfo} { | ||
| 20 | assert_not_equal [S $id command docs $cmd] {} | ||
| 21 | assert_not_equal [lsearch $command_list $cmd] -1 | ||
| 22 | } | ||
| 23 | |||
| 24 | foreach cmd {save bgrewriteaof blpop replicaof} { | ||
| 25 | assert_equal [S $id command docs $cmd] {} | ||
| 26 | assert_equal [lsearch $command_list $cmd] -1 | ||
| 27 | assert_error {ERR unknown command*} {S $id $cmd} | ||
| 28 | } | ||
| 29 | |||
| 30 | assert_error {ERR unknown subcommand*} {S $id client no-touch} | ||
| 31 | } | ||
| 32 | } | ||
| 33 | |||
| 34 | test "SENTINEL HELP output the sentinel subcommand help" { | ||
| 35 | assert_match "*SENTINEL <subcommand> *" [S 0 SENTINEL HELP] | ||
| 36 | } | ||
| 37 | |||
| 38 | test "SENTINEL MYID return the sentinel instance ID" { | ||
| 39 | assert_equal 40 [string length [S 0 SENTINEL MYID]] | ||
| 40 | assert_equal [S 0 SENTINEL MYID] [S 0 SENTINEL MYID] | ||
| 41 | } | ||
| 42 | |||
| 43 | test "SENTINEL INFO CACHE returns the cached info" { | ||
| 44 | set res [S 0 SENTINEL INFO-CACHE mymaster] | ||
| 45 | assert_morethan_equal [llength $res] 2 | ||
| 46 | assert_equal "mymaster" [lindex $res 0] | ||
| 47 | |||
| 48 | set res [lindex $res 1] | ||
| 49 | assert_morethan_equal [llength $res] 2 | ||
| 50 | assert_morethan [lindex $res 0] 0 | ||
| 51 | assert_match "*# Server*" [lindex $res 1] | ||
| 52 | } | ||
| 53 | |||
| 54 | test "SENTINEL PENDING-SCRIPTS returns the information about pending scripts" { | ||
| 55 | # may or may not have a value, so assert greater than or equal to 0. | ||
| 56 | assert_morethan_equal [llength [S 0 SENTINEL PENDING-SCRIPTS]] 0 | ||
| 57 | } | ||
| 58 | |||
| 59 | test "SENTINEL MASTERS returns a list of monitored masters" { | ||
| 60 | assert_match "*mymaster*" [S 0 SENTINEL MASTERS] | ||
| 61 | assert_morethan_equal [llength [S 0 SENTINEL MASTERS]] 1 | ||
| 62 | } | ||
| 63 | |||
| 64 | test "SENTINEL SENTINELS returns a list of sentinel instances" { | ||
| 65 | assert_morethan_equal [llength [S 0 SENTINEL SENTINELS mymaster]] 1 | ||
| 66 | } | ||
| 67 | |||
| 68 | test "SENTINEL SLAVES returns a list of the monitored replicas" { | ||
| 69 | assert_morethan_equal [llength [S 0 SENTINEL SLAVES mymaster]] 1 | ||
| 70 | } | ||
| 71 | |||
| 72 | test "SENTINEL SIMULATE-FAILURE HELP list supported flags" { | ||
| 73 | set res [S 0 SENTINEL SIMULATE-FAILURE HELP] | ||
| 74 | assert_morethan_equal [llength $res] 2 | ||
| 75 | assert_equal {crash-after-election crash-after-promotion} $res | ||
| 76 | } | ||
| 77 | |||
| 78 | test "Basic failover works if the master is down" { | ||
| 79 | set old_port [RPort $master_id] | ||
| 80 | set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] | ||
| 81 | assert {[lindex $addr 1] == $old_port} | ||
| 82 | kill_instance redis $master_id | ||
| 83 | foreach_sentinel_id id { | ||
| 84 | S $id sentinel debug ping-period 500 | ||
| 85 | S $id sentinel debug ask-period 500 | ||
| 86 | wait_for_condition 1000 100 { | ||
| 87 | [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port | ||
| 88 | } else { | ||
| 89 | fail "At least one Sentinel did not receive failover info" | ||
| 90 | } | ||
| 91 | } | ||
| 92 | restart_instance redis $master_id | ||
| 93 | set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] | ||
| 94 | set master_id [get_instance_id_by_port redis [lindex $addr 1]] | ||
| 95 | } | ||
| 96 | |||
| 97 | test "New master [join $addr {:}] role matches" { | ||
| 98 | assert {[RI $master_id role] eq {master}} | ||
| 99 | } | ||
| 100 | |||
| 101 | test "All the other slaves now point to the new master" { | ||
| 102 | foreach_redis_id id { | ||
| 103 | if {$id != $master_id && $id != 0} { | ||
| 104 | wait_for_condition 1000 50 { | ||
| 105 | [RI $id master_port] == [lindex $addr 1] | ||
| 106 | } else { | ||
| 107 | fail "Redis ID $id not configured to replicate with new master" | ||
| 108 | } | ||
| 109 | } | ||
| 110 | } | ||
| 111 | } | ||
| 112 | |||
| 113 | test "The old master eventually gets reconfigured as a slave" { | ||
| 114 | wait_for_condition 1000 50 { | ||
| 115 | [RI 0 master_port] == [lindex $addr 1] | ||
| 116 | } else { | ||
| 117 | fail "Old master not reconfigured as slave of new master" | ||
| 118 | } | ||
| 119 | } | ||
| 120 | |||
| 121 | test "ODOWN is not possible without N (quorum) Sentinels reports" { | ||
| 122 | foreach_sentinel_id id { | ||
| 123 | S $id SENTINEL SET mymaster quorum [expr $sentinels+1] | ||
| 124 | } | ||
| 125 | set old_port [RPort $master_id] | ||
| 126 | set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] | ||
| 127 | assert {[lindex $addr 1] == $old_port} | ||
| 128 | kill_instance redis $master_id | ||
| 129 | |||
| 130 | # Make sure failover did not happened. | ||
| 131 | set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] | ||
| 132 | assert {[lindex $addr 1] == $old_port} | ||
| 133 | restart_instance redis $master_id | ||
| 134 | } | ||
| 135 | |||
| 136 | test "Failover is not possible without majority agreement" { | ||
| 137 | foreach_sentinel_id id { | ||
| 138 | S $id SENTINEL SET mymaster quorum $quorum | ||
| 139 | } | ||
| 140 | |||
| 141 | # Crash majority of sentinels | ||
| 142 | for {set id 0} {$id < $quorum} {incr id} { | ||
| 143 | kill_instance sentinel $id | ||
| 144 | } | ||
| 145 | |||
| 146 | # Kill the current master | ||
| 147 | kill_instance redis $master_id | ||
| 148 | |||
| 149 | # Make sure failover did not happened. | ||
| 150 | set addr [S $quorum SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] | ||
| 151 | assert {[lindex $addr 1] == $old_port} | ||
| 152 | restart_instance redis $master_id | ||
| 153 | |||
| 154 | # Cleanup: restart Sentinels to monitor the master. | ||
| 155 | for {set id 0} {$id < $quorum} {incr id} { | ||
| 156 | restart_instance sentinel $id | ||
| 157 | } | ||
| 158 | } | ||
| 159 | |||
| 160 | test "Failover works if we configure for absolute agreement" { | ||
| 161 | foreach_sentinel_id id { | ||
| 162 | S $id SENTINEL SET mymaster quorum $sentinels | ||
| 163 | } | ||
| 164 | |||
| 165 | # Wait for Sentinels to monitor the master again | ||
| 166 | foreach_sentinel_id id { | ||
| 167 | wait_for_condition 1000 100 { | ||
| 168 | [dict get [S $id SENTINEL MASTER mymaster] info-refresh] < 100000 | ||
| 169 | } else { | ||
| 170 | fail "At least one Sentinel is not monitoring the master" | ||
| 171 | } | ||
| 172 | } | ||
| 173 | |||
| 174 | kill_instance redis $master_id | ||
| 175 | |||
| 176 | foreach_sentinel_id id { | ||
| 177 | wait_for_condition 1000 100 { | ||
| 178 | [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port | ||
| 179 | } else { | ||
| 180 | fail "At least one Sentinel did not receive failover info" | ||
| 181 | } | ||
| 182 | } | ||
| 183 | restart_instance redis $master_id | ||
| 184 | set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] | ||
| 185 | set master_id [get_instance_id_by_port redis [lindex $addr 1]] | ||
| 186 | |||
| 187 | # Set the min ODOWN agreement back to strict majority. | ||
| 188 | foreach_sentinel_id id { | ||
| 189 | S $id SENTINEL SET mymaster quorum $quorum | ||
| 190 | } | ||
| 191 | } | ||
| 192 | |||
| 193 | test "New master [join $addr {:}] role matches" { | ||
| 194 | assert {[RI $master_id role] eq {master}} | ||
| 195 | } | ||
| 196 | |||
| 197 | test "SENTINEL RESET can resets the master" { | ||
| 198 | # After SENTINEL RESET, sometimes the sentinel can sense the master again, | ||
| 199 | # causing the test to fail. Here we give it a few more chances. | ||
| 200 | for {set j 0} {$j < 10} {incr j} { | ||
| 201 | assert_equal 1 [S 0 SENTINEL RESET mymaster] | ||
| 202 | set res1 [llength [S 0 SENTINEL SENTINELS mymaster]] | ||
| 203 | set res2 [llength [S 0 SENTINEL SLAVES mymaster]] | ||
| 204 | set res3 [llength [S 0 SENTINEL REPLICAS mymaster]] | ||
| 205 | if {$res1 eq 0 && $res2 eq 0 && $res3 eq 0} break | ||
| 206 | } | ||
| 207 | assert_equal 0 $res1 | ||
| 208 | assert_equal 0 $res2 | ||
| 209 | assert_equal 0 $res3 | ||
| 210 | } | ||
diff --git a/examples/redis-unstable/tests/sentinel/tests/01-conf-update.tcl b/examples/redis-unstable/tests/sentinel/tests/01-conf-update.tcl new file mode 100644 index 0000000..fe29bb0 --- /dev/null +++ b/examples/redis-unstable/tests/sentinel/tests/01-conf-update.tcl | |||
| @@ -0,0 +1,50 @@ | |||
| 1 | # Test Sentinel configuration consistency after partitions heal. | ||
| 2 | |||
| 3 | source "../tests/includes/init-tests.tcl" | ||
| 4 | |||
| 5 | test "We can failover with Sentinel 1 crashed" { | ||
| 6 | set old_port [RPort $master_id] | ||
| 7 | set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] | ||
| 8 | assert {[lindex $addr 1] == $old_port} | ||
| 9 | |||
| 10 | # Crash Sentinel 1 | ||
| 11 | kill_instance sentinel 1 | ||
| 12 | |||
| 13 | kill_instance redis $master_id | ||
| 14 | foreach_sentinel_id id { | ||
| 15 | if {$id != 1} { | ||
| 16 | wait_for_condition 1000 50 { | ||
| 17 | [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port | ||
| 18 | } else { | ||
| 19 | fail "Sentinel $id did not receive failover info" | ||
| 20 | } | ||
| 21 | } | ||
| 22 | } | ||
| 23 | restart_instance redis $master_id | ||
| 24 | set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] | ||
| 25 | set master_id [get_instance_id_by_port redis [lindex $addr 1]] | ||
| 26 | } | ||
| 27 | |||
| 28 | test "After Sentinel 1 is restarted, its config gets updated" { | ||
| 29 | restart_instance sentinel 1 | ||
| 30 | wait_for_condition 1000 50 { | ||
| 31 | [lindex [S 1 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port | ||
| 32 | } else { | ||
| 33 | fail "Restarted Sentinel did not receive failover info" | ||
| 34 | } | ||
| 35 | } | ||
| 36 | |||
| 37 | test "New master [join $addr {:}] role matches" { | ||
| 38 | assert {[RI $master_id role] eq {master}} | ||
| 39 | } | ||
| 40 | |||
| 41 | test "Update log level" { | ||
| 42 | set current_loglevel [S 0 SENTINEL CONFIG GET loglevel] | ||
| 43 | assert {[lindex $current_loglevel 1] == {notice}} | ||
| 44 | |||
| 45 | foreach {loglevel} {debug verbose notice warning nothing} { | ||
| 46 | S 0 SENTINEL CONFIG SET loglevel $loglevel | ||
| 47 | set updated_loglevel [S 0 SENTINEL CONFIG GET loglevel] | ||
| 48 | assert {[lindex $updated_loglevel 1] == $loglevel} | ||
| 49 | } | ||
| 50 | } | ||
diff --git a/examples/redis-unstable/tests/sentinel/tests/02-slaves-reconf.tcl b/examples/redis-unstable/tests/sentinel/tests/02-slaves-reconf.tcl new file mode 100644 index 0000000..8196b60 --- /dev/null +++ b/examples/redis-unstable/tests/sentinel/tests/02-slaves-reconf.tcl | |||
| @@ -0,0 +1,91 @@ | |||
| 1 | # Check that slaves are reconfigured at a latter time if they are partitioned. | ||
| 2 | # | ||
| 3 | # Here we should test: | ||
| 4 | # 1) That slaves point to the new master after failover. | ||
| 5 | # 2) That partitioned slaves point to new master when they are partitioned | ||
| 6 | # away during failover and return at a latter time. | ||
| 7 | |||
| 8 | source "../tests/includes/init-tests.tcl" | ||
| 9 | |||
| 10 | proc 02_test_slaves_replication {} { | ||
| 11 | uplevel 1 { | ||
| 12 | test "Check that slaves replicate from current master" { | ||
| 13 | set master_port [RPort $master_id] | ||
| 14 | foreach_redis_id id { | ||
| 15 | if {$id == $master_id} continue | ||
| 16 | if {[instance_is_killed redis $id]} continue | ||
| 17 | wait_for_condition 1000 50 { | ||
| 18 | ([RI $id master_port] == $master_port) && | ||
| 19 | ([RI $id master_link_status] eq {up}) | ||
| 20 | } else { | ||
| 21 | fail "Redis slave $id is replicating from wrong master" | ||
| 22 | } | ||
| 23 | } | ||
| 24 | } | ||
| 25 | } | ||
| 26 | } | ||
| 27 | |||
| 28 | proc 02_crash_and_failover {} { | ||
| 29 | uplevel 1 { | ||
| 30 | test "Crash the master and force a failover" { | ||
| 31 | set old_port [RPort $master_id] | ||
| 32 | set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] | ||
| 33 | assert {[lindex $addr 1] == $old_port} | ||
| 34 | kill_instance redis $master_id | ||
| 35 | foreach_sentinel_id id { | ||
| 36 | wait_for_condition 1000 50 { | ||
| 37 | [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port | ||
| 38 | } else { | ||
| 39 | fail "At least one Sentinel did not receive failover info" | ||
| 40 | } | ||
| 41 | } | ||
| 42 | restart_instance redis $master_id | ||
| 43 | set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] | ||
| 44 | set master_id [get_instance_id_by_port redis [lindex $addr 1]] | ||
| 45 | } | ||
| 46 | } | ||
| 47 | } | ||
| 48 | |||
| 49 | 02_test_slaves_replication | ||
| 50 | 02_crash_and_failover | ||
| 51 | |||
| 52 | foreach_sentinel_id id { | ||
| 53 | S $id sentinel debug info-period 100 | ||
| 54 | S $id sentinel debug default-down-after 1000 | ||
| 55 | S $id sentinel debug publish-period 100 | ||
| 56 | } | ||
| 57 | |||
| 58 | 02_test_slaves_replication | ||
| 59 | |||
| 60 | test "Kill a slave instance" { | ||
| 61 | foreach_redis_id id { | ||
| 62 | if {$id == $master_id} continue | ||
| 63 | set killed_slave_id $id | ||
| 64 | kill_instance redis $id | ||
| 65 | break | ||
| 66 | } | ||
| 67 | } | ||
| 68 | |||
| 69 | 02_crash_and_failover | ||
| 70 | 02_test_slaves_replication | ||
| 71 | |||
| 72 | test "Wait for failover to end" { | ||
| 73 | set inprogress 1 | ||
| 74 | while {$inprogress} { | ||
| 75 | set inprogress 0 | ||
| 76 | foreach_sentinel_id id { | ||
| 77 | if {[dict exists [S $id SENTINEL MASTER mymaster] failover-state]} { | ||
| 78 | incr inprogress | ||
| 79 | } | ||
| 80 | } | ||
| 81 | if {$inprogress} {after 100} | ||
| 82 | } | ||
| 83 | } | ||
| 84 | |||
| 85 | test "Restart killed slave and test replication of slaves again..." { | ||
| 86 | restart_instance redis $killed_slave_id | ||
| 87 | } | ||
| 88 | |||
| 89 | # Now we check if the slave rejoining the partition is reconfigured even | ||
| 90 | # if the failover finished. | ||
| 91 | 02_test_slaves_replication | ||
diff --git a/examples/redis-unstable/tests/sentinel/tests/03-runtime-reconf.tcl b/examples/redis-unstable/tests/sentinel/tests/03-runtime-reconf.tcl new file mode 100644 index 0000000..bd6eecc --- /dev/null +++ b/examples/redis-unstable/tests/sentinel/tests/03-runtime-reconf.tcl | |||
| @@ -0,0 +1,225 @@ | |||
| 1 | # Test runtime reconfiguration command SENTINEL SET. | ||
| 2 | source "../tests/includes/init-tests.tcl" | ||
| 3 | set num_sentinels [llength $::sentinel_instances] | ||
| 4 | |||
| 5 | set ::user "testuser" | ||
| 6 | set ::password "secret" | ||
| 7 | |||
| 8 | proc server_set_password {} { | ||
| 9 | foreach_redis_id id { | ||
| 10 | assert_equal {OK} [R $id CONFIG SET requirepass $::password] | ||
| 11 | assert_equal {OK} [R $id AUTH $::password] | ||
| 12 | assert_equal {OK} [R $id CONFIG SET masterauth $::password] | ||
| 13 | } | ||
| 14 | } | ||
| 15 | |||
| 16 | proc server_reset_password {} { | ||
| 17 | foreach_redis_id id { | ||
| 18 | assert_equal {OK} [R $id CONFIG SET requirepass ""] | ||
| 19 | assert_equal {OK} [R $id CONFIG SET masterauth ""] | ||
| 20 | } | ||
| 21 | } | ||
| 22 | |||
| 23 | proc server_set_acl {id} { | ||
| 24 | assert_equal {OK} [R $id ACL SETUSER $::user on >$::password allchannels +@all] | ||
| 25 | assert_equal {OK} [R $id ACL SETUSER default off] | ||
| 26 | |||
| 27 | R $id CLIENT KILL USER default SKIPME no | ||
| 28 | assert_equal {OK} [R $id AUTH $::user $::password] | ||
| 29 | assert_equal {OK} [R $id CONFIG SET masteruser $::user] | ||
| 30 | assert_equal {OK} [R $id CONFIG SET masterauth $::password] | ||
| 31 | } | ||
| 32 | |||
| 33 | proc server_reset_acl {id} { | ||
| 34 | assert_equal {OK} [R $id ACL SETUSER default on] | ||
| 35 | assert_equal {1} [R $id ACL DELUSER $::user] | ||
| 36 | |||
| 37 | assert_equal {OK} [R $id CONFIG SET masteruser ""] | ||
| 38 | assert_equal {OK} [R $id CONFIG SET masterauth ""] | ||
| 39 | } | ||
| 40 | |||
| 41 | proc verify_sentinel_connect_replicas {id} { | ||
| 42 | foreach replica [S $id SENTINEL REPLICAS mymaster] { | ||
| 43 | if {[string match "*disconnected*" [dict get $replica flags]]} { | ||
| 44 | return 0 | ||
| 45 | } | ||
| 46 | } | ||
| 47 | return 1 | ||
| 48 | } | ||
| 49 | |||
| 50 | proc wait_for_sentinels_connect_servers { {is_connect 1} } { | ||
| 51 | foreach_sentinel_id id { | ||
| 52 | wait_for_condition 1000 50 { | ||
| 53 | [string match "*disconnected*" [dict get [S $id SENTINEL MASTER mymaster] flags]] != $is_connect | ||
| 54 | } else { | ||
| 55 | fail "At least some sentinel can't connect to master" | ||
| 56 | } | ||
| 57 | |||
| 58 | wait_for_condition 1000 50 { | ||
| 59 | [verify_sentinel_connect_replicas $id] == $is_connect | ||
| 60 | } else { | ||
| 61 | fail "At least some sentinel can't connect to replica" | ||
| 62 | } | ||
| 63 | } | ||
| 64 | } | ||
| 65 | |||
| 66 | test "Sentinels (re)connection following SENTINEL SET mymaster auth-pass" { | ||
| 67 | # 3 types of sentinels to test: | ||
| 68 | # (re)started while master changed pwd. Manage to connect only after setting pwd | ||
| 69 | set sent2re 0 | ||
| 70 | # (up)dated in advance with master new password | ||
| 71 | set sent2up 1 | ||
| 72 | # (un)touched. Yet manage to maintain (old) connection | ||
| 73 | set sent2un 2 | ||
| 74 | |||
| 75 | wait_for_sentinels_connect_servers | ||
| 76 | kill_instance sentinel $sent2re | ||
| 77 | server_set_password | ||
| 78 | assert_equal {OK} [S $sent2up SENTINEL SET mymaster auth-pass $::password] | ||
| 79 | restart_instance sentinel $sent2re | ||
| 80 | |||
| 81 | # Verify sentinel that restarted failed to connect master | ||
| 82 | wait_for_condition 100 50 { | ||
| 83 | [string match "*disconnected*" [dict get [S $sent2re SENTINEL MASTER mymaster] flags]] != 0 | ||
| 84 | } else { | ||
| 85 | fail "Expected to be disconnected from master due to wrong password" | ||
| 86 | } | ||
| 87 | |||
| 88 | # Update restarted sentinel with master password | ||
| 89 | assert_equal {OK} [S $sent2re SENTINEL SET mymaster auth-pass $::password] | ||
| 90 | |||
| 91 | # All sentinels expected to connect successfully | ||
| 92 | wait_for_sentinels_connect_servers | ||
| 93 | |||
| 94 | # remove requirepass and verify sentinels manage to connect servers | ||
| 95 | server_reset_password | ||
| 96 | wait_for_sentinels_connect_servers | ||
| 97 | # Sanity check | ||
| 98 | verify_sentinel_auto_discovery | ||
| 99 | } | ||
| 100 | |||
| 101 | test "Sentinels (re)connection following master ACL change" { | ||
| 102 | # Three types of sentinels to test during ACL change: | ||
| 103 | # 1. (re)started Sentinel. Manage to connect only after setting new pwd | ||
| 104 | # 2. (up)dated Sentinel, get just before ACL change the new password | ||
| 105 | # 3. (un)touched Sentinel that kept old connection with master and didn't | ||
| 106 | # set new ACL password won't persist ACL pwd change (unlike legacy auth-pass) | ||
| 107 | set sent2re 0 | ||
| 108 | set sent2up 1 | ||
| 109 | set sent2un 2 | ||
| 110 | |||
| 111 | wait_for_sentinels_connect_servers | ||
| 112 | # kill sentinel 'sent2re' and restart it after ACL change | ||
| 113 | kill_instance sentinel $sent2re | ||
| 114 | |||
| 115 | # Update sentinel 'sent2up' with new user and pwd | ||
| 116 | assert_equal {OK} [S $sent2up SENTINEL SET mymaster auth-user $::user] | ||
| 117 | assert_equal {OK} [S $sent2up SENTINEL SET mymaster auth-pass $::password] | ||
| 118 | |||
| 119 | foreach_redis_id id { | ||
| 120 | server_set_acl $id | ||
| 121 | } | ||
| 122 | |||
| 123 | restart_instance sentinel $sent2re | ||
| 124 | |||
| 125 | # Verify sentinel that restarted failed to reconnect master | ||
| 126 | wait_for_condition 100 50 { | ||
| 127 | [string match "*disconnected*" [dict get [S $sent2re SENTINEL MASTER mymaster] flags]] != 0 | ||
| 128 | } else { | ||
| 129 | fail "Expected: Restarted sentinel to be disconnected from master due to obsolete password" | ||
| 130 | } | ||
| 131 | |||
| 132 | # Verify sentinel with updated password managed to connect (wait for sentinelTimer to reconnect) | ||
| 133 | wait_for_condition 100 50 { | ||
| 134 | [string match "*disconnected*" [dict get [S $sent2up SENTINEL MASTER mymaster] flags]] == 0 | ||
| 135 | } else { | ||
| 136 | fail "Expected: Sentinel to be connected to master" | ||
| 137 | } | ||
| 138 | |||
| 139 | # Verify sentinel untouched gets failed to connect master | ||
| 140 | wait_for_condition 100 50 { | ||
| 141 | [string match "*disconnected*" [dict get [S $sent2un SENTINEL MASTER mymaster] flags]] != 0 | ||
| 142 | } else { | ||
| 143 | fail "Expected: Sentinel to be disconnected from master due to obsolete password" | ||
| 144 | } | ||
| 145 | |||
| 146 | # Now update all sentinels with new password | ||
| 147 | foreach_sentinel_id id { | ||
| 148 | assert_equal {OK} [S $id SENTINEL SET mymaster auth-user $::user] | ||
| 149 | assert_equal {OK} [S $id SENTINEL SET mymaster auth-pass $::password] | ||
| 150 | } | ||
| 151 | |||
| 152 | # All sentinels expected to connect successfully | ||
| 153 | wait_for_sentinels_connect_servers | ||
| 154 | |||
| 155 | # remove requirepass and verify sentinels manage to connect servers | ||
| 156 | foreach_redis_id id { | ||
| 157 | server_reset_acl $id | ||
| 158 | } | ||
| 159 | |||
| 160 | wait_for_sentinels_connect_servers | ||
| 161 | # Sanity check | ||
| 162 | verify_sentinel_auto_discovery | ||
| 163 | } | ||
| 164 | |||
| 165 | test "Set parameters in normal case" { | ||
| 166 | |||
| 167 | set info [S 0 SENTINEL master mymaster] | ||
| 168 | set origin_quorum [dict get $info quorum] | ||
| 169 | set origin_down_after_milliseconds [dict get $info down-after-milliseconds] | ||
| 170 | set update_quorum [expr $origin_quorum+1] | ||
| 171 | set update_down_after_milliseconds [expr $origin_down_after_milliseconds+1000] | ||
| 172 | |||
| 173 | assert_equal [S 0 SENTINEL SET mymaster quorum $update_quorum] "OK" | ||
| 174 | assert_equal [S 0 SENTINEL SET mymaster down-after-milliseconds $update_down_after_milliseconds] "OK" | ||
| 175 | |||
| 176 | set update_info [S 0 SENTINEL master mymaster] | ||
| 177 | assert {[dict get $update_info quorum] != $origin_quorum} | ||
| 178 | assert {[dict get $update_info down-after-milliseconds] != $origin_down_after_milliseconds} | ||
| 179 | |||
| 180 | #restore to origin config parameters | ||
| 181 | assert_equal [S 0 SENTINEL SET mymaster quorum $origin_quorum] "OK" | ||
| 182 | assert_equal [S 0 SENTINEL SET mymaster down-after-milliseconds $origin_down_after_milliseconds] "OK" | ||
| 183 | } | ||
| 184 | |||
| 185 | test "Set parameters in normal case with bad format" { | ||
| 186 | |||
| 187 | set info [S 0 SENTINEL master mymaster] | ||
| 188 | set origin_down_after_milliseconds [dict get $info down-after-milliseconds] | ||
| 189 | |||
| 190 | assert_error "ERR Invalid argument '-20' for SENTINEL SET 'down-after-milliseconds'*" {S 0 SENTINEL SET mymaster down-after-milliseconds -20} | ||
| 191 | assert_error "ERR Invalid argument 'abc' for SENTINEL SET 'down-after-milliseconds'*" {S 0 SENTINEL SET mymaster down-after-milliseconds "abc"} | ||
| 192 | |||
| 193 | set current_info [S 0 SENTINEL master mymaster] | ||
| 194 | assert {[dict get $current_info down-after-milliseconds] == $origin_down_after_milliseconds} | ||
| 195 | } | ||
| 196 | |||
| 197 | test "Sentinel Set with other error situations" { | ||
| 198 | |||
| 199 | # non-existing script | ||
| 200 | assert_error "ERR Notification script seems non existing*" {S 0 SENTINEL SET mymaster notification-script test.txt} | ||
| 201 | |||
| 202 | # wrong parameter number | ||
| 203 | assert_error "ERR wrong number of arguments for 'sentinel|set' command" {S 0 SENTINEL SET mymaster fakeoption} | ||
| 204 | |||
| 205 | # unknown parameter option | ||
| 206 | assert_error "ERR Unknown option or number of arguments for SENTINEL SET 'fakeoption'" {S 0 SENTINEL SET mymaster fakeoption fakevalue} | ||
| 207 | |||
| 208 | # save new config to disk failed | ||
| 209 | set info [S 0 SENTINEL master mymaster] | ||
| 210 | set origin_quorum [dict get $info quorum] | ||
| 211 | set update_quorum [expr $origin_quorum+1] | ||
| 212 | set sentinel_id 0 | ||
| 213 | set configfilename [file join "sentinel_$sentinel_id" "sentinel.conf"] | ||
| 214 | set configfilename_bak [file join "sentinel_$sentinel_id" "sentinel.conf.bak"] | ||
| 215 | |||
| 216 | file rename $configfilename $configfilename_bak | ||
| 217 | file mkdir $configfilename | ||
| 218 | |||
| 219 | catch {[S 0 SENTINEL SET mymaster quorum $update_quorum]} err | ||
| 220 | |||
| 221 | file delete $configfilename | ||
| 222 | file rename $configfilename_bak $configfilename | ||
| 223 | |||
| 224 | assert_match "ERR Failed to save config file*" $err | ||
| 225 | } | ||
diff --git a/examples/redis-unstable/tests/sentinel/tests/04-slave-selection.tcl b/examples/redis-unstable/tests/sentinel/tests/04-slave-selection.tcl new file mode 100644 index 0000000..3d2ca64 --- /dev/null +++ b/examples/redis-unstable/tests/sentinel/tests/04-slave-selection.tcl | |||
| @@ -0,0 +1,5 @@ | |||
| 1 | # Test slave selection algorithm. | ||
| 2 | # | ||
| 3 | # This unit should test: | ||
| 4 | # 1) That when there are no suitable slaves no failover is performed. | ||
| 5 | # 2) That among the available slaves, the one with better offset is picked. | ||
diff --git a/examples/redis-unstable/tests/sentinel/tests/05-manual.tcl b/examples/redis-unstable/tests/sentinel/tests/05-manual.tcl new file mode 100644 index 0000000..95e8d41 --- /dev/null +++ b/examples/redis-unstable/tests/sentinel/tests/05-manual.tcl | |||
| @@ -0,0 +1,94 @@ | |||
| 1 | # Test manual failover | ||
| 2 | |||
| 3 | source "../tests/includes/init-tests.tcl" | ||
| 4 | |||
| 5 | foreach_sentinel_id id { | ||
| 6 | S $id sentinel debug info-period 2000 | ||
| 7 | S $id sentinel debug default-down-after 6000 | ||
| 8 | S $id sentinel debug publish-period 1000 | ||
| 9 | } | ||
| 10 | |||
| 11 | test "Manual failover works" { | ||
| 12 | set old_port [RPort $master_id] | ||
| 13 | set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] | ||
| 14 | assert {[lindex $addr 1] == $old_port} | ||
| 15 | |||
| 16 | # Since we reduced the info-period (default 10000) above immediately, | ||
| 17 | # sentinel - replica may not have enough time to exchange INFO and update | ||
| 18 | # the replica's info-period, so the test may get a NOGOODSLAVE. | ||
| 19 | wait_for_condition 300 50 { | ||
| 20 | [catch {S 0 SENTINEL FAILOVER mymaster}] == 0 | ||
| 21 | } else { | ||
| 22 | catch {S 0 SENTINEL FAILOVER mymaster} reply | ||
| 23 | puts [S 0 SENTINEL REPLICAS mymaster] | ||
| 24 | fail "Sentinel manual failover did not work, got: $reply" | ||
| 25 | } | ||
| 26 | |||
| 27 | catch {S 0 SENTINEL FAILOVER mymaster} reply | ||
| 28 | assert_match {*INPROG*} $reply ;# Failover already in progress | ||
| 29 | |||
| 30 | foreach_sentinel_id id { | ||
| 31 | wait_for_condition 1000 50 { | ||
| 32 | [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port | ||
| 33 | } else { | ||
| 34 | fail "At least one Sentinel did not receive failover info" | ||
| 35 | } | ||
| 36 | } | ||
| 37 | set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] | ||
| 38 | set master_id [get_instance_id_by_port redis [lindex $addr 1]] | ||
| 39 | } | ||
| 40 | |||
| 41 | test "New master [join $addr {:}] role matches" { | ||
| 42 | assert {[RI $master_id role] eq {master}} | ||
| 43 | } | ||
| 44 | |||
| 45 | test "All the other slaves now point to the new master" { | ||
| 46 | foreach_redis_id id { | ||
| 47 | if {$id != $master_id && $id != 0} { | ||
| 48 | wait_for_condition 1000 50 { | ||
| 49 | [RI $id master_port] == [lindex $addr 1] | ||
| 50 | } else { | ||
| 51 | fail "Redis ID $id not configured to replicate with new master" | ||
| 52 | } | ||
| 53 | } | ||
| 54 | } | ||
| 55 | } | ||
| 56 | |||
| 57 | test "The old master eventually gets reconfigured as a slave" { | ||
| 58 | wait_for_condition 1000 50 { | ||
| 59 | [RI 0 master_port] == [lindex $addr 1] | ||
| 60 | } else { | ||
| 61 | fail "Old master not reconfigured as slave of new master" | ||
| 62 | } | ||
| 63 | } | ||
| 64 | |||
| 65 | foreach flag {crash-after-election crash-after-promotion} { | ||
| 66 | # Before each SIMULATE-FAILURE test, re-source init-tests to get a clean environment | ||
| 67 | source "../tests/includes/init-tests.tcl" | ||
| 68 | |||
| 69 | test "SENTINEL SIMULATE-FAILURE $flag works" { | ||
| 70 | assert_equal {OK} [S 0 SENTINEL SIMULATE-FAILURE $flag] | ||
| 71 | |||
| 72 | # Trigger a failover, failover will trigger leader election, replica promotion | ||
| 73 | # Sentinel may enter failover and exit before the command, catch it and allow it | ||
| 74 | wait_for_condition 300 50 { | ||
| 75 | [catch {S 0 SENTINEL FAILOVER mymaster}] == 0 | ||
| 76 | || | ||
| 77 | ([catch {S 0 SENTINEL FAILOVER mymaster} reply] == 1 && | ||
| 78 | [string match {*couldn't open socket: connection refused*} $reply]) | ||
| 79 | } else { | ||
| 80 | catch {S 0 SENTINEL FAILOVER mymaster} reply | ||
| 81 | fail "Sentinel manual failover did not work, got: $reply" | ||
| 82 | } | ||
| 83 | |||
| 84 | # Wait for sentinel to exit (due to simulate-failure flags) | ||
| 85 | wait_for_condition 1000 50 { | ||
| 86 | [catch {S 0 PING}] == 1 | ||
| 87 | } else { | ||
| 88 | fail "Sentinel set $flag but did not exit" | ||
| 89 | } | ||
| 90 | assert_error {*couldn't open socket: connection refused*} {S 0 PING} | ||
| 91 | |||
| 92 | restart_instance sentinel 0 | ||
| 93 | } | ||
| 94 | } | ||
diff --git a/examples/redis-unstable/tests/sentinel/tests/06-ckquorum.tcl b/examples/redis-unstable/tests/sentinel/tests/06-ckquorum.tcl new file mode 100644 index 0000000..36c3dc6 --- /dev/null +++ b/examples/redis-unstable/tests/sentinel/tests/06-ckquorum.tcl | |||
| @@ -0,0 +1,42 @@ | |||
| 1 | # Test for the SENTINEL CKQUORUM command | ||
| 2 | |||
| 3 | source "../tests/includes/init-tests.tcl" | ||
| 4 | set num_sentinels [llength $::sentinel_instances] | ||
| 5 | |||
| 6 | test "CKQUORUM reports OK and the right amount of Sentinels" { | ||
| 7 | foreach_sentinel_id id { | ||
| 8 | assert_match "*OK $num_sentinels usable*" [S $id SENTINEL CKQUORUM mymaster] | ||
| 9 | } | ||
| 10 | } | ||
| 11 | |||
| 12 | test "CKQUORUM detects quorum cannot be reached" { | ||
| 13 | set orig_quorum [expr {$num_sentinels/2+1}] | ||
| 14 | S 0 SENTINEL SET mymaster quorum [expr {$num_sentinels+1}] | ||
| 15 | catch {[S 0 SENTINEL CKQUORUM mymaster]} err | ||
| 16 | assert_match "*NOQUORUM*" $err | ||
| 17 | S 0 SENTINEL SET mymaster quorum $orig_quorum | ||
| 18 | } | ||
| 19 | |||
| 20 | test "CKQUORUM detects failover authorization cannot be reached" { | ||
| 21 | set orig_quorum [expr {$num_sentinels/2+1}] | ||
| 22 | S 0 SENTINEL SET mymaster quorum 1 | ||
| 23 | for {set i 0} {$i < $orig_quorum} {incr i} { | ||
| 24 | kill_instance sentinel [expr {$i + 1}] | ||
| 25 | } | ||
| 26 | |||
| 27 | # We need to make sure that other sentinels are in `DOWN` state | ||
| 28 | # from the point of view of S 0 before we executing `CKQUORUM`. | ||
| 29 | wait_for_condition 300 50 { | ||
| 30 | [catch {S 0 SENTINEL CKQUORUM mymaster}] == 1 | ||
| 31 | } else { | ||
| 32 | fail "At least $orig_quorum sentinels did not enter the down state." | ||
| 33 | } | ||
| 34 | |||
| 35 | assert_error "*NOQUORUM*" {S 0 SENTINEL CKQUORUM mymaster} | ||
| 36 | |||
| 37 | S 0 SENTINEL SET mymaster quorum $orig_quorum | ||
| 38 | for {set i 0} {$i < $orig_quorum} {incr i} { | ||
| 39 | restart_instance sentinel [expr {$i + 1}] | ||
| 40 | } | ||
| 41 | } | ||
| 42 | |||
diff --git a/examples/redis-unstable/tests/sentinel/tests/07-down-conditions.tcl b/examples/redis-unstable/tests/sentinel/tests/07-down-conditions.tcl new file mode 100644 index 0000000..dabbc14 --- /dev/null +++ b/examples/redis-unstable/tests/sentinel/tests/07-down-conditions.tcl | |||
| @@ -0,0 +1,104 @@ | |||
| 1 | # Test conditions where an instance is considered to be down | ||
| 2 | |||
| 3 | source "../tests/includes/init-tests.tcl" | ||
| 4 | source "../../../tests/support/cli.tcl" | ||
| 5 | |||
| 6 | foreach_sentinel_id id { | ||
| 7 | S $id sentinel debug info-period 1000 | ||
| 8 | S $id sentinel debug ask-period 100 | ||
| 9 | S $id sentinel debug default-down-after 3000 | ||
| 10 | S $id sentinel debug publish-period 200 | ||
| 11 | S $id sentinel debug ping-period 100 | ||
| 12 | } | ||
| 13 | |||
| 14 | set ::alive_sentinel [expr {$::instances_count/2+2}] | ||
| 15 | proc ensure_master_up {} { | ||
| 16 | S $::alive_sentinel sentinel debug info-period 1000 | ||
| 17 | S $::alive_sentinel sentinel debug ping-period 100 | ||
| 18 | S $::alive_sentinel sentinel debug ask-period 100 | ||
| 19 | S $::alive_sentinel sentinel debug publish-period 100 | ||
| 20 | wait_for_condition 1000 50 { | ||
| 21 | [dict get [S $::alive_sentinel sentinel master mymaster] flags] eq "master" | ||
| 22 | } else { | ||
| 23 | fail "Master flags are not just 'master'" | ||
| 24 | } | ||
| 25 | } | ||
| 26 | |||
| 27 | proc ensure_master_down {} { | ||
| 28 | S $::alive_sentinel sentinel debug info-period 1000 | ||
| 29 | S $::alive_sentinel sentinel debug ping-period 100 | ||
| 30 | S $::alive_sentinel sentinel debug ask-period 100 | ||
| 31 | S $::alive_sentinel sentinel debug publish-period 100 | ||
| 32 | wait_for_condition 1000 50 { | ||
| 33 | [string match *down* \ | ||
| 34 | [dict get [S $::alive_sentinel sentinel master mymaster] flags]] | ||
| 35 | } else { | ||
| 36 | fail "Master is not flagged SDOWN" | ||
| 37 | } | ||
| 38 | } | ||
| 39 | |||
| 40 | test "Crash the majority of Sentinels to prevent failovers for this unit" { | ||
| 41 | for {set id 0} {$id < $quorum} {incr id} { | ||
| 42 | kill_instance sentinel $id | ||
| 43 | } | ||
| 44 | } | ||
| 45 | |||
| 46 | test "SDOWN is triggered by non-responding but not crashed instance" { | ||
| 47 | ensure_master_up | ||
| 48 | set master_addr [S $::alive_sentinel SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] | ||
| 49 | set master_id [get_instance_id_by_port redis [lindex $master_addr 1]] | ||
| 50 | |||
| 51 | set pid [get_instance_attrib redis $master_id pid] | ||
| 52 | pause_process $pid | ||
| 53 | ensure_master_down | ||
| 54 | resume_process $pid | ||
| 55 | ensure_master_up | ||
| 56 | } | ||
| 57 | |||
| 58 | test "SDOWN is triggered by crashed instance" { | ||
| 59 | lassign [S $::alive_sentinel SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] host port | ||
| 60 | ensure_master_up | ||
| 61 | kill_instance redis 0 | ||
| 62 | ensure_master_down | ||
| 63 | restart_instance redis 0 | ||
| 64 | ensure_master_up | ||
| 65 | } | ||
| 66 | |||
| 67 | test "SDOWN is triggered by masters advertising as slaves" { | ||
| 68 | ensure_master_up | ||
| 69 | R 0 slaveof 127.0.0.1 34567 | ||
| 70 | ensure_master_down | ||
| 71 | R 0 slaveof no one | ||
| 72 | ensure_master_up | ||
| 73 | } | ||
| 74 | |||
| 75 | if {!$::log_req_res} { # this test changes 'dir' config to '/' and logreqres.c cannot open protocol dump file under the root directory. | ||
| 76 | test "SDOWN is triggered by misconfigured instance replying with errors" { | ||
| 77 | ensure_master_up | ||
| 78 | set orig_dir [lindex [R 0 config get dir] 1] | ||
| 79 | set orig_save [lindex [R 0 config get save] 1] | ||
| 80 | # Set dir to / and filename to "tmp" to make sure it will fail. | ||
| 81 | R 0 config set dir / | ||
| 82 | R 0 config set dbfilename tmp | ||
| 83 | R 0 config set save "1000000 1000000" | ||
| 84 | after 5000 | ||
| 85 | R 0 bgsave | ||
| 86 | after 5000 | ||
| 87 | ensure_master_down | ||
| 88 | R 0 config set save $orig_save | ||
| 89 | R 0 config set dir $orig_dir | ||
| 90 | R 0 config set dbfilename dump.rdb | ||
| 91 | R 0 bgsave | ||
| 92 | ensure_master_up | ||
| 93 | } | ||
| 94 | } | ||
| 95 | |||
| 96 | # We use this test setup to also test command renaming, as a side | ||
| 97 | # effect of the master going down if we send PONG instead of PING | ||
| 98 | test "SDOWN is triggered if we rename PING to PONG" { | ||
| 99 | ensure_master_up | ||
| 100 | S $::alive_sentinel SENTINEL SET mymaster rename-command PING PONG | ||
| 101 | ensure_master_down | ||
| 102 | S $::alive_sentinel SENTINEL SET mymaster rename-command PING PING | ||
| 103 | ensure_master_up | ||
| 104 | } | ||
diff --git a/examples/redis-unstable/tests/sentinel/tests/08-hostname-conf.tcl b/examples/redis-unstable/tests/sentinel/tests/08-hostname-conf.tcl new file mode 100644 index 0000000..263b06f --- /dev/null +++ b/examples/redis-unstable/tests/sentinel/tests/08-hostname-conf.tcl | |||
| @@ -0,0 +1,69 @@ | |||
| 1 | source "../tests/includes/utils.tcl" | ||
| 2 | |||
| 3 | proc set_redis_announce_ip {addr} { | ||
| 4 | foreach_redis_id id { | ||
| 5 | R $id config set replica-announce-ip $addr | ||
| 6 | } | ||
| 7 | } | ||
| 8 | |||
| 9 | proc set_sentinel_config {keyword value} { | ||
| 10 | foreach_sentinel_id id { | ||
| 11 | S $id sentinel config set $keyword $value | ||
| 12 | } | ||
| 13 | } | ||
| 14 | |||
| 15 | proc set_all_instances_hostname {hostname} { | ||
| 16 | foreach_sentinel_id id { | ||
| 17 | set_instance_attrib sentinel $id host $hostname | ||
| 18 | } | ||
| 19 | foreach_redis_id id { | ||
| 20 | set_instance_attrib redis $id host $hostname | ||
| 21 | } | ||
| 22 | } | ||
| 23 | |||
| 24 | test "(pre-init) Configure instances and sentinel for hostname use" { | ||
| 25 | set ::host "localhost" | ||
| 26 | restart_killed_instances | ||
| 27 | set_all_instances_hostname $::host | ||
| 28 | set_redis_announce_ip $::host | ||
| 29 | set_sentinel_config resolve-hostnames yes | ||
| 30 | set_sentinel_config announce-hostnames yes | ||
| 31 | } | ||
| 32 | |||
| 33 | source "../tests/includes/init-tests.tcl" | ||
| 34 | |||
| 35 | proc verify_hostname_announced {hostname} { | ||
| 36 | foreach_sentinel_id id { | ||
| 37 | # Master is reported with its hostname | ||
| 38 | if {![string equal [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 0] $hostname]} { | ||
| 39 | return 0 | ||
| 40 | } | ||
| 41 | |||
| 42 | # Replicas are reported with their hostnames | ||
| 43 | foreach replica [S $id SENTINEL REPLICAS mymaster] { | ||
| 44 | if {![string equal [dict get $replica ip] $hostname]} { | ||
| 45 | return 0 | ||
| 46 | } | ||
| 47 | } | ||
| 48 | } | ||
| 49 | return 1 | ||
| 50 | } | ||
| 51 | |||
| 52 | test "Sentinel announces hostnames" { | ||
| 53 | # Check initial state | ||
| 54 | verify_hostname_announced $::host | ||
| 55 | |||
| 56 | # Disable announce-hostnames and confirm IPs are used | ||
| 57 | set_sentinel_config announce-hostnames no | ||
| 58 | assert {[verify_hostname_announced "127.0.0.1"] || [verify_hostname_announced "::1"]} | ||
| 59 | } | ||
| 60 | |||
| 61 | # We need to revert any special configuration because all tests currently | ||
| 62 | # share the same instances. | ||
| 63 | test "(post-cleanup) Configure instances and sentinel for IPs" { | ||
| 64 | set ::host "127.0.0.1" | ||
| 65 | set_all_instances_hostname $::host | ||
| 66 | set_redis_announce_ip $::host | ||
| 67 | set_sentinel_config resolve-hostnames no | ||
| 68 | set_sentinel_config announce-hostnames no | ||
| 69 | } \ No newline at end of file | ||
diff --git a/examples/redis-unstable/tests/sentinel/tests/09-acl-support.tcl b/examples/redis-unstable/tests/sentinel/tests/09-acl-support.tcl new file mode 100644 index 0000000..a754dac --- /dev/null +++ b/examples/redis-unstable/tests/sentinel/tests/09-acl-support.tcl | |||
| @@ -0,0 +1,56 @@ | |||
| 1 | |||
| 2 | source "../tests/includes/init-tests.tcl" | ||
| 3 | |||
| 4 | set ::user "testuser" | ||
| 5 | set ::password "secret" | ||
| 6 | |||
| 7 | proc setup_acl {} { | ||
| 8 | foreach_sentinel_id id { | ||
| 9 | assert_equal {OK} [S $id ACL SETUSER $::user >$::password +@all on] | ||
| 10 | assert_equal {OK} [S $id ACL SETUSER default off] | ||
| 11 | |||
| 12 | S $id CLIENT KILL USER default SKIPME no | ||
| 13 | assert_equal {OK} [S $id AUTH $::user $::password] | ||
| 14 | } | ||
| 15 | } | ||
| 16 | |||
| 17 | proc teardown_acl {} { | ||
| 18 | foreach_sentinel_id id { | ||
| 19 | assert_equal {OK} [S $id ACL SETUSER default on] | ||
| 20 | assert_equal {1} [S $id ACL DELUSER $::user] | ||
| 21 | |||
| 22 | S $id SENTINEL CONFIG SET sentinel-user "" | ||
| 23 | S $id SENTINEL CONFIG SET sentinel-pass "" | ||
| 24 | } | ||
| 25 | } | ||
| 26 | |||
| 27 | test "(post-init) Set up ACL configuration" { | ||
| 28 | setup_acl | ||
| 29 | assert_equal $::user [S 1 ACL WHOAMI] | ||
| 30 | } | ||
| 31 | |||
| 32 | test "SENTINEL CONFIG SET handles on-the-fly credentials reconfiguration" { | ||
| 33 | # Make sure we're starting with a broken state... | ||
| 34 | wait_for_condition 200 50 { | ||
| 35 | [catch {S 1 SENTINEL CKQUORUM mymaster}] == 1 | ||
| 36 | } else { | ||
| 37 | fail "Expected: Sentinel to be disconnected from master due to wrong password" | ||
| 38 | } | ||
| 39 | assert_error "*NOQUORUM*" {S 1 SENTINEL CKQUORUM mymaster} | ||
| 40 | |||
| 41 | foreach_sentinel_id id { | ||
| 42 | assert_equal {OK} [S $id SENTINEL CONFIG SET sentinel-user $::user] | ||
| 43 | assert_equal {OK} [S $id SENTINEL CONFIG SET sentinel-pass $::password] | ||
| 44 | } | ||
| 45 | |||
| 46 | wait_for_condition 200 50 { | ||
| 47 | [catch {S 1 SENTINEL CKQUORUM mymaster}] == 0 | ||
| 48 | } else { | ||
| 49 | fail "Expected: Sentinel to be connected to master after setting password" | ||
| 50 | } | ||
| 51 | assert_match {*OK*} [S 1 SENTINEL CKQUORUM mymaster] | ||
| 52 | } | ||
| 53 | |||
| 54 | test "(post-cleanup) Tear down ACL configuration" { | ||
| 55 | teardown_acl | ||
| 56 | } | ||
diff --git a/examples/redis-unstable/tests/sentinel/tests/10-replica-priority.tcl b/examples/redis-unstable/tests/sentinel/tests/10-replica-priority.tcl new file mode 100644 index 0000000..d3f868a --- /dev/null +++ b/examples/redis-unstable/tests/sentinel/tests/10-replica-priority.tcl | |||
| @@ -0,0 +1,76 @@ | |||
| 1 | source "../tests/includes/init-tests.tcl" | ||
| 2 | |||
| 3 | test "Check acceptable replica-priority values" { | ||
| 4 | foreach_redis_id id { | ||
| 5 | if {$id == $master_id} continue | ||
| 6 | |||
| 7 | # ensure replica-announced accepts yes and no | ||
| 8 | catch {R $id CONFIG SET replica-announced no} e | ||
| 9 | if {$e ne "OK"} { | ||
| 10 | fail "Unable to set replica-announced to no" | ||
| 11 | } | ||
| 12 | catch {R $id CONFIG SET replica-announced yes} e | ||
| 13 | if {$e ne "OK"} { | ||
| 14 | fail "Unable to set replica-announced to yes" | ||
| 15 | } | ||
| 16 | |||
| 17 | # ensure a random value throw error | ||
| 18 | catch {R $id CONFIG SET replica-announced 321} e | ||
| 19 | if {$e eq "OK"} { | ||
| 20 | fail "Able to set replica-announced with something else than yes or no (321) whereas it should not be possible" | ||
| 21 | } | ||
| 22 | catch {R $id CONFIG SET replica-announced a3b2c1} e | ||
| 23 | if {$e eq "OK"} { | ||
| 24 | fail "Able to set replica-announced with something else than yes or no (a3b2c1) whereas it should not be possible" | ||
| 25 | } | ||
| 26 | |||
| 27 | # test only the first redis replica, no need to double test | ||
| 28 | break | ||
| 29 | } | ||
| 30 | } | ||
| 31 | |||
| 32 | proc 10_test_number_of_replicas {n_replicas_expected} { | ||
| 33 | test "Check sentinel replies with $n_replicas_expected replicas" { | ||
| 34 | # ensure sentinels replies with the right number of replicas | ||
| 35 | foreach_sentinel_id id { | ||
| 36 | S $id sentinel debug info-period 100 | ||
| 37 | S $id sentinel debug default-down-after 1000 | ||
| 38 | S $id sentinel debug publish-period 100 | ||
| 39 | set len [llength [S $id SENTINEL REPLICAS mymaster]] | ||
| 40 | wait_for_condition 200 100 { | ||
| 41 | [llength [S $id SENTINEL REPLICAS mymaster]] == $n_replicas_expected | ||
| 42 | } else { | ||
| 43 | fail "Sentinel replies with a wrong number of replicas with replica-announced=yes (expected $n_replicas_expected but got $len) on sentinel $id" | ||
| 44 | } | ||
| 45 | } | ||
| 46 | } | ||
| 47 | } | ||
| 48 | |||
| 49 | proc 10_set_replica_announced {master_id announced n_replicas} { | ||
| 50 | test "Set replica-announced=$announced on $n_replicas replicas" { | ||
| 51 | set i 0 | ||
| 52 | foreach_redis_id id { | ||
| 53 | if {$id == $master_id} continue | ||
| 54 | #puts "set replica-announce=$announced on redis #$id" | ||
| 55 | R $id CONFIG SET replica-announced "$announced" | ||
| 56 | incr i | ||
| 57 | if { $n_replicas!="all" && $i >= $n_replicas } { break } | ||
| 58 | } | ||
| 59 | } | ||
| 60 | } | ||
| 61 | |||
| 62 | # ensure all replicas are announced | ||
| 63 | 10_set_replica_announced $master_id "yes" "all" | ||
| 64 | # ensure all replicas are announced by sentinels | ||
| 65 | 10_test_number_of_replicas 4 | ||
| 66 | |||
| 67 | # ensure the first 2 replicas are not announced | ||
| 68 | 10_set_replica_announced $master_id "no" 2 | ||
| 69 | # ensure sentinels are not announcing the first 2 replicas that have been set unannounced | ||
| 70 | 10_test_number_of_replicas 2 | ||
| 71 | |||
| 72 | # ensure all replicas are announced | ||
| 73 | 10_set_replica_announced $master_id "yes" "all" | ||
| 74 | # ensure all replicas are not announced by sentinels | ||
| 75 | 10_test_number_of_replicas 4 | ||
| 76 | |||
diff --git a/examples/redis-unstable/tests/sentinel/tests/11-port-0.tcl b/examples/redis-unstable/tests/sentinel/tests/11-port-0.tcl new file mode 100644 index 0000000..a3e8bdb --- /dev/null +++ b/examples/redis-unstable/tests/sentinel/tests/11-port-0.tcl | |||
| @@ -0,0 +1,33 @@ | |||
| 1 | source "../tests/includes/init-tests.tcl" | ||
| 2 | |||
| 3 | test "Start/Stop sentinel on same port with a different runID should not change the total number of sentinels" { | ||
| 4 | set sentinel_id [expr $::instances_count-1] | ||
| 5 | # Kill sentinel instance | ||
| 6 | kill_instance sentinel $sentinel_id | ||
| 7 | |||
| 8 | # Delete line with myid in sentinels config file | ||
| 9 | set orgfilename [file join "sentinel_$sentinel_id" "sentinel.conf"] | ||
| 10 | set tmpfilename "sentinel.conf_tmp" | ||
| 11 | set dirname "sentinel_$sentinel_id" | ||
| 12 | |||
| 13 | delete_lines_with_pattern $orgfilename $tmpfilename "myid" | ||
| 14 | |||
| 15 | # Get count of total sentinels | ||
| 16 | set a [S 0 SENTINEL master mymaster] | ||
| 17 | set original_count [lindex $a 33] | ||
| 18 | |||
| 19 | # Restart sentinel with the modified config file | ||
| 20 | set pid [exec_instance "sentinel" $dirname $orgfilename] | ||
| 21 | lappend ::pids $pid | ||
| 22 | |||
| 23 | after 1000 | ||
| 24 | |||
| 25 | # Get new count of total sentinel | ||
| 26 | set b [S 0 SENTINEL master mymaster] | ||
| 27 | set curr_count [lindex $b 33] | ||
| 28 | |||
| 29 | # If the count is not the same then fail the test | ||
| 30 | if {$original_count != $curr_count} { | ||
| 31 | fail "Sentinel count is incorrect, original count being $original_count and current count is $curr_count" | ||
| 32 | } | ||
| 33 | } | ||
diff --git a/examples/redis-unstable/tests/sentinel/tests/12-master-reboot.tcl b/examples/redis-unstable/tests/sentinel/tests/12-master-reboot.tcl new file mode 100644 index 0000000..3d0d828 --- /dev/null +++ b/examples/redis-unstable/tests/sentinel/tests/12-master-reboot.tcl | |||
| @@ -0,0 +1,112 @@ | |||
| 1 | # Check the basic monitoring and failover capabilities. | ||
| 2 | source "../tests/includes/init-tests.tcl" | ||
| 3 | |||
| 4 | |||
| 5 | if {$::simulate_error} { | ||
| 6 | test "This test will fail" { | ||
| 7 | fail "Simulated error" | ||
| 8 | } | ||
| 9 | } | ||
| 10 | |||
| 11 | |||
| 12 | # Reboot an instance previously in very short time but do not check if it is loading | ||
| 13 | proc reboot_instance {type id} { | ||
| 14 | set dirname "${type}_${id}" | ||
| 15 | set cfgfile [file join $dirname $type.conf] | ||
| 16 | set port [get_instance_attrib $type $id port] | ||
| 17 | |||
| 18 | # Execute the instance with its old setup and append the new pid | ||
| 19 | # file for cleanup. | ||
| 20 | set pid [exec_instance $type $dirname $cfgfile] | ||
| 21 | set_instance_attrib $type $id pid $pid | ||
| 22 | lappend ::pids $pid | ||
| 23 | |||
| 24 | # Check that the instance is running | ||
| 25 | if {[server_is_up 127.0.0.1 $port 100] == 0} { | ||
| 26 | set logfile [file join $dirname log.txt] | ||
| 27 | puts [exec tail $logfile] | ||
| 28 | abort_sentinel_test "Problems starting $type #$id: ping timeout, maybe server start failed, check $logfile" | ||
| 29 | } | ||
| 30 | |||
| 31 | # Connect with it with a fresh link | ||
| 32 | set link [redis 127.0.0.1 $port 0 $::tls] | ||
| 33 | $link reconnect 1 | ||
| 34 | set_instance_attrib $type $id link $link | ||
| 35 | } | ||
| 36 | |||
| 37 | |||
| 38 | test "Master reboot in very short time" { | ||
| 39 | set old_port [RPort $master_id] | ||
| 40 | set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] | ||
| 41 | assert {[lindex $addr 1] == $old_port} | ||
| 42 | |||
| 43 | R $master_id debug populate 20000 | ||
| 44 | R $master_id bgsave | ||
| 45 | R $master_id config set key-load-delay 1500 | ||
| 46 | R $master_id config set loading-process-events-interval-bytes 1024 | ||
| 47 | R $master_id config rewrite | ||
| 48 | |||
| 49 | foreach_sentinel_id id { | ||
| 50 | S $id SENTINEL SET mymaster master-reboot-down-after-period 5000 | ||
| 51 | S $id sentinel debug ping-period 500 | ||
| 52 | S $id sentinel debug ask-period 500 | ||
| 53 | } | ||
| 54 | |||
| 55 | kill_instance redis $master_id | ||
| 56 | reboot_instance redis $master_id | ||
| 57 | |||
| 58 | set max_tries 1000 | ||
| 59 | if {$::tsan} { | ||
| 60 | set max_tries 5000 | ||
| 61 | } | ||
| 62 | |||
| 63 | foreach_sentinel_id id { | ||
| 64 | wait_for_condition $max_tries 100 { | ||
| 65 | [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port | ||
| 66 | } else { | ||
| 67 | fail "At least one Sentinel did not receive failover info" | ||
| 68 | } | ||
| 69 | } | ||
| 70 | |||
| 71 | # Reset configuration to avoid unnecessary delays | ||
| 72 | R $master_id config set key-load-delay 0 | ||
| 73 | R $master_id config rewrite | ||
| 74 | |||
| 75 | set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] | ||
| 76 | set master_id [get_instance_id_by_port redis [lindex $addr 1]] | ||
| 77 | |||
| 78 | # Make sure the instance load all the dataset | ||
| 79 | while 1 { | ||
| 80 | catch {[$link ping]} retval | ||
| 81 | if {[string match {*LOADING*} $retval]} { | ||
| 82 | after 100 | ||
| 83 | continue | ||
| 84 | } else { | ||
| 85 | break | ||
| 86 | } | ||
| 87 | } | ||
| 88 | } | ||
| 89 | |||
| 90 | test "New master [join $addr {:}] role matches" { | ||
| 91 | assert {[RI $master_id role] eq {master}} | ||
| 92 | } | ||
| 93 | |||
| 94 | test "All the other slaves now point to the new master" { | ||
| 95 | foreach_redis_id id { | ||
| 96 | if {$id != $master_id && $id != 0} { | ||
| 97 | wait_for_condition 1000 50 { | ||
| 98 | [RI $id master_port] == [lindex $addr 1] | ||
| 99 | } else { | ||
| 100 | fail "Redis ID $id not configured to replicate with new master" | ||
| 101 | } | ||
| 102 | } | ||
| 103 | } | ||
| 104 | } | ||
| 105 | |||
| 106 | test "The old master eventually gets reconfigured as a slave" { | ||
| 107 | wait_for_condition 1000 50 { | ||
| 108 | [RI 0 master_port] == [lindex $addr 1] | ||
| 109 | } else { | ||
| 110 | fail "Old master not reconfigured as slave of new master" | ||
| 111 | } | ||
| 112 | } | ||
diff --git a/examples/redis-unstable/tests/sentinel/tests/13-info-command.tcl b/examples/redis-unstable/tests/sentinel/tests/13-info-command.tcl new file mode 100644 index 0000000..ef9dc01 --- /dev/null +++ b/examples/redis-unstable/tests/sentinel/tests/13-info-command.tcl | |||
| @@ -0,0 +1,47 @@ | |||
| 1 | source "../tests/includes/init-tests.tcl" | ||
| 2 | |||
| 3 | test "info command with at most one argument" { | ||
| 4 | set subCommandList {} | ||
| 5 | foreach arg {"" "all" "default" "everything"} { | ||
| 6 | if {$arg == ""} { | ||
| 7 | set info [S 0 info] | ||
| 8 | } else { | ||
| 9 | set info [S 0 info $arg] | ||
| 10 | } | ||
| 11 | assert { [string match "*redis_version*" $info] } | ||
| 12 | assert { [string match "*maxclients*" $info] } | ||
| 13 | assert { [string match "*used_cpu_user*" $info] } | ||
| 14 | assert { [string match "*sentinel_tilt*" $info] } | ||
| 15 | assert { ![string match "*used_memory*" $info] } | ||
| 16 | assert { ![string match "*rdb_last_bgsave*" $info] } | ||
| 17 | assert { ![string match "*master_repl_offset*" $info] } | ||
| 18 | assert { ![string match "*cluster_enabled*" $info] } | ||
| 19 | } | ||
| 20 | } | ||
| 21 | |||
| 22 | test "info command with one sub-section" { | ||
| 23 | set info [S 0 info cpu] | ||
| 24 | assert { [string match "*used_cpu_user*" $info] } | ||
| 25 | assert { ![string match "*sentinel_tilt*" $info] } | ||
| 26 | assert { ![string match "*redis_version*" $info] } | ||
| 27 | |||
| 28 | set info [S 0 info sentinel] | ||
| 29 | assert { [string match "*sentinel_tilt*" $info] } | ||
| 30 | assert { ![string match "*used_cpu_user*" $info] } | ||
| 31 | assert { ![string match "*redis_version*" $info] } | ||
| 32 | } | ||
| 33 | |||
| 34 | test "info command with multiple sub-sections" { | ||
| 35 | set info [S 0 info server sentinel replication] | ||
| 36 | assert { [string match "*redis_version*" $info] } | ||
| 37 | assert { [string match "*sentinel_tilt*" $info] } | ||
| 38 | assert { ![string match "*used_memory*" $info] } | ||
| 39 | assert { ![string match "*used_cpu_user*" $info] } | ||
| 40 | |||
| 41 | set info [S 0 info cpu all] | ||
| 42 | assert { [string match "*used_cpu_user*" $info] } | ||
| 43 | assert { [string match "*sentinel_tilt*" $info] } | ||
| 44 | assert { [string match "*redis_version*" $info] } | ||
| 45 | assert { ![string match "*used_memory*" $info] } | ||
| 46 | assert { ![string match "*master_repl_offset*" $info] } | ||
| 47 | } | ||
diff --git a/examples/redis-unstable/tests/sentinel/tests/14-debug-command.tcl b/examples/redis-unstable/tests/sentinel/tests/14-debug-command.tcl new file mode 100644 index 0000000..dccb992 --- /dev/null +++ b/examples/redis-unstable/tests/sentinel/tests/14-debug-command.tcl | |||
| @@ -0,0 +1,9 @@ | |||
| 1 | source "../tests/includes/init-tests.tcl" | ||
| 2 | |||
| 3 | test "Sentinel debug test with arguments and without argument" { | ||
| 4 | set current_info_period [lindex [S 0 SENTINEL DEBUG] 1] | ||
| 5 | S 0 SENTINEL DEBUG info-period 8888 | ||
| 6 | assert_equal {8888} [lindex [S 0 SENTINEL DEBUG] 1] | ||
| 7 | S 0 SENTINEL DEBUG info-period $current_info_period | ||
| 8 | assert_equal $current_info_period [lindex [S 0 SENTINEL DEBUG] 1] | ||
| 9 | } | ||
diff --git a/examples/redis-unstable/tests/sentinel/tests/15-config-set-config-get.tcl b/examples/redis-unstable/tests/sentinel/tests/15-config-set-config-get.tcl new file mode 100644 index 0000000..f9831f8 --- /dev/null +++ b/examples/redis-unstable/tests/sentinel/tests/15-config-set-config-get.tcl | |||
| @@ -0,0 +1,58 @@ | |||
| 1 | source "../tests/includes/init-tests.tcl" | ||
| 2 | |||
| 3 | test "SENTINEL CONFIG SET and SENTINEL CONFIG GET handles multiple variables" { | ||
| 4 | foreach_sentinel_id id { | ||
| 5 | assert_equal {OK} [S $id SENTINEL CONFIG SET resolve-hostnames yes announce-port 1234] | ||
| 6 | } | ||
| 7 | assert_match {*yes*1234*} [S 1 SENTINEL CONFIG GET resolve-hostnames announce-port] | ||
| 8 | assert_match {announce-port 1234} [S 1 SENTINEL CONFIG GET announce-port] | ||
| 9 | } | ||
| 10 | |||
| 11 | test "SENTINEL CONFIG GET for duplicate and unknown variables" { | ||
| 12 | assert_equal {OK} [S 1 SENTINEL CONFIG SET resolve-hostnames yes announce-port 1234] | ||
| 13 | assert_match {resolve-hostnames yes} [S 1 SENTINEL CONFIG GET resolve-hostnames resolve-hostnames does-not-exist] | ||
| 14 | } | ||
| 15 | |||
| 16 | test "SENTINEL CONFIG GET for patterns" { | ||
| 17 | assert_equal {OK} [S 1 SENTINEL CONFIG SET loglevel notice announce-port 1234 announce-hostnames yes ] | ||
| 18 | assert_match {loglevel notice} [S 1 SENTINEL CONFIG GET log* *level loglevel] | ||
| 19 | assert_match {announce-hostnames yes announce-ip*announce-port 1234} [S 1 SENTINEL CONFIG GET announce*] | ||
| 20 | } | ||
| 21 | |||
| 22 | test "SENTINEL CONFIG SET duplicate variables" { | ||
| 23 | catch {[S 1 SENTINEL CONFIG SET resolve-hostnames yes announce-port 1234 announce-port 100]} e | ||
| 24 | if {![string match "*Duplicate argument*" $e]} { | ||
| 25 | fail "Should give wrong arity error" | ||
| 26 | } | ||
| 27 | } | ||
| 28 | |||
| 29 | test "SENTINEL CONFIG SET, one option does not exist" { | ||
| 30 | foreach_sentinel_id id { | ||
| 31 | assert_equal {OK} [S $id SENTINEL CONFIG SET announce-port 111] | ||
| 32 | catch {[S $id SENTINEL CONFIG SET does-not-exist yes announce-port 1234]} e | ||
| 33 | if {![string match "*Invalid argument*" $e]} { | ||
| 34 | fail "Should give Invalid argument error" | ||
| 35 | } | ||
| 36 | } | ||
| 37 | # The announce-port should not be set to 1234 as it was called with a wrong argument | ||
| 38 | assert_match {*111*} [S 1 SENTINEL CONFIG GET announce-port] | ||
| 39 | } | ||
| 40 | |||
| 41 | test "SENTINEL CONFIG SET, one option with wrong value" { | ||
| 42 | foreach_sentinel_id id { | ||
| 43 | assert_equal {OK} [S $id SENTINEL CONFIG SET resolve-hostnames no] | ||
| 44 | catch {[S $id SENTINEL CONFIG SET announce-port -1234 resolve-hostnames yes]} e | ||
| 45 | if {![string match "*Invalid value*" $e]} { | ||
| 46 | fail "Expected to return Invalid value error" | ||
| 47 | } | ||
| 48 | } | ||
| 49 | # The resolve-hostnames should not be set to yes as it was called after an argument with an invalid value | ||
| 50 | assert_match {*no*} [S 1 SENTINEL CONFIG GET resolve-hostnames] | ||
| 51 | } | ||
| 52 | |||
| 53 | test "SENTINEL CONFIG SET, wrong number of arguments" { | ||
| 54 | catch {[S 1 SENTINEL CONFIG SET resolve-hostnames yes announce-port 1234 announce-ip]} e | ||
| 55 | if {![string match "*Missing argument*" $e]} { | ||
| 56 | fail "Expected to return Missing argument error" | ||
| 57 | } | ||
| 58 | } | ||
diff --git a/examples/redis-unstable/tests/sentinel/tests/helpers/check_leaked_fds.tcl b/examples/redis-unstable/tests/sentinel/tests/helpers/check_leaked_fds.tcl new file mode 100755 index 0000000..482b3e0 --- /dev/null +++ b/examples/redis-unstable/tests/sentinel/tests/helpers/check_leaked_fds.tcl | |||
| @@ -0,0 +1,79 @@ | |||
| 1 | #!/usr/bin/env tclsh | ||
| 2 | # | ||
| 3 | # This script detects file descriptors that have leaked from a parent process. | ||
| 4 | # | ||
| 5 | # Our goal is to detect file descriptors that were opened by the parent and | ||
| 6 | # not cleaned up prior to exec(), but not file descriptors that were inherited | ||
| 7 | # from the grandparent which the parent knows nothing about. To do that, we | ||
| 8 | # look up every potential leak and try to match it against open files by the | ||
| 9 | # grandparent process. | ||
| 10 | |||
| 11 | # Get PID of parent process | ||
| 12 | proc get_parent_pid {_pid} { | ||
| 13 | set fd [open "/proc/$_pid/status" "r"] | ||
| 14 | set content [read $fd] | ||
| 15 | close $fd | ||
| 16 | |||
| 17 | if {[regexp {\nPPid:\s+(\d+)} $content _ ppid]} { | ||
| 18 | return $ppid | ||
| 19 | } | ||
| 20 | |||
| 21 | error "failed to get parent pid" | ||
| 22 | } | ||
| 23 | |||
| 24 | # Read symlink to get info about the specified fd of the specified process. | ||
| 25 | # The result can be the file name or an arbitrary string that identifies it. | ||
| 26 | # When not able to read, an empty string is returned. | ||
| 27 | proc get_fdlink {_pid fd} { | ||
| 28 | if { [catch {set fdlink [file readlink "/proc/$_pid/fd/$fd"]} err] } { | ||
| 29 | return "" | ||
| 30 | } | ||
| 31 | return $fdlink | ||
| 32 | } | ||
| 33 | |||
| 34 | # Linux only | ||
| 35 | set os [exec uname] | ||
| 36 | if {$os != "Linux"} { | ||
| 37 | puts "Only Linux is supported." | ||
| 38 | exit 0 | ||
| 39 | } | ||
| 40 | |||
| 41 | if {![info exists env(LEAKED_FDS_FILE)]} { | ||
| 42 | puts "Missing LEAKED_FDS_FILE environment variable." | ||
| 43 | exit 0 | ||
| 44 | } | ||
| 45 | |||
| 46 | set outfile $::env(LEAKED_FDS_FILE) | ||
| 47 | set parent_pid [get_parent_pid [pid]] | ||
| 48 | set grandparent_pid [get_parent_pid $parent_pid] | ||
| 49 | set leaked_fds {} | ||
| 50 | |||
| 51 | # Look for fds that were directly inherited from our parent but not from | ||
| 52 | # our grandparent (tcl) | ||
| 53 | foreach fd [glob -tails -directory "/proc/self/fd" *] { | ||
| 54 | # Ignore stdin/stdout/stderr | ||
| 55 | if {$fd == 0 || $fd == 1 || $fd == 2} { | ||
| 56 | continue | ||
| 57 | } | ||
| 58 | |||
| 59 | set fdlink [get_fdlink "self" $fd] | ||
| 60 | if {$fdlink == ""} { | ||
| 61 | continue | ||
| 62 | } | ||
| 63 | |||
| 64 | # We ignore fds that existed in the grandparent, or fds that don't exist | ||
| 65 | # in our parent (Sentinel process). | ||
| 66 | if {[get_fdlink $grandparent_pid $fd] == $fdlink || | ||
| 67 | [get_fdlink $parent_pid $fd] != $fdlink} { | ||
| 68 | continue | ||
| 69 | } | ||
| 70 | |||
| 71 | lappend leaked_fds [list $fd $fdlink] | ||
| 72 | } | ||
| 73 | |||
| 74 | # Produce report only if we found leaks | ||
| 75 | if {[llength $leaked_fds] > 0} { | ||
| 76 | set fd [open $outfile "w"] | ||
| 77 | puts $fd [join $leaked_fds "\n"] | ||
| 78 | close $fd | ||
| 79 | } | ||
diff --git a/examples/redis-unstable/tests/sentinel/tests/includes/init-tests.tcl b/examples/redis-unstable/tests/sentinel/tests/includes/init-tests.tcl new file mode 100644 index 0000000..ddb1319 --- /dev/null +++ b/examples/redis-unstable/tests/sentinel/tests/includes/init-tests.tcl | |||
| @@ -0,0 +1,63 @@ | |||
| 1 | # Initialization tests -- most units will start including this. | ||
| 2 | source "../tests/includes/utils.tcl" | ||
| 3 | |||
| 4 | test "(init) Restart killed instances" { | ||
| 5 | restart_killed_instances | ||
| 6 | } | ||
| 7 | |||
| 8 | test "(init) Remove old master entry from sentinels" { | ||
| 9 | foreach_sentinel_id id { | ||
| 10 | catch {S $id SENTINEL REMOVE mymaster} | ||
| 11 | } | ||
| 12 | } | ||
| 13 | |||
| 14 | set redis_slaves [expr $::instances_count - 1] | ||
| 15 | test "(init) Create a master-slaves cluster of [expr $redis_slaves+1] instances" { | ||
| 16 | create_redis_master_slave_cluster [expr {$redis_slaves+1}] | ||
| 17 | } | ||
| 18 | set master_id 0 | ||
| 19 | |||
| 20 | test "(init) Sentinels can start monitoring a master" { | ||
| 21 | set sentinels [llength $::sentinel_instances] | ||
| 22 | set quorum [expr {$sentinels/2+1}] | ||
| 23 | foreach_sentinel_id id { | ||
| 24 | S $id SENTINEL MONITOR mymaster \ | ||
| 25 | [get_instance_attrib redis $master_id host] \ | ||
| 26 | [get_instance_attrib redis $master_id port] $quorum | ||
| 27 | } | ||
| 28 | foreach_sentinel_id id { | ||
| 29 | assert {[S $id sentinel master mymaster] ne {}} | ||
| 30 | S $id SENTINEL SET mymaster down-after-milliseconds 2000 | ||
| 31 | S $id SENTINEL SET mymaster failover-timeout 10000 | ||
| 32 | S $id SENTINEL debug tilt-period 5000 | ||
| 33 | S $id SENTINEL SET mymaster parallel-syncs 10 | ||
| 34 | if {$::leaked_fds_file != "" && [exec uname] == "Linux"} { | ||
| 35 | S $id SENTINEL SET mymaster notification-script ../../tests/helpers/check_leaked_fds.tcl | ||
| 36 | S $id SENTINEL SET mymaster client-reconfig-script ../../tests/helpers/check_leaked_fds.tcl | ||
| 37 | } | ||
| 38 | } | ||
| 39 | } | ||
| 40 | |||
| 41 | test "(init) Sentinels can talk with the master" { | ||
| 42 | foreach_sentinel_id id { | ||
| 43 | wait_for_condition 1000 50 { | ||
| 44 | [catch {S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster}] == 0 | ||
| 45 | } else { | ||
| 46 | fail "Sentinel $id can't talk with the master." | ||
| 47 | } | ||
| 48 | } | ||
| 49 | } | ||
| 50 | |||
| 51 | test "(init) Sentinels are able to auto-discover other sentinels" { | ||
| 52 | verify_sentinel_auto_discovery | ||
| 53 | } | ||
| 54 | |||
| 55 | test "(init) Sentinels are able to auto-discover slaves" { | ||
| 56 | foreach_sentinel_id id { | ||
| 57 | wait_for_condition 1000 50 { | ||
| 58 | [dict get [S $id SENTINEL MASTER mymaster] num-slaves] == $redis_slaves | ||
| 59 | } else { | ||
| 60 | fail "At least some sentinel can't detect some slave" | ||
| 61 | } | ||
| 62 | } | ||
| 63 | } | ||
diff --git a/examples/redis-unstable/tests/sentinel/tests/includes/sentinel.conf b/examples/redis-unstable/tests/sentinel/tests/includes/sentinel.conf new file mode 100644 index 0000000..1275236 --- /dev/null +++ b/examples/redis-unstable/tests/sentinel/tests/includes/sentinel.conf | |||
| @@ -0,0 +1,9 @@ | |||
| 1 | # assume master is down after being unresponsive for 20s | ||
| 2 | sentinel down-after-milliseconds setmaster 20000 | ||
| 3 | # reconfigure one slave at a time | ||
| 4 | sentinel parallel-syncs setmaster 2 | ||
| 5 | # wait for 4m before assuming failover went wrong | ||
| 6 | sentinel failover-timeout setmaster 240000 | ||
| 7 | # monitoring set | ||
| 8 | sentinel monitor setmaster 10.0.0.1 30000 2 | ||
| 9 | |||
diff --git a/examples/redis-unstable/tests/sentinel/tests/includes/start-init-tests.tcl b/examples/redis-unstable/tests/sentinel/tests/includes/start-init-tests.tcl new file mode 100644 index 0000000..b052350 --- /dev/null +++ b/examples/redis-unstable/tests/sentinel/tests/includes/start-init-tests.tcl | |||
| @@ -0,0 +1,18 @@ | |||
| 1 | test "(start-init) Flush config and compare rewrite config file lines" { | ||
| 2 | foreach_sentinel_id id { | ||
| 3 | assert_match "OK" [S $id SENTINEL FLUSHCONFIG] | ||
| 4 | set file1 ../tests/includes/sentinel.conf | ||
| 5 | set file2 [file join "sentinel_${id}" "sentinel.conf"] | ||
| 6 | set fh1 [open $file1 r] | ||
| 7 | set fh2 [open $file2 r] | ||
| 8 | while {[gets $fh1 line1]} { | ||
| 9 | if {[gets $fh2 line2]} { | ||
| 10 | assert [string equal $line1 $line2] | ||
| 11 | } else { | ||
| 12 | fail "sentinel config file rewrite sequence changed" | ||
| 13 | } | ||
| 14 | } | ||
| 15 | close $fh1 | ||
| 16 | close $fh2 | ||
| 17 | } | ||
| 18 | } \ No newline at end of file | ||
diff --git a/examples/redis-unstable/tests/sentinel/tests/includes/utils.tcl b/examples/redis-unstable/tests/sentinel/tests/includes/utils.tcl new file mode 100644 index 0000000..adfd91c --- /dev/null +++ b/examples/redis-unstable/tests/sentinel/tests/includes/utils.tcl | |||
| @@ -0,0 +1,22 @@ | |||
| 1 | proc restart_killed_instances {} { | ||
| 2 | foreach type {redis sentinel} { | ||
| 3 | foreach_${type}_id id { | ||
| 4 | if {[get_instance_attrib $type $id pid] == -1} { | ||
| 5 | puts -nonewline "$type/$id " | ||
| 6 | flush stdout | ||
| 7 | restart_instance $type $id | ||
| 8 | } | ||
| 9 | } | ||
| 10 | } | ||
| 11 | } | ||
| 12 | |||
| 13 | proc verify_sentinel_auto_discovery {} { | ||
| 14 | set sentinels [llength $::sentinel_instances] | ||
| 15 | foreach_sentinel_id id { | ||
| 16 | wait_for_condition 1000 50 { | ||
| 17 | [dict get [S $id SENTINEL MASTER mymaster] num-other-sentinels] == ($sentinels-1) | ||
| 18 | } else { | ||
| 19 | fail "At least some sentinel can't detect some other sentinel" | ||
| 20 | } | ||
| 21 | } | ||
| 22 | } | ||
diff --git a/examples/redis-unstable/tests/sentinel/tmp/.gitignore b/examples/redis-unstable/tests/sentinel/tmp/.gitignore new file mode 100644 index 0000000..f581f73 --- /dev/null +++ b/examples/redis-unstable/tests/sentinel/tmp/.gitignore | |||
| @@ -0,0 +1,2 @@ | |||
| 1 | redis_* | ||
| 2 | sentinel_* | ||
