aboutsummaryrefslogtreecommitdiff
path: root/examples/redis-unstable/tests/sentinel
diff options
context:
space:
mode:
Diffstat (limited to 'examples/redis-unstable/tests/sentinel')
-rw-r--r--examples/redis-unstable/tests/sentinel/run.tcl41
-rw-r--r--examples/redis-unstable/tests/sentinel/tests/00-base.tcl210
-rw-r--r--examples/redis-unstable/tests/sentinel/tests/01-conf-update.tcl50
-rw-r--r--examples/redis-unstable/tests/sentinel/tests/02-slaves-reconf.tcl91
-rw-r--r--examples/redis-unstable/tests/sentinel/tests/03-runtime-reconf.tcl225
-rw-r--r--examples/redis-unstable/tests/sentinel/tests/04-slave-selection.tcl5
-rw-r--r--examples/redis-unstable/tests/sentinel/tests/05-manual.tcl94
-rw-r--r--examples/redis-unstable/tests/sentinel/tests/06-ckquorum.tcl42
-rw-r--r--examples/redis-unstable/tests/sentinel/tests/07-down-conditions.tcl104
-rw-r--r--examples/redis-unstable/tests/sentinel/tests/08-hostname-conf.tcl69
-rw-r--r--examples/redis-unstable/tests/sentinel/tests/09-acl-support.tcl56
-rw-r--r--examples/redis-unstable/tests/sentinel/tests/10-replica-priority.tcl76
-rw-r--r--examples/redis-unstable/tests/sentinel/tests/11-port-0.tcl33
-rw-r--r--examples/redis-unstable/tests/sentinel/tests/12-master-reboot.tcl112
-rw-r--r--examples/redis-unstable/tests/sentinel/tests/13-info-command.tcl47
-rw-r--r--examples/redis-unstable/tests/sentinel/tests/14-debug-command.tcl9
-rw-r--r--examples/redis-unstable/tests/sentinel/tests/15-config-set-config-get.tcl58
-rwxr-xr-xexamples/redis-unstable/tests/sentinel/tests/helpers/check_leaked_fds.tcl79
-rw-r--r--examples/redis-unstable/tests/sentinel/tests/includes/init-tests.tcl63
-rw-r--r--examples/redis-unstable/tests/sentinel/tests/includes/sentinel.conf9
-rw-r--r--examples/redis-unstable/tests/sentinel/tests/includes/start-init-tests.tcl18
-rw-r--r--examples/redis-unstable/tests/sentinel/tests/includes/utils.tcl22
-rw-r--r--examples/redis-unstable/tests/sentinel/tmp/.gitignore2
23 files changed, 1515 insertions, 0 deletions
diff --git a/examples/redis-unstable/tests/sentinel/run.tcl b/examples/redis-unstable/tests/sentinel/run.tcl
new file mode 100644
index 0000000..4f2a656
--- /dev/null
+++ b/examples/redis-unstable/tests/sentinel/run.tcl
@@ -0,0 +1,41 @@
1# Sentinel test suite.
2#
3# Copyright (C) 2014-Present, Redis Ltd.
4# All Rights reserved.
5#
6# Licensed under your choice of (a) the Redis Source Available License 2.0
7# (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the
8# GNU Affero General Public License v3 (AGPLv3).
9
10cd tests/sentinel
11source ../instances.tcl
12
13set ::instances_count 5 ; # How many instances we use at max.
14set ::tlsdir "../../tls"
15
16proc main {} {
17 parse_options
18 if {$::leaked_fds_file != ""} {
19 set ::env(LEAKED_FDS_FILE) $::leaked_fds_file
20 }
21 spawn_instance sentinel $::sentinel_base_port $::instances_count {
22 "sentinel deny-scripts-reconfig no"
23 "enable-protected-configs yes"
24 "enable-debug-command yes"
25 } "../tests/includes/sentinel.conf"
26
27 spawn_instance redis $::redis_base_port $::instances_count {
28 "enable-protected-configs yes"
29 "enable-debug-command yes"
30 "save ''"
31 }
32 run_tests
33 cleanup
34 end_tests
35}
36
37if {[catch main e]} {
38 puts $::errorInfo
39 cleanup
40 exit 1
41}
diff --git a/examples/redis-unstable/tests/sentinel/tests/00-base.tcl b/examples/redis-unstable/tests/sentinel/tests/00-base.tcl
new file mode 100644
index 0000000..7b64395
--- /dev/null
+++ b/examples/redis-unstable/tests/sentinel/tests/00-base.tcl
@@ -0,0 +1,210 @@
1# Check the basic monitoring and failover capabilities.
2source "../tests/includes/start-init-tests.tcl"
3source "../tests/includes/init-tests.tcl"
4
5foreach_sentinel_id id {
6 S $id sentinel debug default-down-after 1000
7}
8
9if {$::simulate_error} {
10 test "This test will fail" {
11 fail "Simulated error"
12 }
13}
14
15test "Sentinel command flag infrastructure works correctly" {
16 foreach_sentinel_id id {
17 set command_list [S $id command list]
18
19 foreach cmd {ping info subscribe client|setinfo} {
20 assert_not_equal [S $id command docs $cmd] {}
21 assert_not_equal [lsearch $command_list $cmd] -1
22 }
23
24 foreach cmd {save bgrewriteaof blpop replicaof} {
25 assert_equal [S $id command docs $cmd] {}
26 assert_equal [lsearch $command_list $cmd] -1
27 assert_error {ERR unknown command*} {S $id $cmd}
28 }
29
30 assert_error {ERR unknown subcommand*} {S $id client no-touch}
31 }
32}
33
34test "SENTINEL HELP output the sentinel subcommand help" {
35 assert_match "*SENTINEL <subcommand> *" [S 0 SENTINEL HELP]
36}
37
38test "SENTINEL MYID return the sentinel instance ID" {
39 assert_equal 40 [string length [S 0 SENTINEL MYID]]
40 assert_equal [S 0 SENTINEL MYID] [S 0 SENTINEL MYID]
41}
42
43test "SENTINEL INFO CACHE returns the cached info" {
44 set res [S 0 SENTINEL INFO-CACHE mymaster]
45 assert_morethan_equal [llength $res] 2
46 assert_equal "mymaster" [lindex $res 0]
47
48 set res [lindex $res 1]
49 assert_morethan_equal [llength $res] 2
50 assert_morethan [lindex $res 0] 0
51 assert_match "*# Server*" [lindex $res 1]
52}
53
54test "SENTINEL PENDING-SCRIPTS returns the information about pending scripts" {
55 # may or may not have a value, so assert greater than or equal to 0.
56 assert_morethan_equal [llength [S 0 SENTINEL PENDING-SCRIPTS]] 0
57}
58
59test "SENTINEL MASTERS returns a list of monitored masters" {
60 assert_match "*mymaster*" [S 0 SENTINEL MASTERS]
61 assert_morethan_equal [llength [S 0 SENTINEL MASTERS]] 1
62}
63
64test "SENTINEL SENTINELS returns a list of sentinel instances" {
65 assert_morethan_equal [llength [S 0 SENTINEL SENTINELS mymaster]] 1
66}
67
68test "SENTINEL SLAVES returns a list of the monitored replicas" {
69 assert_morethan_equal [llength [S 0 SENTINEL SLAVES mymaster]] 1
70}
71
72test "SENTINEL SIMULATE-FAILURE HELP list supported flags" {
73 set res [S 0 SENTINEL SIMULATE-FAILURE HELP]
74 assert_morethan_equal [llength $res] 2
75 assert_equal {crash-after-election crash-after-promotion} $res
76}
77
78test "Basic failover works if the master is down" {
79 set old_port [RPort $master_id]
80 set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster]
81 assert {[lindex $addr 1] == $old_port}
82 kill_instance redis $master_id
83 foreach_sentinel_id id {
84 S $id sentinel debug ping-period 500
85 S $id sentinel debug ask-period 500
86 wait_for_condition 1000 100 {
87 [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port
88 } else {
89 fail "At least one Sentinel did not receive failover info"
90 }
91 }
92 restart_instance redis $master_id
93 set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster]
94 set master_id [get_instance_id_by_port redis [lindex $addr 1]]
95}
96
97test "New master [join $addr {:}] role matches" {
98 assert {[RI $master_id role] eq {master}}
99}
100
101test "All the other slaves now point to the new master" {
102 foreach_redis_id id {
103 if {$id != $master_id && $id != 0} {
104 wait_for_condition 1000 50 {
105 [RI $id master_port] == [lindex $addr 1]
106 } else {
107 fail "Redis ID $id not configured to replicate with new master"
108 }
109 }
110 }
111}
112
113test "The old master eventually gets reconfigured as a slave" {
114 wait_for_condition 1000 50 {
115 [RI 0 master_port] == [lindex $addr 1]
116 } else {
117 fail "Old master not reconfigured as slave of new master"
118 }
119}
120
121test "ODOWN is not possible without N (quorum) Sentinels reports" {
122 foreach_sentinel_id id {
123 S $id SENTINEL SET mymaster quorum [expr $sentinels+1]
124 }
125 set old_port [RPort $master_id]
126 set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster]
127 assert {[lindex $addr 1] == $old_port}
128 kill_instance redis $master_id
129
130 # Make sure failover did not happened.
131 set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster]
132 assert {[lindex $addr 1] == $old_port}
133 restart_instance redis $master_id
134}
135
136test "Failover is not possible without majority agreement" {
137 foreach_sentinel_id id {
138 S $id SENTINEL SET mymaster quorum $quorum
139 }
140
141 # Crash majority of sentinels
142 for {set id 0} {$id < $quorum} {incr id} {
143 kill_instance sentinel $id
144 }
145
146 # Kill the current master
147 kill_instance redis $master_id
148
149 # Make sure failover did not happened.
150 set addr [S $quorum SENTINEL GET-MASTER-ADDR-BY-NAME mymaster]
151 assert {[lindex $addr 1] == $old_port}
152 restart_instance redis $master_id
153
154 # Cleanup: restart Sentinels to monitor the master.
155 for {set id 0} {$id < $quorum} {incr id} {
156 restart_instance sentinel $id
157 }
158}
159
160test "Failover works if we configure for absolute agreement" {
161 foreach_sentinel_id id {
162 S $id SENTINEL SET mymaster quorum $sentinels
163 }
164
165 # Wait for Sentinels to monitor the master again
166 foreach_sentinel_id id {
167 wait_for_condition 1000 100 {
168 [dict get [S $id SENTINEL MASTER mymaster] info-refresh] < 100000
169 } else {
170 fail "At least one Sentinel is not monitoring the master"
171 }
172 }
173
174 kill_instance redis $master_id
175
176 foreach_sentinel_id id {
177 wait_for_condition 1000 100 {
178 [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port
179 } else {
180 fail "At least one Sentinel did not receive failover info"
181 }
182 }
183 restart_instance redis $master_id
184 set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster]
185 set master_id [get_instance_id_by_port redis [lindex $addr 1]]
186
187 # Set the min ODOWN agreement back to strict majority.
188 foreach_sentinel_id id {
189 S $id SENTINEL SET mymaster quorum $quorum
190 }
191}
192
193test "New master [join $addr {:}] role matches" {
194 assert {[RI $master_id role] eq {master}}
195}
196
197test "SENTINEL RESET can resets the master" {
198 # After SENTINEL RESET, sometimes the sentinel can sense the master again,
199 # causing the test to fail. Here we give it a few more chances.
200 for {set j 0} {$j < 10} {incr j} {
201 assert_equal 1 [S 0 SENTINEL RESET mymaster]
202 set res1 [llength [S 0 SENTINEL SENTINELS mymaster]]
203 set res2 [llength [S 0 SENTINEL SLAVES mymaster]]
204 set res3 [llength [S 0 SENTINEL REPLICAS mymaster]]
205 if {$res1 eq 0 && $res2 eq 0 && $res3 eq 0} break
206 }
207 assert_equal 0 $res1
208 assert_equal 0 $res2
209 assert_equal 0 $res3
210}
diff --git a/examples/redis-unstable/tests/sentinel/tests/01-conf-update.tcl b/examples/redis-unstable/tests/sentinel/tests/01-conf-update.tcl
new file mode 100644
index 0000000..fe29bb0
--- /dev/null
+++ b/examples/redis-unstable/tests/sentinel/tests/01-conf-update.tcl
@@ -0,0 +1,50 @@
1# Test Sentinel configuration consistency after partitions heal.
2
3source "../tests/includes/init-tests.tcl"
4
5test "We can failover with Sentinel 1 crashed" {
6 set old_port [RPort $master_id]
7 set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster]
8 assert {[lindex $addr 1] == $old_port}
9
10 # Crash Sentinel 1
11 kill_instance sentinel 1
12
13 kill_instance redis $master_id
14 foreach_sentinel_id id {
15 if {$id != 1} {
16 wait_for_condition 1000 50 {
17 [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port
18 } else {
19 fail "Sentinel $id did not receive failover info"
20 }
21 }
22 }
23 restart_instance redis $master_id
24 set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster]
25 set master_id [get_instance_id_by_port redis [lindex $addr 1]]
26}
27
28test "After Sentinel 1 is restarted, its config gets updated" {
29 restart_instance sentinel 1
30 wait_for_condition 1000 50 {
31 [lindex [S 1 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port
32 } else {
33 fail "Restarted Sentinel did not receive failover info"
34 }
35}
36
37test "New master [join $addr {:}] role matches" {
38 assert {[RI $master_id role] eq {master}}
39}
40
41test "Update log level" {
42 set current_loglevel [S 0 SENTINEL CONFIG GET loglevel]
43 assert {[lindex $current_loglevel 1] == {notice}}
44
45 foreach {loglevel} {debug verbose notice warning nothing} {
46 S 0 SENTINEL CONFIG SET loglevel $loglevel
47 set updated_loglevel [S 0 SENTINEL CONFIG GET loglevel]
48 assert {[lindex $updated_loglevel 1] == $loglevel}
49 }
50}
diff --git a/examples/redis-unstable/tests/sentinel/tests/02-slaves-reconf.tcl b/examples/redis-unstable/tests/sentinel/tests/02-slaves-reconf.tcl
new file mode 100644
index 0000000..8196b60
--- /dev/null
+++ b/examples/redis-unstable/tests/sentinel/tests/02-slaves-reconf.tcl
@@ -0,0 +1,91 @@
1# Check that slaves are reconfigured at a latter time if they are partitioned.
2#
3# Here we should test:
4# 1) That slaves point to the new master after failover.
5# 2) That partitioned slaves point to new master when they are partitioned
6# away during failover and return at a latter time.
7
8source "../tests/includes/init-tests.tcl"
9
10proc 02_test_slaves_replication {} {
11 uplevel 1 {
12 test "Check that slaves replicate from current master" {
13 set master_port [RPort $master_id]
14 foreach_redis_id id {
15 if {$id == $master_id} continue
16 if {[instance_is_killed redis $id]} continue
17 wait_for_condition 1000 50 {
18 ([RI $id master_port] == $master_port) &&
19 ([RI $id master_link_status] eq {up})
20 } else {
21 fail "Redis slave $id is replicating from wrong master"
22 }
23 }
24 }
25 }
26}
27
28proc 02_crash_and_failover {} {
29 uplevel 1 {
30 test "Crash the master and force a failover" {
31 set old_port [RPort $master_id]
32 set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster]
33 assert {[lindex $addr 1] == $old_port}
34 kill_instance redis $master_id
35 foreach_sentinel_id id {
36 wait_for_condition 1000 50 {
37 [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port
38 } else {
39 fail "At least one Sentinel did not receive failover info"
40 }
41 }
42 restart_instance redis $master_id
43 set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster]
44 set master_id [get_instance_id_by_port redis [lindex $addr 1]]
45 }
46 }
47}
48
4902_test_slaves_replication
5002_crash_and_failover
51
52foreach_sentinel_id id {
53 S $id sentinel debug info-period 100
54 S $id sentinel debug default-down-after 1000
55 S $id sentinel debug publish-period 100
56}
57
5802_test_slaves_replication
59
60test "Kill a slave instance" {
61 foreach_redis_id id {
62 if {$id == $master_id} continue
63 set killed_slave_id $id
64 kill_instance redis $id
65 break
66 }
67}
68
6902_crash_and_failover
7002_test_slaves_replication
71
72test "Wait for failover to end" {
73 set inprogress 1
74 while {$inprogress} {
75 set inprogress 0
76 foreach_sentinel_id id {
77 if {[dict exists [S $id SENTINEL MASTER mymaster] failover-state]} {
78 incr inprogress
79 }
80 }
81 if {$inprogress} {after 100}
82 }
83}
84
85test "Restart killed slave and test replication of slaves again..." {
86 restart_instance redis $killed_slave_id
87}
88
89# Now we check if the slave rejoining the partition is reconfigured even
90# if the failover finished.
9102_test_slaves_replication
diff --git a/examples/redis-unstable/tests/sentinel/tests/03-runtime-reconf.tcl b/examples/redis-unstable/tests/sentinel/tests/03-runtime-reconf.tcl
new file mode 100644
index 0000000..bd6eecc
--- /dev/null
+++ b/examples/redis-unstable/tests/sentinel/tests/03-runtime-reconf.tcl
@@ -0,0 +1,225 @@
1# Test runtime reconfiguration command SENTINEL SET.
2source "../tests/includes/init-tests.tcl"
3set num_sentinels [llength $::sentinel_instances]
4
5set ::user "testuser"
6set ::password "secret"
7
8proc server_set_password {} {
9 foreach_redis_id id {
10 assert_equal {OK} [R $id CONFIG SET requirepass $::password]
11 assert_equal {OK} [R $id AUTH $::password]
12 assert_equal {OK} [R $id CONFIG SET masterauth $::password]
13 }
14}
15
16proc server_reset_password {} {
17 foreach_redis_id id {
18 assert_equal {OK} [R $id CONFIG SET requirepass ""]
19 assert_equal {OK} [R $id CONFIG SET masterauth ""]
20 }
21}
22
23proc server_set_acl {id} {
24 assert_equal {OK} [R $id ACL SETUSER $::user on >$::password allchannels +@all]
25 assert_equal {OK} [R $id ACL SETUSER default off]
26
27 R $id CLIENT KILL USER default SKIPME no
28 assert_equal {OK} [R $id AUTH $::user $::password]
29 assert_equal {OK} [R $id CONFIG SET masteruser $::user]
30 assert_equal {OK} [R $id CONFIG SET masterauth $::password]
31}
32
33proc server_reset_acl {id} {
34 assert_equal {OK} [R $id ACL SETUSER default on]
35 assert_equal {1} [R $id ACL DELUSER $::user]
36
37 assert_equal {OK} [R $id CONFIG SET masteruser ""]
38 assert_equal {OK} [R $id CONFIG SET masterauth ""]
39}
40
41proc verify_sentinel_connect_replicas {id} {
42 foreach replica [S $id SENTINEL REPLICAS mymaster] {
43 if {[string match "*disconnected*" [dict get $replica flags]]} {
44 return 0
45 }
46 }
47 return 1
48}
49
50proc wait_for_sentinels_connect_servers { {is_connect 1} } {
51 foreach_sentinel_id id {
52 wait_for_condition 1000 50 {
53 [string match "*disconnected*" [dict get [S $id SENTINEL MASTER mymaster] flags]] != $is_connect
54 } else {
55 fail "At least some sentinel can't connect to master"
56 }
57
58 wait_for_condition 1000 50 {
59 [verify_sentinel_connect_replicas $id] == $is_connect
60 } else {
61 fail "At least some sentinel can't connect to replica"
62 }
63 }
64}
65
66test "Sentinels (re)connection following SENTINEL SET mymaster auth-pass" {
67 # 3 types of sentinels to test:
68 # (re)started while master changed pwd. Manage to connect only after setting pwd
69 set sent2re 0
70 # (up)dated in advance with master new password
71 set sent2up 1
72 # (un)touched. Yet manage to maintain (old) connection
73 set sent2un 2
74
75 wait_for_sentinels_connect_servers
76 kill_instance sentinel $sent2re
77 server_set_password
78 assert_equal {OK} [S $sent2up SENTINEL SET mymaster auth-pass $::password]
79 restart_instance sentinel $sent2re
80
81 # Verify sentinel that restarted failed to connect master
82 wait_for_condition 100 50 {
83 [string match "*disconnected*" [dict get [S $sent2re SENTINEL MASTER mymaster] flags]] != 0
84 } else {
85 fail "Expected to be disconnected from master due to wrong password"
86 }
87
88 # Update restarted sentinel with master password
89 assert_equal {OK} [S $sent2re SENTINEL SET mymaster auth-pass $::password]
90
91 # All sentinels expected to connect successfully
92 wait_for_sentinels_connect_servers
93
94 # remove requirepass and verify sentinels manage to connect servers
95 server_reset_password
96 wait_for_sentinels_connect_servers
97 # Sanity check
98 verify_sentinel_auto_discovery
99}
100
101test "Sentinels (re)connection following master ACL change" {
102 # Three types of sentinels to test during ACL change:
103 # 1. (re)started Sentinel. Manage to connect only after setting new pwd
104 # 2. (up)dated Sentinel, get just before ACL change the new password
105 # 3. (un)touched Sentinel that kept old connection with master and didn't
106 # set new ACL password won't persist ACL pwd change (unlike legacy auth-pass)
107 set sent2re 0
108 set sent2up 1
109 set sent2un 2
110
111 wait_for_sentinels_connect_servers
112 # kill sentinel 'sent2re' and restart it after ACL change
113 kill_instance sentinel $sent2re
114
115 # Update sentinel 'sent2up' with new user and pwd
116 assert_equal {OK} [S $sent2up SENTINEL SET mymaster auth-user $::user]
117 assert_equal {OK} [S $sent2up SENTINEL SET mymaster auth-pass $::password]
118
119 foreach_redis_id id {
120 server_set_acl $id
121 }
122
123 restart_instance sentinel $sent2re
124
125 # Verify sentinel that restarted failed to reconnect master
126 wait_for_condition 100 50 {
127 [string match "*disconnected*" [dict get [S $sent2re SENTINEL MASTER mymaster] flags]] != 0
128 } else {
129 fail "Expected: Restarted sentinel to be disconnected from master due to obsolete password"
130 }
131
132 # Verify sentinel with updated password managed to connect (wait for sentinelTimer to reconnect)
133 wait_for_condition 100 50 {
134 [string match "*disconnected*" [dict get [S $sent2up SENTINEL MASTER mymaster] flags]] == 0
135 } else {
136 fail "Expected: Sentinel to be connected to master"
137 }
138
139 # Verify sentinel untouched gets failed to connect master
140 wait_for_condition 100 50 {
141 [string match "*disconnected*" [dict get [S $sent2un SENTINEL MASTER mymaster] flags]] != 0
142 } else {
143 fail "Expected: Sentinel to be disconnected from master due to obsolete password"
144 }
145
146 # Now update all sentinels with new password
147 foreach_sentinel_id id {
148 assert_equal {OK} [S $id SENTINEL SET mymaster auth-user $::user]
149 assert_equal {OK} [S $id SENTINEL SET mymaster auth-pass $::password]
150 }
151
152 # All sentinels expected to connect successfully
153 wait_for_sentinels_connect_servers
154
155 # remove requirepass and verify sentinels manage to connect servers
156 foreach_redis_id id {
157 server_reset_acl $id
158 }
159
160 wait_for_sentinels_connect_servers
161 # Sanity check
162 verify_sentinel_auto_discovery
163}
164
165test "Set parameters in normal case" {
166
167 set info [S 0 SENTINEL master mymaster]
168 set origin_quorum [dict get $info quorum]
169 set origin_down_after_milliseconds [dict get $info down-after-milliseconds]
170 set update_quorum [expr $origin_quorum+1]
171 set update_down_after_milliseconds [expr $origin_down_after_milliseconds+1000]
172
173 assert_equal [S 0 SENTINEL SET mymaster quorum $update_quorum] "OK"
174 assert_equal [S 0 SENTINEL SET mymaster down-after-milliseconds $update_down_after_milliseconds] "OK"
175
176 set update_info [S 0 SENTINEL master mymaster]
177 assert {[dict get $update_info quorum] != $origin_quorum}
178 assert {[dict get $update_info down-after-milliseconds] != $origin_down_after_milliseconds}
179
180 #restore to origin config parameters
181 assert_equal [S 0 SENTINEL SET mymaster quorum $origin_quorum] "OK"
182 assert_equal [S 0 SENTINEL SET mymaster down-after-milliseconds $origin_down_after_milliseconds] "OK"
183}
184
185test "Set parameters in normal case with bad format" {
186
187 set info [S 0 SENTINEL master mymaster]
188 set origin_down_after_milliseconds [dict get $info down-after-milliseconds]
189
190 assert_error "ERR Invalid argument '-20' for SENTINEL SET 'down-after-milliseconds'*" {S 0 SENTINEL SET mymaster down-after-milliseconds -20}
191 assert_error "ERR Invalid argument 'abc' for SENTINEL SET 'down-after-milliseconds'*" {S 0 SENTINEL SET mymaster down-after-milliseconds "abc"}
192
193 set current_info [S 0 SENTINEL master mymaster]
194 assert {[dict get $current_info down-after-milliseconds] == $origin_down_after_milliseconds}
195}
196
197test "Sentinel Set with other error situations" {
198
199 # non-existing script
200 assert_error "ERR Notification script seems non existing*" {S 0 SENTINEL SET mymaster notification-script test.txt}
201
202 # wrong parameter number
203 assert_error "ERR wrong number of arguments for 'sentinel|set' command" {S 0 SENTINEL SET mymaster fakeoption}
204
205 # unknown parameter option
206 assert_error "ERR Unknown option or number of arguments for SENTINEL SET 'fakeoption'" {S 0 SENTINEL SET mymaster fakeoption fakevalue}
207
208 # save new config to disk failed
209 set info [S 0 SENTINEL master mymaster]
210 set origin_quorum [dict get $info quorum]
211 set update_quorum [expr $origin_quorum+1]
212 set sentinel_id 0
213 set configfilename [file join "sentinel_$sentinel_id" "sentinel.conf"]
214 set configfilename_bak [file join "sentinel_$sentinel_id" "sentinel.conf.bak"]
215
216 file rename $configfilename $configfilename_bak
217 file mkdir $configfilename
218
219 catch {[S 0 SENTINEL SET mymaster quorum $update_quorum]} err
220
221 file delete $configfilename
222 file rename $configfilename_bak $configfilename
223
224 assert_match "ERR Failed to save config file*" $err
225}
diff --git a/examples/redis-unstable/tests/sentinel/tests/04-slave-selection.tcl b/examples/redis-unstable/tests/sentinel/tests/04-slave-selection.tcl
new file mode 100644
index 0000000..3d2ca64
--- /dev/null
+++ b/examples/redis-unstable/tests/sentinel/tests/04-slave-selection.tcl
@@ -0,0 +1,5 @@
1# Test slave selection algorithm.
2#
3# This unit should test:
4# 1) That when there are no suitable slaves no failover is performed.
5# 2) That among the available slaves, the one with better offset is picked.
diff --git a/examples/redis-unstable/tests/sentinel/tests/05-manual.tcl b/examples/redis-unstable/tests/sentinel/tests/05-manual.tcl
new file mode 100644
index 0000000..95e8d41
--- /dev/null
+++ b/examples/redis-unstable/tests/sentinel/tests/05-manual.tcl
@@ -0,0 +1,94 @@
1# Test manual failover
2
3source "../tests/includes/init-tests.tcl"
4
5foreach_sentinel_id id {
6 S $id sentinel debug info-period 2000
7 S $id sentinel debug default-down-after 6000
8 S $id sentinel debug publish-period 1000
9}
10
11test "Manual failover works" {
12 set old_port [RPort $master_id]
13 set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster]
14 assert {[lindex $addr 1] == $old_port}
15
16 # Since we reduced the info-period (default 10000) above immediately,
17 # sentinel - replica may not have enough time to exchange INFO and update
18 # the replica's info-period, so the test may get a NOGOODSLAVE.
19 wait_for_condition 300 50 {
20 [catch {S 0 SENTINEL FAILOVER mymaster}] == 0
21 } else {
22 catch {S 0 SENTINEL FAILOVER mymaster} reply
23 puts [S 0 SENTINEL REPLICAS mymaster]
24 fail "Sentinel manual failover did not work, got: $reply"
25 }
26
27 catch {S 0 SENTINEL FAILOVER mymaster} reply
28 assert_match {*INPROG*} $reply ;# Failover already in progress
29
30 foreach_sentinel_id id {
31 wait_for_condition 1000 50 {
32 [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port
33 } else {
34 fail "At least one Sentinel did not receive failover info"
35 }
36 }
37 set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster]
38 set master_id [get_instance_id_by_port redis [lindex $addr 1]]
39}
40
41test "New master [join $addr {:}] role matches" {
42 assert {[RI $master_id role] eq {master}}
43}
44
45test "All the other slaves now point to the new master" {
46 foreach_redis_id id {
47 if {$id != $master_id && $id != 0} {
48 wait_for_condition 1000 50 {
49 [RI $id master_port] == [lindex $addr 1]
50 } else {
51 fail "Redis ID $id not configured to replicate with new master"
52 }
53 }
54 }
55}
56
57test "The old master eventually gets reconfigured as a slave" {
58 wait_for_condition 1000 50 {
59 [RI 0 master_port] == [lindex $addr 1]
60 } else {
61 fail "Old master not reconfigured as slave of new master"
62 }
63}
64
65foreach flag {crash-after-election crash-after-promotion} {
66 # Before each SIMULATE-FAILURE test, re-source init-tests to get a clean environment
67 source "../tests/includes/init-tests.tcl"
68
69 test "SENTINEL SIMULATE-FAILURE $flag works" {
70 assert_equal {OK} [S 0 SENTINEL SIMULATE-FAILURE $flag]
71
72 # Trigger a failover, failover will trigger leader election, replica promotion
73 # Sentinel may enter failover and exit before the command, catch it and allow it
74 wait_for_condition 300 50 {
75 [catch {S 0 SENTINEL FAILOVER mymaster}] == 0
76 ||
77 ([catch {S 0 SENTINEL FAILOVER mymaster} reply] == 1 &&
78 [string match {*couldn't open socket: connection refused*} $reply])
79 } else {
80 catch {S 0 SENTINEL FAILOVER mymaster} reply
81 fail "Sentinel manual failover did not work, got: $reply"
82 }
83
84 # Wait for sentinel to exit (due to simulate-failure flags)
85 wait_for_condition 1000 50 {
86 [catch {S 0 PING}] == 1
87 } else {
88 fail "Sentinel set $flag but did not exit"
89 }
90 assert_error {*couldn't open socket: connection refused*} {S 0 PING}
91
92 restart_instance sentinel 0
93 }
94}
diff --git a/examples/redis-unstable/tests/sentinel/tests/06-ckquorum.tcl b/examples/redis-unstable/tests/sentinel/tests/06-ckquorum.tcl
new file mode 100644
index 0000000..36c3dc6
--- /dev/null
+++ b/examples/redis-unstable/tests/sentinel/tests/06-ckquorum.tcl
@@ -0,0 +1,42 @@
1# Test for the SENTINEL CKQUORUM command
2
3source "../tests/includes/init-tests.tcl"
4set num_sentinels [llength $::sentinel_instances]
5
6test "CKQUORUM reports OK and the right amount of Sentinels" {
7 foreach_sentinel_id id {
8 assert_match "*OK $num_sentinels usable*" [S $id SENTINEL CKQUORUM mymaster]
9 }
10}
11
12test "CKQUORUM detects quorum cannot be reached" {
13 set orig_quorum [expr {$num_sentinels/2+1}]
14 S 0 SENTINEL SET mymaster quorum [expr {$num_sentinels+1}]
15 catch {[S 0 SENTINEL CKQUORUM mymaster]} err
16 assert_match "*NOQUORUM*" $err
17 S 0 SENTINEL SET mymaster quorum $orig_quorum
18}
19
20test "CKQUORUM detects failover authorization cannot be reached" {
21 set orig_quorum [expr {$num_sentinels/2+1}]
22 S 0 SENTINEL SET mymaster quorum 1
23 for {set i 0} {$i < $orig_quorum} {incr i} {
24 kill_instance sentinel [expr {$i + 1}]
25 }
26
27 # We need to make sure that other sentinels are in `DOWN` state
28 # from the point of view of S 0 before we executing `CKQUORUM`.
29 wait_for_condition 300 50 {
30 [catch {S 0 SENTINEL CKQUORUM mymaster}] == 1
31 } else {
32 fail "At least $orig_quorum sentinels did not enter the down state."
33 }
34
35 assert_error "*NOQUORUM*" {S 0 SENTINEL CKQUORUM mymaster}
36
37 S 0 SENTINEL SET mymaster quorum $orig_quorum
38 for {set i 0} {$i < $orig_quorum} {incr i} {
39 restart_instance sentinel [expr {$i + 1}]
40 }
41}
42
diff --git a/examples/redis-unstable/tests/sentinel/tests/07-down-conditions.tcl b/examples/redis-unstable/tests/sentinel/tests/07-down-conditions.tcl
new file mode 100644
index 0000000..dabbc14
--- /dev/null
+++ b/examples/redis-unstable/tests/sentinel/tests/07-down-conditions.tcl
@@ -0,0 +1,104 @@
1# Test conditions where an instance is considered to be down
2
3source "../tests/includes/init-tests.tcl"
4source "../../../tests/support/cli.tcl"
5
6foreach_sentinel_id id {
7 S $id sentinel debug info-period 1000
8 S $id sentinel debug ask-period 100
9 S $id sentinel debug default-down-after 3000
10 S $id sentinel debug publish-period 200
11 S $id sentinel debug ping-period 100
12}
13
14set ::alive_sentinel [expr {$::instances_count/2+2}]
15proc ensure_master_up {} {
16 S $::alive_sentinel sentinel debug info-period 1000
17 S $::alive_sentinel sentinel debug ping-period 100
18 S $::alive_sentinel sentinel debug ask-period 100
19 S $::alive_sentinel sentinel debug publish-period 100
20 wait_for_condition 1000 50 {
21 [dict get [S $::alive_sentinel sentinel master mymaster] flags] eq "master"
22 } else {
23 fail "Master flags are not just 'master'"
24 }
25}
26
27proc ensure_master_down {} {
28 S $::alive_sentinel sentinel debug info-period 1000
29 S $::alive_sentinel sentinel debug ping-period 100
30 S $::alive_sentinel sentinel debug ask-period 100
31 S $::alive_sentinel sentinel debug publish-period 100
32 wait_for_condition 1000 50 {
33 [string match *down* \
34 [dict get [S $::alive_sentinel sentinel master mymaster] flags]]
35 } else {
36 fail "Master is not flagged SDOWN"
37 }
38}
39
40test "Crash the majority of Sentinels to prevent failovers for this unit" {
41 for {set id 0} {$id < $quorum} {incr id} {
42 kill_instance sentinel $id
43 }
44}
45
46test "SDOWN is triggered by non-responding but not crashed instance" {
47 ensure_master_up
48 set master_addr [S $::alive_sentinel SENTINEL GET-MASTER-ADDR-BY-NAME mymaster]
49 set master_id [get_instance_id_by_port redis [lindex $master_addr 1]]
50
51 set pid [get_instance_attrib redis $master_id pid]
52 pause_process $pid
53 ensure_master_down
54 resume_process $pid
55 ensure_master_up
56}
57
58test "SDOWN is triggered by crashed instance" {
59 lassign [S $::alive_sentinel SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] host port
60 ensure_master_up
61 kill_instance redis 0
62 ensure_master_down
63 restart_instance redis 0
64 ensure_master_up
65}
66
67test "SDOWN is triggered by masters advertising as slaves" {
68 ensure_master_up
69 R 0 slaveof 127.0.0.1 34567
70 ensure_master_down
71 R 0 slaveof no one
72 ensure_master_up
73}
74
75if {!$::log_req_res} { # this test changes 'dir' config to '/' and logreqres.c cannot open protocol dump file under the root directory.
76test "SDOWN is triggered by misconfigured instance replying with errors" {
77 ensure_master_up
78 set orig_dir [lindex [R 0 config get dir] 1]
79 set orig_save [lindex [R 0 config get save] 1]
80 # Set dir to / and filename to "tmp" to make sure it will fail.
81 R 0 config set dir /
82 R 0 config set dbfilename tmp
83 R 0 config set save "1000000 1000000"
84 after 5000
85 R 0 bgsave
86 after 5000
87 ensure_master_down
88 R 0 config set save $orig_save
89 R 0 config set dir $orig_dir
90 R 0 config set dbfilename dump.rdb
91 R 0 bgsave
92 ensure_master_up
93}
94}
95
96# We use this test setup to also test command renaming, as a side
97# effect of the master going down if we send PONG instead of PING
98test "SDOWN is triggered if we rename PING to PONG" {
99 ensure_master_up
100 S $::alive_sentinel SENTINEL SET mymaster rename-command PING PONG
101 ensure_master_down
102 S $::alive_sentinel SENTINEL SET mymaster rename-command PING PING
103 ensure_master_up
104}
diff --git a/examples/redis-unstable/tests/sentinel/tests/08-hostname-conf.tcl b/examples/redis-unstable/tests/sentinel/tests/08-hostname-conf.tcl
new file mode 100644
index 0000000..263b06f
--- /dev/null
+++ b/examples/redis-unstable/tests/sentinel/tests/08-hostname-conf.tcl
@@ -0,0 +1,69 @@
1source "../tests/includes/utils.tcl"
2
3proc set_redis_announce_ip {addr} {
4 foreach_redis_id id {
5 R $id config set replica-announce-ip $addr
6 }
7}
8
9proc set_sentinel_config {keyword value} {
10 foreach_sentinel_id id {
11 S $id sentinel config set $keyword $value
12 }
13}
14
15proc set_all_instances_hostname {hostname} {
16 foreach_sentinel_id id {
17 set_instance_attrib sentinel $id host $hostname
18 }
19 foreach_redis_id id {
20 set_instance_attrib redis $id host $hostname
21 }
22}
23
24test "(pre-init) Configure instances and sentinel for hostname use" {
25 set ::host "localhost"
26 restart_killed_instances
27 set_all_instances_hostname $::host
28 set_redis_announce_ip $::host
29 set_sentinel_config resolve-hostnames yes
30 set_sentinel_config announce-hostnames yes
31}
32
33source "../tests/includes/init-tests.tcl"
34
35proc verify_hostname_announced {hostname} {
36 foreach_sentinel_id id {
37 # Master is reported with its hostname
38 if {![string equal [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 0] $hostname]} {
39 return 0
40 }
41
42 # Replicas are reported with their hostnames
43 foreach replica [S $id SENTINEL REPLICAS mymaster] {
44 if {![string equal [dict get $replica ip] $hostname]} {
45 return 0
46 }
47 }
48 }
49 return 1
50}
51
52test "Sentinel announces hostnames" {
53 # Check initial state
54 verify_hostname_announced $::host
55
56 # Disable announce-hostnames and confirm IPs are used
57 set_sentinel_config announce-hostnames no
58 assert {[verify_hostname_announced "127.0.0.1"] || [verify_hostname_announced "::1"]}
59}
60
61# We need to revert any special configuration because all tests currently
62# share the same instances.
63test "(post-cleanup) Configure instances and sentinel for IPs" {
64 set ::host "127.0.0.1"
65 set_all_instances_hostname $::host
66 set_redis_announce_ip $::host
67 set_sentinel_config resolve-hostnames no
68 set_sentinel_config announce-hostnames no
69} \ No newline at end of file
diff --git a/examples/redis-unstable/tests/sentinel/tests/09-acl-support.tcl b/examples/redis-unstable/tests/sentinel/tests/09-acl-support.tcl
new file mode 100644
index 0000000..a754dac
--- /dev/null
+++ b/examples/redis-unstable/tests/sentinel/tests/09-acl-support.tcl
@@ -0,0 +1,56 @@
1
2source "../tests/includes/init-tests.tcl"
3
4set ::user "testuser"
5set ::password "secret"
6
7proc setup_acl {} {
8 foreach_sentinel_id id {
9 assert_equal {OK} [S $id ACL SETUSER $::user >$::password +@all on]
10 assert_equal {OK} [S $id ACL SETUSER default off]
11
12 S $id CLIENT KILL USER default SKIPME no
13 assert_equal {OK} [S $id AUTH $::user $::password]
14 }
15}
16
17proc teardown_acl {} {
18 foreach_sentinel_id id {
19 assert_equal {OK} [S $id ACL SETUSER default on]
20 assert_equal {1} [S $id ACL DELUSER $::user]
21
22 S $id SENTINEL CONFIG SET sentinel-user ""
23 S $id SENTINEL CONFIG SET sentinel-pass ""
24 }
25}
26
27test "(post-init) Set up ACL configuration" {
28 setup_acl
29 assert_equal $::user [S 1 ACL WHOAMI]
30}
31
32test "SENTINEL CONFIG SET handles on-the-fly credentials reconfiguration" {
33 # Make sure we're starting with a broken state...
34 wait_for_condition 200 50 {
35 [catch {S 1 SENTINEL CKQUORUM mymaster}] == 1
36 } else {
37 fail "Expected: Sentinel to be disconnected from master due to wrong password"
38 }
39 assert_error "*NOQUORUM*" {S 1 SENTINEL CKQUORUM mymaster}
40
41 foreach_sentinel_id id {
42 assert_equal {OK} [S $id SENTINEL CONFIG SET sentinel-user $::user]
43 assert_equal {OK} [S $id SENTINEL CONFIG SET sentinel-pass $::password]
44 }
45
46 wait_for_condition 200 50 {
47 [catch {S 1 SENTINEL CKQUORUM mymaster}] == 0
48 } else {
49 fail "Expected: Sentinel to be connected to master after setting password"
50 }
51 assert_match {*OK*} [S 1 SENTINEL CKQUORUM mymaster]
52}
53
54test "(post-cleanup) Tear down ACL configuration" {
55 teardown_acl
56}
diff --git a/examples/redis-unstable/tests/sentinel/tests/10-replica-priority.tcl b/examples/redis-unstable/tests/sentinel/tests/10-replica-priority.tcl
new file mode 100644
index 0000000..d3f868a
--- /dev/null
+++ b/examples/redis-unstable/tests/sentinel/tests/10-replica-priority.tcl
@@ -0,0 +1,76 @@
1source "../tests/includes/init-tests.tcl"
2
3test "Check acceptable replica-priority values" {
4 foreach_redis_id id {
5 if {$id == $master_id} continue
6
7 # ensure replica-announced accepts yes and no
8 catch {R $id CONFIG SET replica-announced no} e
9 if {$e ne "OK"} {
10 fail "Unable to set replica-announced to no"
11 }
12 catch {R $id CONFIG SET replica-announced yes} e
13 if {$e ne "OK"} {
14 fail "Unable to set replica-announced to yes"
15 }
16
17 # ensure a random value throw error
18 catch {R $id CONFIG SET replica-announced 321} e
19 if {$e eq "OK"} {
20 fail "Able to set replica-announced with something else than yes or no (321) whereas it should not be possible"
21 }
22 catch {R $id CONFIG SET replica-announced a3b2c1} e
23 if {$e eq "OK"} {
24 fail "Able to set replica-announced with something else than yes or no (a3b2c1) whereas it should not be possible"
25 }
26
27 # test only the first redis replica, no need to double test
28 break
29 }
30}
31
32proc 10_test_number_of_replicas {n_replicas_expected} {
33 test "Check sentinel replies with $n_replicas_expected replicas" {
34 # ensure sentinels replies with the right number of replicas
35 foreach_sentinel_id id {
36 S $id sentinel debug info-period 100
37 S $id sentinel debug default-down-after 1000
38 S $id sentinel debug publish-period 100
39 set len [llength [S $id SENTINEL REPLICAS mymaster]]
40 wait_for_condition 200 100 {
41 [llength [S $id SENTINEL REPLICAS mymaster]] == $n_replicas_expected
42 } else {
43 fail "Sentinel replies with a wrong number of replicas with replica-announced=yes (expected $n_replicas_expected but got $len) on sentinel $id"
44 }
45 }
46 }
47}
48
49proc 10_set_replica_announced {master_id announced n_replicas} {
50 test "Set replica-announced=$announced on $n_replicas replicas" {
51 set i 0
52 foreach_redis_id id {
53 if {$id == $master_id} continue
54 #puts "set replica-announce=$announced on redis #$id"
55 R $id CONFIG SET replica-announced "$announced"
56 incr i
57 if { $n_replicas!="all" && $i >= $n_replicas } { break }
58 }
59 }
60}
61
62# ensure all replicas are announced
6310_set_replica_announced $master_id "yes" "all"
64# ensure all replicas are announced by sentinels
6510_test_number_of_replicas 4
66
67# ensure the first 2 replicas are not announced
6810_set_replica_announced $master_id "no" 2
69# ensure sentinels are not announcing the first 2 replicas that have been set unannounced
7010_test_number_of_replicas 2
71
72# ensure all replicas are announced
7310_set_replica_announced $master_id "yes" "all"
74# ensure all replicas are not announced by sentinels
7510_test_number_of_replicas 4
76
diff --git a/examples/redis-unstable/tests/sentinel/tests/11-port-0.tcl b/examples/redis-unstable/tests/sentinel/tests/11-port-0.tcl
new file mode 100644
index 0000000..a3e8bdb
--- /dev/null
+++ b/examples/redis-unstable/tests/sentinel/tests/11-port-0.tcl
@@ -0,0 +1,33 @@
1source "../tests/includes/init-tests.tcl"
2
3test "Start/Stop sentinel on same port with a different runID should not change the total number of sentinels" {
4 set sentinel_id [expr $::instances_count-1]
5 # Kill sentinel instance
6 kill_instance sentinel $sentinel_id
7
8 # Delete line with myid in sentinels config file
9 set orgfilename [file join "sentinel_$sentinel_id" "sentinel.conf"]
10 set tmpfilename "sentinel.conf_tmp"
11 set dirname "sentinel_$sentinel_id"
12
13 delete_lines_with_pattern $orgfilename $tmpfilename "myid"
14
15 # Get count of total sentinels
16 set a [S 0 SENTINEL master mymaster]
17 set original_count [lindex $a 33]
18
19 # Restart sentinel with the modified config file
20 set pid [exec_instance "sentinel" $dirname $orgfilename]
21 lappend ::pids $pid
22
23 after 1000
24
25 # Get new count of total sentinel
26 set b [S 0 SENTINEL master mymaster]
27 set curr_count [lindex $b 33]
28
29 # If the count is not the same then fail the test
30 if {$original_count != $curr_count} {
31 fail "Sentinel count is incorrect, original count being $original_count and current count is $curr_count"
32 }
33}
diff --git a/examples/redis-unstable/tests/sentinel/tests/12-master-reboot.tcl b/examples/redis-unstable/tests/sentinel/tests/12-master-reboot.tcl
new file mode 100644
index 0000000..3d0d828
--- /dev/null
+++ b/examples/redis-unstable/tests/sentinel/tests/12-master-reboot.tcl
@@ -0,0 +1,112 @@
1# Check the basic monitoring and failover capabilities.
2source "../tests/includes/init-tests.tcl"
3
4
5if {$::simulate_error} {
6 test "This test will fail" {
7 fail "Simulated error"
8 }
9}
10
11
12# Reboot an instance previously in very short time but do not check if it is loading
13proc reboot_instance {type id} {
14 set dirname "${type}_${id}"
15 set cfgfile [file join $dirname $type.conf]
16 set port [get_instance_attrib $type $id port]
17
18 # Execute the instance with its old setup and append the new pid
19 # file for cleanup.
20 set pid [exec_instance $type $dirname $cfgfile]
21 set_instance_attrib $type $id pid $pid
22 lappend ::pids $pid
23
24 # Check that the instance is running
25 if {[server_is_up 127.0.0.1 $port 100] == 0} {
26 set logfile [file join $dirname log.txt]
27 puts [exec tail $logfile]
28 abort_sentinel_test "Problems starting $type #$id: ping timeout, maybe server start failed, check $logfile"
29 }
30
31 # Connect with it with a fresh link
32 set link [redis 127.0.0.1 $port 0 $::tls]
33 $link reconnect 1
34 set_instance_attrib $type $id link $link
35}
36
37
38test "Master reboot in very short time" {
39 set old_port [RPort $master_id]
40 set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster]
41 assert {[lindex $addr 1] == $old_port}
42
43 R $master_id debug populate 20000
44 R $master_id bgsave
45 R $master_id config set key-load-delay 1500
46 R $master_id config set loading-process-events-interval-bytes 1024
47 R $master_id config rewrite
48
49 foreach_sentinel_id id {
50 S $id SENTINEL SET mymaster master-reboot-down-after-period 5000
51 S $id sentinel debug ping-period 500
52 S $id sentinel debug ask-period 500
53 }
54
55 kill_instance redis $master_id
56 reboot_instance redis $master_id
57
58 set max_tries 1000
59 if {$::tsan} {
60 set max_tries 5000
61 }
62
63 foreach_sentinel_id id {
64 wait_for_condition $max_tries 100 {
65 [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port
66 } else {
67 fail "At least one Sentinel did not receive failover info"
68 }
69 }
70
71 # Reset configuration to avoid unnecessary delays
72 R $master_id config set key-load-delay 0
73 R $master_id config rewrite
74
75 set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster]
76 set master_id [get_instance_id_by_port redis [lindex $addr 1]]
77
78 # Make sure the instance load all the dataset
79 while 1 {
80 catch {[$link ping]} retval
81 if {[string match {*LOADING*} $retval]} {
82 after 100
83 continue
84 } else {
85 break
86 }
87 }
88}
89
90test "New master [join $addr {:}] role matches" {
91 assert {[RI $master_id role] eq {master}}
92}
93
94test "All the other slaves now point to the new master" {
95 foreach_redis_id id {
96 if {$id != $master_id && $id != 0} {
97 wait_for_condition 1000 50 {
98 [RI $id master_port] == [lindex $addr 1]
99 } else {
100 fail "Redis ID $id not configured to replicate with new master"
101 }
102 }
103 }
104}
105
106test "The old master eventually gets reconfigured as a slave" {
107 wait_for_condition 1000 50 {
108 [RI 0 master_port] == [lindex $addr 1]
109 } else {
110 fail "Old master not reconfigured as slave of new master"
111 }
112}
diff --git a/examples/redis-unstable/tests/sentinel/tests/13-info-command.tcl b/examples/redis-unstable/tests/sentinel/tests/13-info-command.tcl
new file mode 100644
index 0000000..ef9dc01
--- /dev/null
+++ b/examples/redis-unstable/tests/sentinel/tests/13-info-command.tcl
@@ -0,0 +1,47 @@
1source "../tests/includes/init-tests.tcl"
2
3test "info command with at most one argument" {
4 set subCommandList {}
5 foreach arg {"" "all" "default" "everything"} {
6 if {$arg == ""} {
7 set info [S 0 info]
8 } else {
9 set info [S 0 info $arg]
10 }
11 assert { [string match "*redis_version*" $info] }
12 assert { [string match "*maxclients*" $info] }
13 assert { [string match "*used_cpu_user*" $info] }
14 assert { [string match "*sentinel_tilt*" $info] }
15 assert { ![string match "*used_memory*" $info] }
16 assert { ![string match "*rdb_last_bgsave*" $info] }
17 assert { ![string match "*master_repl_offset*" $info] }
18 assert { ![string match "*cluster_enabled*" $info] }
19 }
20}
21
22test "info command with one sub-section" {
23 set info [S 0 info cpu]
24 assert { [string match "*used_cpu_user*" $info] }
25 assert { ![string match "*sentinel_tilt*" $info] }
26 assert { ![string match "*redis_version*" $info] }
27
28 set info [S 0 info sentinel]
29 assert { [string match "*sentinel_tilt*" $info] }
30 assert { ![string match "*used_cpu_user*" $info] }
31 assert { ![string match "*redis_version*" $info] }
32}
33
34test "info command with multiple sub-sections" {
35 set info [S 0 info server sentinel replication]
36 assert { [string match "*redis_version*" $info] }
37 assert { [string match "*sentinel_tilt*" $info] }
38 assert { ![string match "*used_memory*" $info] }
39 assert { ![string match "*used_cpu_user*" $info] }
40
41 set info [S 0 info cpu all]
42 assert { [string match "*used_cpu_user*" $info] }
43 assert { [string match "*sentinel_tilt*" $info] }
44 assert { [string match "*redis_version*" $info] }
45 assert { ![string match "*used_memory*" $info] }
46 assert { ![string match "*master_repl_offset*" $info] }
47}
diff --git a/examples/redis-unstable/tests/sentinel/tests/14-debug-command.tcl b/examples/redis-unstable/tests/sentinel/tests/14-debug-command.tcl
new file mode 100644
index 0000000..dccb992
--- /dev/null
+++ b/examples/redis-unstable/tests/sentinel/tests/14-debug-command.tcl
@@ -0,0 +1,9 @@
1source "../tests/includes/init-tests.tcl"
2
3test "Sentinel debug test with arguments and without argument" {
4 set current_info_period [lindex [S 0 SENTINEL DEBUG] 1]
5 S 0 SENTINEL DEBUG info-period 8888
6 assert_equal {8888} [lindex [S 0 SENTINEL DEBUG] 1]
7 S 0 SENTINEL DEBUG info-period $current_info_period
8 assert_equal $current_info_period [lindex [S 0 SENTINEL DEBUG] 1]
9}
diff --git a/examples/redis-unstable/tests/sentinel/tests/15-config-set-config-get.tcl b/examples/redis-unstable/tests/sentinel/tests/15-config-set-config-get.tcl
new file mode 100644
index 0000000..f9831f8
--- /dev/null
+++ b/examples/redis-unstable/tests/sentinel/tests/15-config-set-config-get.tcl
@@ -0,0 +1,58 @@
1source "../tests/includes/init-tests.tcl"
2
3test "SENTINEL CONFIG SET and SENTINEL CONFIG GET handles multiple variables" {
4 foreach_sentinel_id id {
5 assert_equal {OK} [S $id SENTINEL CONFIG SET resolve-hostnames yes announce-port 1234]
6 }
7 assert_match {*yes*1234*} [S 1 SENTINEL CONFIG GET resolve-hostnames announce-port]
8 assert_match {announce-port 1234} [S 1 SENTINEL CONFIG GET announce-port]
9}
10
11test "SENTINEL CONFIG GET for duplicate and unknown variables" {
12 assert_equal {OK} [S 1 SENTINEL CONFIG SET resolve-hostnames yes announce-port 1234]
13 assert_match {resolve-hostnames yes} [S 1 SENTINEL CONFIG GET resolve-hostnames resolve-hostnames does-not-exist]
14}
15
16test "SENTINEL CONFIG GET for patterns" {
17 assert_equal {OK} [S 1 SENTINEL CONFIG SET loglevel notice announce-port 1234 announce-hostnames yes ]
18 assert_match {loglevel notice} [S 1 SENTINEL CONFIG GET log* *level loglevel]
19 assert_match {announce-hostnames yes announce-ip*announce-port 1234} [S 1 SENTINEL CONFIG GET announce*]
20}
21
22test "SENTINEL CONFIG SET duplicate variables" {
23 catch {[S 1 SENTINEL CONFIG SET resolve-hostnames yes announce-port 1234 announce-port 100]} e
24 if {![string match "*Duplicate argument*" $e]} {
25 fail "Should give wrong arity error"
26 }
27}
28
29test "SENTINEL CONFIG SET, one option does not exist" {
30 foreach_sentinel_id id {
31 assert_equal {OK} [S $id SENTINEL CONFIG SET announce-port 111]
32 catch {[S $id SENTINEL CONFIG SET does-not-exist yes announce-port 1234]} e
33 if {![string match "*Invalid argument*" $e]} {
34 fail "Should give Invalid argument error"
35 }
36 }
37 # The announce-port should not be set to 1234 as it was called with a wrong argument
38 assert_match {*111*} [S 1 SENTINEL CONFIG GET announce-port]
39}
40
41test "SENTINEL CONFIG SET, one option with wrong value" {
42 foreach_sentinel_id id {
43 assert_equal {OK} [S $id SENTINEL CONFIG SET resolve-hostnames no]
44 catch {[S $id SENTINEL CONFIG SET announce-port -1234 resolve-hostnames yes]} e
45 if {![string match "*Invalid value*" $e]} {
46 fail "Expected to return Invalid value error"
47 }
48 }
49 # The resolve-hostnames should not be set to yes as it was called after an argument with an invalid value
50 assert_match {*no*} [S 1 SENTINEL CONFIG GET resolve-hostnames]
51}
52
53test "SENTINEL CONFIG SET, wrong number of arguments" {
54 catch {[S 1 SENTINEL CONFIG SET resolve-hostnames yes announce-port 1234 announce-ip]} e
55 if {![string match "*Missing argument*" $e]} {
56 fail "Expected to return Missing argument error"
57 }
58}
diff --git a/examples/redis-unstable/tests/sentinel/tests/helpers/check_leaked_fds.tcl b/examples/redis-unstable/tests/sentinel/tests/helpers/check_leaked_fds.tcl
new file mode 100755
index 0000000..482b3e0
--- /dev/null
+++ b/examples/redis-unstable/tests/sentinel/tests/helpers/check_leaked_fds.tcl
@@ -0,0 +1,79 @@
1#!/usr/bin/env tclsh
2#
3# This script detects file descriptors that have leaked from a parent process.
4#
5# Our goal is to detect file descriptors that were opened by the parent and
6# not cleaned up prior to exec(), but not file descriptors that were inherited
7# from the grandparent which the parent knows nothing about. To do that, we
8# look up every potential leak and try to match it against open files by the
9# grandparent process.
10
11# Get PID of parent process
12proc get_parent_pid {_pid} {
13 set fd [open "/proc/$_pid/status" "r"]
14 set content [read $fd]
15 close $fd
16
17 if {[regexp {\nPPid:\s+(\d+)} $content _ ppid]} {
18 return $ppid
19 }
20
21 error "failed to get parent pid"
22}
23
24# Read symlink to get info about the specified fd of the specified process.
25# The result can be the file name or an arbitrary string that identifies it.
26# When not able to read, an empty string is returned.
27proc get_fdlink {_pid fd} {
28 if { [catch {set fdlink [file readlink "/proc/$_pid/fd/$fd"]} err] } {
29 return ""
30 }
31 return $fdlink
32}
33
34# Linux only
35set os [exec uname]
36if {$os != "Linux"} {
37 puts "Only Linux is supported."
38 exit 0
39}
40
41if {![info exists env(LEAKED_FDS_FILE)]} {
42 puts "Missing LEAKED_FDS_FILE environment variable."
43 exit 0
44}
45
46set outfile $::env(LEAKED_FDS_FILE)
47set parent_pid [get_parent_pid [pid]]
48set grandparent_pid [get_parent_pid $parent_pid]
49set leaked_fds {}
50
51# Look for fds that were directly inherited from our parent but not from
52# our grandparent (tcl)
53foreach fd [glob -tails -directory "/proc/self/fd" *] {
54 # Ignore stdin/stdout/stderr
55 if {$fd == 0 || $fd == 1 || $fd == 2} {
56 continue
57 }
58
59 set fdlink [get_fdlink "self" $fd]
60 if {$fdlink == ""} {
61 continue
62 }
63
64 # We ignore fds that existed in the grandparent, or fds that don't exist
65 # in our parent (Sentinel process).
66 if {[get_fdlink $grandparent_pid $fd] == $fdlink ||
67 [get_fdlink $parent_pid $fd] != $fdlink} {
68 continue
69 }
70
71 lappend leaked_fds [list $fd $fdlink]
72}
73
74# Produce report only if we found leaks
75if {[llength $leaked_fds] > 0} {
76 set fd [open $outfile "w"]
77 puts $fd [join $leaked_fds "\n"]
78 close $fd
79}
diff --git a/examples/redis-unstable/tests/sentinel/tests/includes/init-tests.tcl b/examples/redis-unstable/tests/sentinel/tests/includes/init-tests.tcl
new file mode 100644
index 0000000..ddb1319
--- /dev/null
+++ b/examples/redis-unstable/tests/sentinel/tests/includes/init-tests.tcl
@@ -0,0 +1,63 @@
1# Initialization tests -- most units will start including this.
2source "../tests/includes/utils.tcl"
3
4test "(init) Restart killed instances" {
5 restart_killed_instances
6}
7
8test "(init) Remove old master entry from sentinels" {
9 foreach_sentinel_id id {
10 catch {S $id SENTINEL REMOVE mymaster}
11 }
12}
13
14set redis_slaves [expr $::instances_count - 1]
15test "(init) Create a master-slaves cluster of [expr $redis_slaves+1] instances" {
16 create_redis_master_slave_cluster [expr {$redis_slaves+1}]
17}
18set master_id 0
19
20test "(init) Sentinels can start monitoring a master" {
21 set sentinels [llength $::sentinel_instances]
22 set quorum [expr {$sentinels/2+1}]
23 foreach_sentinel_id id {
24 S $id SENTINEL MONITOR mymaster \
25 [get_instance_attrib redis $master_id host] \
26 [get_instance_attrib redis $master_id port] $quorum
27 }
28 foreach_sentinel_id id {
29 assert {[S $id sentinel master mymaster] ne {}}
30 S $id SENTINEL SET mymaster down-after-milliseconds 2000
31 S $id SENTINEL SET mymaster failover-timeout 10000
32 S $id SENTINEL debug tilt-period 5000
33 S $id SENTINEL SET mymaster parallel-syncs 10
34 if {$::leaked_fds_file != "" && [exec uname] == "Linux"} {
35 S $id SENTINEL SET mymaster notification-script ../../tests/helpers/check_leaked_fds.tcl
36 S $id SENTINEL SET mymaster client-reconfig-script ../../tests/helpers/check_leaked_fds.tcl
37 }
38 }
39}
40
41test "(init) Sentinels can talk with the master" {
42 foreach_sentinel_id id {
43 wait_for_condition 1000 50 {
44 [catch {S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster}] == 0
45 } else {
46 fail "Sentinel $id can't talk with the master."
47 }
48 }
49}
50
51test "(init) Sentinels are able to auto-discover other sentinels" {
52 verify_sentinel_auto_discovery
53}
54
55test "(init) Sentinels are able to auto-discover slaves" {
56 foreach_sentinel_id id {
57 wait_for_condition 1000 50 {
58 [dict get [S $id SENTINEL MASTER mymaster] num-slaves] == $redis_slaves
59 } else {
60 fail "At least some sentinel can't detect some slave"
61 }
62 }
63}
diff --git a/examples/redis-unstable/tests/sentinel/tests/includes/sentinel.conf b/examples/redis-unstable/tests/sentinel/tests/includes/sentinel.conf
new file mode 100644
index 0000000..1275236
--- /dev/null
+++ b/examples/redis-unstable/tests/sentinel/tests/includes/sentinel.conf
@@ -0,0 +1,9 @@
1# assume master is down after being unresponsive for 20s
2sentinel down-after-milliseconds setmaster 20000
3# reconfigure one slave at a time
4sentinel parallel-syncs setmaster 2
5# wait for 4m before assuming failover went wrong
6sentinel failover-timeout setmaster 240000
7# monitoring set
8sentinel monitor setmaster 10.0.0.1 30000 2
9
diff --git a/examples/redis-unstable/tests/sentinel/tests/includes/start-init-tests.tcl b/examples/redis-unstable/tests/sentinel/tests/includes/start-init-tests.tcl
new file mode 100644
index 0000000..b052350
--- /dev/null
+++ b/examples/redis-unstable/tests/sentinel/tests/includes/start-init-tests.tcl
@@ -0,0 +1,18 @@
1test "(start-init) Flush config and compare rewrite config file lines" {
2 foreach_sentinel_id id {
3 assert_match "OK" [S $id SENTINEL FLUSHCONFIG]
4 set file1 ../tests/includes/sentinel.conf
5 set file2 [file join "sentinel_${id}" "sentinel.conf"]
6 set fh1 [open $file1 r]
7 set fh2 [open $file2 r]
8 while {[gets $fh1 line1]} {
9 if {[gets $fh2 line2]} {
10 assert [string equal $line1 $line2]
11 } else {
12 fail "sentinel config file rewrite sequence changed"
13 }
14 }
15 close $fh1
16 close $fh2
17 }
18} \ No newline at end of file
diff --git a/examples/redis-unstable/tests/sentinel/tests/includes/utils.tcl b/examples/redis-unstable/tests/sentinel/tests/includes/utils.tcl
new file mode 100644
index 0000000..adfd91c
--- /dev/null
+++ b/examples/redis-unstable/tests/sentinel/tests/includes/utils.tcl
@@ -0,0 +1,22 @@
1proc restart_killed_instances {} {
2 foreach type {redis sentinel} {
3 foreach_${type}_id id {
4 if {[get_instance_attrib $type $id pid] == -1} {
5 puts -nonewline "$type/$id "
6 flush stdout
7 restart_instance $type $id
8 }
9 }
10 }
11}
12
13proc verify_sentinel_auto_discovery {} {
14 set sentinels [llength $::sentinel_instances]
15 foreach_sentinel_id id {
16 wait_for_condition 1000 50 {
17 [dict get [S $id SENTINEL MASTER mymaster] num-other-sentinels] == ($sentinels-1)
18 } else {
19 fail "At least some sentinel can't detect some other sentinel"
20 }
21 }
22}
diff --git a/examples/redis-unstable/tests/sentinel/tmp/.gitignore b/examples/redis-unstable/tests/sentinel/tmp/.gitignore
new file mode 100644
index 0000000..f581f73
--- /dev/null
+++ b/examples/redis-unstable/tests/sentinel/tmp/.gitignore
@@ -0,0 +1,2 @@
1redis_*
2sentinel_*