diff options
Diffstat (limited to 'examples/redis-unstable/tests/cluster/cluster.tcl')
| -rw-r--r-- | examples/redis-unstable/tests/cluster/cluster.tcl | 270 |
1 files changed, 270 insertions, 0 deletions
diff --git a/examples/redis-unstable/tests/cluster/cluster.tcl b/examples/redis-unstable/tests/cluster/cluster.tcl new file mode 100644 index 0000000..15218d1 --- /dev/null +++ b/examples/redis-unstable/tests/cluster/cluster.tcl | |||
| @@ -0,0 +1,270 @@ | |||
| 1 | # Cluster-specific test functions. | ||
| 2 | # | ||
| 3 | # Copyright (C) 2014-Present, Redis Ltd. | ||
| 4 | # All Rights reserved. | ||
| 5 | # | ||
| 6 | # Licensed under your choice of (a) the Redis Source Available License 2.0 | ||
| 7 | # (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the | ||
| 8 | # GNU Affero General Public License v3 (AGPLv3). | ||
| 9 | |||
| 10 | # Track cluster configuration as created by create_cluster below | ||
| 11 | set ::cluster_master_nodes 0 | ||
| 12 | set ::cluster_replica_nodes 0 | ||
| 13 | |||
| 14 | # Returns a parsed CLUSTER NODES output as a list of dictionaries. Optional status field | ||
| 15 | # can be specified to only returns entries that match the provided status. | ||
| 16 | proc get_cluster_nodes {id {status "*"}} { | ||
| 17 | set lines [split [R $id cluster nodes] "\r\n"] | ||
| 18 | set nodes {} | ||
| 19 | foreach l $lines { | ||
| 20 | set l [string trim $l] | ||
| 21 | if {$l eq {}} continue | ||
| 22 | set args [split $l] | ||
| 23 | set node [dict create \ | ||
| 24 | id [lindex $args 0] \ | ||
| 25 | addr [lindex $args 1] \ | ||
| 26 | flags [split [lindex $args 2] ,] \ | ||
| 27 | slaveof [lindex $args 3] \ | ||
| 28 | ping_sent [lindex $args 4] \ | ||
| 29 | pong_recv [lindex $args 5] \ | ||
| 30 | config_epoch [lindex $args 6] \ | ||
| 31 | linkstate [lindex $args 7] \ | ||
| 32 | slots [lrange $args 8 end] \ | ||
| 33 | ] | ||
| 34 | if {[string match $status [lindex $args 7]]} { | ||
| 35 | lappend nodes $node | ||
| 36 | } | ||
| 37 | } | ||
| 38 | return $nodes | ||
| 39 | } | ||
| 40 | |||
| 41 | # Test node for flag. | ||
| 42 | proc has_flag {node flag} { | ||
| 43 | expr {[lsearch -exact [dict get $node flags] $flag] != -1} | ||
| 44 | } | ||
| 45 | |||
| 46 | # Returns the parsed myself node entry as a dictionary. | ||
| 47 | proc get_myself id { | ||
| 48 | set nodes [get_cluster_nodes $id] | ||
| 49 | foreach n $nodes { | ||
| 50 | if {[has_flag $n myself]} {return $n} | ||
| 51 | } | ||
| 52 | return {} | ||
| 53 | } | ||
| 54 | |||
| 55 | # Get a specific node by ID by parsing the CLUSTER NODES output | ||
| 56 | # of the instance Number 'instance_id' | ||
| 57 | proc get_node_by_id {instance_id node_id} { | ||
| 58 | set nodes [get_cluster_nodes $instance_id] | ||
| 59 | foreach n $nodes { | ||
| 60 | if {[dict get $n id] eq $node_id} {return $n} | ||
| 61 | } | ||
| 62 | return {} | ||
| 63 | } | ||
| 64 | |||
| 65 | # Return the value of the specified CLUSTER INFO field. | ||
| 66 | proc CI {n field} { | ||
| 67 | get_info_field [R $n cluster info] $field | ||
| 68 | } | ||
| 69 | |||
| 70 | # Return the value of the specified INFO field. | ||
| 71 | proc s {n field} { | ||
| 72 | get_info_field [R $n info] $field | ||
| 73 | } | ||
| 74 | |||
| 75 | # Assuming nodes are reset, this function performs slots allocation. | ||
| 76 | # Only the first 'n' nodes are used. | ||
| 77 | proc cluster_allocate_slots {n} { | ||
| 78 | set slot 16383 | ||
| 79 | while {$slot >= 0} { | ||
| 80 | # Allocate successive slots to random nodes. | ||
| 81 | set node [randomInt $n] | ||
| 82 | lappend slots_$node $slot | ||
| 83 | incr slot -1 | ||
| 84 | } | ||
| 85 | for {set j 0} {$j < $n} {incr j} { | ||
| 86 | R $j cluster addslots {*}[set slots_${j}] | ||
| 87 | } | ||
| 88 | } | ||
| 89 | |||
| 90 | # Check that cluster nodes agree about "state", or raise an error. | ||
| 91 | proc assert_cluster_state {state} { | ||
| 92 | foreach_redis_id id { | ||
| 93 | if {[instance_is_killed redis $id]} continue | ||
| 94 | wait_for_condition 1000 50 { | ||
| 95 | [CI $id cluster_state] eq $state | ||
| 96 | } else { | ||
| 97 | fail "Cluster node $id cluster_state:[CI $id cluster_state]" | ||
| 98 | } | ||
| 99 | } | ||
| 100 | |||
| 101 | wait_for_secrets_match 50 100 | ||
| 102 | } | ||
| 103 | |||
| 104 | proc num_unique_secrets {} { | ||
| 105 | set secrets [list] | ||
| 106 | foreach_redis_id id { | ||
| 107 | if {[instance_is_killed redis $id]} continue | ||
| 108 | lappend secrets [R $id debug internal_secret] | ||
| 109 | } | ||
| 110 | set num_secrets [llength [lsort -unique $secrets]] | ||
| 111 | return $num_secrets | ||
| 112 | } | ||
| 113 | |||
| 114 | # Check that cluster nodes agree about "state", or raise an error. | ||
| 115 | proc assert_secrets_match {} { | ||
| 116 | assert_equal {1} [num_unique_secrets] | ||
| 117 | } | ||
| 118 | |||
| 119 | proc wait_for_secrets_match {maxtries delay} { | ||
| 120 | wait_for_condition $maxtries $delay { | ||
| 121 | [num_unique_secrets] eq 1 | ||
| 122 | } else { | ||
| 123 | fail "Failed waiting for secrets to sync" | ||
| 124 | } | ||
| 125 | } | ||
| 126 | |||
| 127 | # Search the first node starting from ID $first that is not | ||
| 128 | # already configured as a slave. | ||
| 129 | proc cluster_find_available_slave {first} { | ||
| 130 | foreach_redis_id id { | ||
| 131 | if {$id < $first} continue | ||
| 132 | if {[instance_is_killed redis $id]} continue | ||
| 133 | set me [get_myself $id] | ||
| 134 | if {[dict get $me slaveof] eq {-}} {return $id} | ||
| 135 | } | ||
| 136 | fail "No available slaves" | ||
| 137 | } | ||
| 138 | |||
| 139 | # Add 'slaves' slaves to a cluster composed of 'masters' masters. | ||
| 140 | # It assumes that masters are allocated sequentially from instance ID 0 | ||
| 141 | # to N-1. | ||
| 142 | proc cluster_allocate_slaves {masters slaves} { | ||
| 143 | for {set j 0} {$j < $slaves} {incr j} { | ||
| 144 | set master_id [expr {$j % $masters}] | ||
| 145 | set slave_id [cluster_find_available_slave $masters] | ||
| 146 | set master_myself [get_myself $master_id] | ||
| 147 | R $slave_id cluster replicate [dict get $master_myself id] | ||
| 148 | } | ||
| 149 | } | ||
| 150 | |||
| 151 | # Create a cluster composed of the specified number of masters and slaves. | ||
| 152 | proc create_cluster {masters slaves} { | ||
| 153 | cluster_allocate_slots $masters | ||
| 154 | if {$slaves} { | ||
| 155 | cluster_allocate_slaves $masters $slaves | ||
| 156 | } | ||
| 157 | assert_cluster_state ok | ||
| 158 | |||
| 159 | set ::cluster_master_nodes $masters | ||
| 160 | set ::cluster_replica_nodes $slaves | ||
| 161 | } | ||
| 162 | |||
| 163 | proc cluster_allocate_with_continuous_slots {n} { | ||
| 164 | set slot 16383 | ||
| 165 | set avg [expr ($slot+1) / $n] | ||
| 166 | while {$slot >= 0} { | ||
| 167 | set node [expr $slot/$avg >= $n ? $n-1 : $slot/$avg] | ||
| 168 | lappend slots_$node $slot | ||
| 169 | incr slot -1 | ||
| 170 | } | ||
| 171 | for {set j 0} {$j < $n} {incr j} { | ||
| 172 | R $j cluster addslots {*}[set slots_${j}] | ||
| 173 | } | ||
| 174 | } | ||
| 175 | |||
| 176 | # Create a cluster composed of the specified number of masters and slaves, | ||
| 177 | # but with a continuous slot range. | ||
| 178 | proc cluster_create_with_continuous_slots {masters slaves} { | ||
| 179 | cluster_allocate_with_continuous_slots $masters | ||
| 180 | if {$slaves} { | ||
| 181 | cluster_allocate_slaves $masters $slaves | ||
| 182 | } | ||
| 183 | assert_cluster_state ok | ||
| 184 | |||
| 185 | set ::cluster_master_nodes $masters | ||
| 186 | set ::cluster_replica_nodes $slaves | ||
| 187 | } | ||
| 188 | |||
| 189 | |||
| 190 | # Set the cluster node-timeout to all the reachalbe nodes. | ||
| 191 | proc set_cluster_node_timeout {to} { | ||
| 192 | foreach_redis_id id { | ||
| 193 | catch {R $id CONFIG SET cluster-node-timeout $to} | ||
| 194 | } | ||
| 195 | } | ||
| 196 | |||
| 197 | # Check if the cluster is writable and readable. Use node "id" | ||
| 198 | # as a starting point to talk with the cluster. | ||
| 199 | proc cluster_write_test {id} { | ||
| 200 | set prefix [randstring 20 20 alpha] | ||
| 201 | set port [get_instance_attrib redis $id port] | ||
| 202 | set cluster [redis_cluster 127.0.0.1:$port] | ||
| 203 | for {set j 0} {$j < 100} {incr j} { | ||
| 204 | $cluster set key.$j $prefix.$j | ||
| 205 | } | ||
| 206 | for {set j 0} {$j < 100} {incr j} { | ||
| 207 | assert {[$cluster get key.$j] eq "$prefix.$j"} | ||
| 208 | } | ||
| 209 | $cluster close | ||
| 210 | } | ||
| 211 | |||
| 212 | # Normalize cluster slots configuration by sorting replicas by node ID | ||
| 213 | proc normalize_cluster_slots {slots_config} { | ||
| 214 | set normalized {} | ||
| 215 | foreach slot_range $slots_config { | ||
| 216 | if {[llength $slot_range] <= 3} { | ||
| 217 | lappend normalized $slot_range | ||
| 218 | } else { | ||
| 219 | # Sort replicas (index 3+) by node ID, keep start/end/master unchanged | ||
| 220 | set replicas [lrange $slot_range 3 end] | ||
| 221 | set sorted_replicas [lsort -index 2 $replicas] | ||
| 222 | lappend normalized [concat [lrange $slot_range 0 2] $sorted_replicas] | ||
| 223 | } | ||
| 224 | } | ||
| 225 | return $normalized | ||
| 226 | } | ||
| 227 | |||
| 228 | # Check if cluster configuration is consistent. | ||
| 229 | proc cluster_config_consistent {} { | ||
| 230 | for {set j 0} {$j < $::cluster_master_nodes + $::cluster_replica_nodes} {incr j} { | ||
| 231 | if {$j == 0} { | ||
| 232 | set base_cfg [R $j cluster slots] | ||
| 233 | set base_secret [R $j debug internal_secret] | ||
| 234 | set normalized_base_cfg [normalize_cluster_slots $base_cfg] | ||
| 235 | } else { | ||
| 236 | set cfg [R $j cluster slots] | ||
| 237 | set secret [R $j debug internal_secret] | ||
| 238 | set normalized_cfg [normalize_cluster_slots $cfg] | ||
| 239 | if {$normalized_cfg != $normalized_base_cfg || $secret != $base_secret} { | ||
| 240 | return 0 | ||
| 241 | } | ||
| 242 | } | ||
| 243 | } | ||
| 244 | |||
| 245 | return 1 | ||
| 246 | } | ||
| 247 | |||
| 248 | # Wait for cluster configuration to propagate and be consistent across nodes. | ||
| 249 | proc wait_for_cluster_propagation {} { | ||
| 250 | wait_for_condition 50 100 { | ||
| 251 | [cluster_config_consistent] eq 1 | ||
| 252 | } else { | ||
| 253 | fail "cluster config did not reach a consistent state" | ||
| 254 | } | ||
| 255 | } | ||
| 256 | |||
| 257 | # Check if cluster's view of hostnames is consistent | ||
| 258 | proc are_hostnames_propagated {match_string} { | ||
| 259 | for {set j 0} {$j < $::cluster_master_nodes + $::cluster_replica_nodes} {incr j} { | ||
| 260 | set cfg [R $j cluster slots] | ||
| 261 | foreach node $cfg { | ||
| 262 | for {set i 2} {$i < [llength $node]} {incr i} { | ||
| 263 | if {! [string match $match_string [lindex [lindex [lindex $node $i] 3] 1]] } { | ||
| 264 | return 0 | ||
| 265 | } | ||
| 266 | } | ||
| 267 | } | ||
| 268 | } | ||
| 269 | return 1 | ||
| 270 | } | ||
