diff options
| author | Mitja Felicijan <mitja.felicijan@gmail.com> | 2026-01-21 22:52:54 +0100 |
|---|---|---|
| committer | Mitja Felicijan <mitja.felicijan@gmail.com> | 2026-01-21 22:52:54 +0100 |
| commit | dcacc00e3750300617ba6e16eb346713f91a783a (patch) | |
| tree | 38e2d4fb5ed9d119711d4295c6eda4b014af73fd /examples/redis-unstable/tests/integration/replication.tcl | |
| parent | 58dac10aeb8f5a041c46bddbeaf4c7966a99b998 (diff) | |
| download | crep-dcacc00e3750300617ba6e16eb346713f91a783a.tar.gz | |
Remove testing data
Diffstat (limited to 'examples/redis-unstable/tests/integration/replication.tcl')
| -rw-r--r-- | examples/redis-unstable/tests/integration/replication.tcl | 1834 |
1 files changed, 0 insertions, 1834 deletions
diff --git a/examples/redis-unstable/tests/integration/replication.tcl b/examples/redis-unstable/tests/integration/replication.tcl deleted file mode 100644 index 1805a3d..0000000 --- a/examples/redis-unstable/tests/integration/replication.tcl +++ /dev/null @@ -1,1834 +0,0 @@ -# -# Copyright (c) 2009-Present, Redis Ltd. -# All rights reserved. -# -# Copyright (c) 2024-present, Valkey contributors. -# All rights reserved. -# -# Licensed under your choice of (a) the Redis Source Available License 2.0 -# (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the -# GNU Affero General Public License v3 (AGPLv3). -# -# Portions of this file are available under BSD3 terms; see REDISCONTRIBUTIONS for more information. -# - -proc log_file_matches {log pattern} { - set fp [open $log r] - set content [read $fp] - close $fp - string match $pattern $content -} - -start_server {tags {"repl network external:skip"}} { - set slave [srv 0 client] - set slave_host [srv 0 host] - set slave_port [srv 0 port] - set slave_log [srv 0 stdout] - start_server {} { - set master [srv 0 client] - set master_host [srv 0 host] - set master_port [srv 0 port] - - # Configure the master in order to hang waiting for the BGSAVE - # operation, so that the slave remains in the handshake state. - $master config set repl-diskless-sync yes - $master config set repl-diskless-sync-delay 1000 - - # Start the replication process... - $slave slaveof $master_host $master_port - - test {Slave enters handshake} { - wait_for_condition 50 1000 { - [string match *handshake* [$slave role]] - } else { - fail "Replica does not enter handshake state" - } - } - - test {Slave enters wait_bgsave} { - # Wait until the rdbchannel is connected to prevent the following - # 'debug sleep' occurring during the rdbchannel handshake. - wait_for_condition 50 1000 { - [string match *state=wait_bgsave* [$master info replication]] && - [llength [split [string trim [$master client list type slave]] "\r\n"]] == 2 - } else { - fail "Replica does not enter wait_bgsave state" - } - } - - # Use a short replication timeout on the slave, so that if there - # are no bugs the timeout is triggered in a reasonable amount - # of time. - $slave config set repl-timeout 5 - - # But make the master unable to send - # the periodic newlines to refresh the connection. The slave - # should detect the timeout. - $master debug sleep 10 - - test {Slave is able to detect timeout during handshake} { - wait_for_condition 50 1000 { - [log_file_matches $slave_log "*Timeout connecting to the MASTER*"] - } else { - fail "Replica is not able to detect timeout" - } - } - } -} - -start_server {tags {"repl external:skip"}} { - set A [srv 0 client] - set A_host [srv 0 host] - set A_port [srv 0 port] - start_server {} { - set B [srv 0 client] - set B_host [srv 0 host] - set B_port [srv 0 port] - - test {Set instance A as slave of B} { - $A slaveof $B_host $B_port - wait_for_condition 50 100 { - [lindex [$A role] 0] eq {slave} && - [string match {*master_link_status:up*} [$A info replication]] - } else { - fail "Can't turn the instance into a replica" - } - } - - test {INCRBYFLOAT replication, should not remove expire} { - r set test 1 EX 100 - r incrbyfloat test 0.1 - wait_for_ofs_sync $A $B - assert_equal [$A debug digest] [$B debug digest] - } - - test {GETSET replication} { - $A config resetstat - $A config set loglevel debug - $B config set loglevel debug - r set test foo - assert_equal [r getset test bar] foo - wait_for_condition 500 10 { - [$A get test] eq "bar" - } else { - fail "getset wasn't propagated" - } - assert_equal [r set test vaz get] bar - wait_for_condition 500 10 { - [$A get test] eq "vaz" - } else { - fail "set get wasn't propagated" - } - assert_match {*calls=3,*} [cmdrstat set $A] - assert_match {} [cmdrstat getset $A] - } - - test {BRPOPLPUSH replication, when blocking against empty list} { - $A config resetstat - set rd [redis_deferring_client] - $rd brpoplpush a b 5 - wait_for_blocked_client - r lpush a foo - wait_for_ofs_sync $B $A - assert_equal [$A debug digest] [$B debug digest] - assert_match {*calls=1,*} [cmdrstat rpoplpush $A] - assert_match {} [cmdrstat lmove $A] - assert_equal [$rd read] {foo} - $rd close - } - - test {BRPOPLPUSH replication, list exists} { - $A config resetstat - r lpush c 1 - r lpush c 2 - r lpush c 3 - assert_equal [r brpoplpush c d 5] {1} - wait_for_ofs_sync $B $A - assert_equal [$A debug digest] [$B debug digest] - assert_match {*calls=1,*} [cmdrstat rpoplpush $A] - assert_match {} [cmdrstat lmove $A] - } - - foreach wherefrom {left right} { - foreach whereto {left right} { - test "BLMOVE ($wherefrom, $whereto) replication, when blocking against empty list" { - $A config resetstat - set rd [redis_deferring_client] - $rd blmove a b $wherefrom $whereto 5 - $rd flush - wait_for_blocked_client - r lpush a foo - wait_for_ofs_sync $B $A - assert_equal [$A debug digest] [$B debug digest] - assert_match {*calls=1,*} [cmdrstat lmove $A] - assert_match {} [cmdrstat rpoplpush $A] - assert_equal [$rd read] {foo} - $rd close - } - - test "BLMOVE ($wherefrom, $whereto) replication, list exists" { - $A config resetstat - r lpush c 1 - r lpush c 2 - r lpush c 3 - r blmove c d $wherefrom $whereto 5 - wait_for_ofs_sync $B $A - assert_equal [$A debug digest] [$B debug digest] - assert_match {*calls=1,*} [cmdrstat lmove $A] - assert_match {} [cmdrstat rpoplpush $A] - } - } - } - - test {BLPOP followed by role change, issue #2473} { - set rd [redis_deferring_client] - $rd blpop foo 0 ; # Block while B is a master - wait_for_blocked_client - - # Turn B into master of A - $A slaveof no one - $B slaveof $A_host $A_port - wait_for_condition 50 100 { - [lindex [$B role] 0] eq {slave} && - [string match {*master_link_status:up*} [$B info replication]] - } else { - fail "Can't turn the instance into a replica" - } - - # Push elements into the "foo" list of the new replica. - # If the client is still attached to the instance, we'll get - # a desync between the two instances. - $A rpush foo a b c - wait_for_ofs_sync $B $A - - wait_for_condition 50 100 { - [$A debug digest] eq [$B debug digest] && - [$A lrange foo 0 -1] eq {a b c} && - [$B lrange foo 0 -1] eq {a b c} - } else { - fail "Master and replica have different digest: [$A debug digest] VS [$B debug digest]" - } - assert_match {*calls=1,*,rejected_calls=0,failed_calls=1*} [cmdrstat blpop $B] - - assert_error {UNBLOCKED*} {$rd read} - $rd close - } - } -} - -start_server {tags {"repl external:skip"}} { - r set mykey foo - - start_server {} { - test {Second server should have role master at first} { - s role - } {master} - - test {SLAVEOF should start with link status "down"} { - r multi - r slaveof [srv -1 host] [srv -1 port] - r info replication - r exec - } {*master_link_status:down*} - - test {The role should immediately be changed to "replica"} { - s role - } {slave} - - wait_for_sync r - test {Sync should have transferred keys from master} { - r get mykey - } {foo} - - test {The link status should be up} { - s master_link_status - } {up} - - test {SET on the master should immediately propagate} { - r -1 set mykey bar - - wait_for_condition 500 100 { - [r 0 get mykey] eq {bar} - } else { - fail "SET on master did not propagated on replica" - } - } - - test {FLUSHDB / FLUSHALL should replicate} { - # we're attaching to a sub-replica, so we need to stop pings on the real master - r -1 config set repl-ping-replica-period 3600 - - set repl [attach_to_replication_stream] - - r -1 set key value - r -1 flushdb - - r -1 set key value2 - r -1 flushall - - wait_for_ofs_sync [srv 0 client] [srv -1 client] - assert_equal [r -1 dbsize] 0 - assert_equal [r 0 dbsize] 0 - - # DB is empty. - r -1 flushdb - r -1 flushdb - r -1 eval {redis.call("flushdb")} 0 - - # DBs are empty. - r -1 flushall - r -1 flushall - r -1 eval {redis.call("flushall")} 0 - - # add another command to check nothing else was propagated after the above - r -1 incr x - - # Assert that each FLUSHDB command is replicated even the DB is empty. - # Assert that each FLUSHALL command is replicated even the DBs are empty. - assert_replication_stream $repl { - {set key value} - {flushdb} - {set key value2} - {flushall} - {flushdb} - {flushdb} - {flushdb} - {flushall} - {flushall} - {flushall} - {incr x} - } - close_replication_stream $repl - } - - test {ROLE in master reports master with a slave} { - set res [r -1 role] - lassign $res role offset slaves - assert {$role eq {master}} - assert {$offset > 0} - assert {[llength $slaves] == 1} - lassign [lindex $slaves 0] master_host master_port slave_offset - assert {$slave_offset <= $offset} - } - - test {ROLE in slave reports slave in connected state} { - set res [r role] - lassign $res role master_host master_port slave_state slave_offset - assert {$role eq {slave}} - assert {$slave_state eq {connected}} - } - } -} - -foreach mdl {no yes} rdbchannel {no yes} { - foreach sdl {disabled swapdb flushdb} { - start_server {tags {"repl external:skip debug_defrag:skip"} overrides {save {}}} { - set master [srv 0 client] - $master config set repl-diskless-sync $mdl - $master config set repl-diskless-sync-delay 5 - $master config set repl-diskless-sync-max-replicas 3 - set master_host [srv 0 host] - set master_port [srv 0 port] - set slaves {} - start_server {overrides {save {}}} { - lappend slaves [srv 0 client] - start_server {overrides {save {}}} { - lappend slaves [srv 0 client] - start_server {overrides {save {}}} { - lappend slaves [srv 0 client] - test "Connect multiple replicas at the same time (issue #141), master diskless=$mdl, replica diskless=$sdl, rdbchannel=$rdbchannel" { - - $master config set repl-rdb-channel $rdbchannel - [lindex $slaves 0] config set repl-rdb-channel $rdbchannel - [lindex $slaves 1] config set repl-rdb-channel $rdbchannel - [lindex $slaves 2] config set repl-rdb-channel $rdbchannel - - # start load handles only inside the test, so that the test can be skipped - set load_handle0 [start_bg_complex_data $master_host $master_port 9 100000000] - set load_handle1 [start_bg_complex_data $master_host $master_port 11 100000000] - set load_handle2 [start_bg_complex_data $master_host $master_port 12 100000000] - set load_handle3 [start_write_load $master_host $master_port 8] - set load_handle4 [start_write_load $master_host $master_port 4] - after 5000 ;# wait for some data to accumulate so that we have RDB part for the fork - - # Send SLAVEOF commands to slaves - [lindex $slaves 0] config set repl-diskless-load $sdl - [lindex $slaves 1] config set repl-diskless-load $sdl - [lindex $slaves 2] config set repl-diskless-load $sdl - [lindex $slaves 0] slaveof $master_host $master_port - [lindex $slaves 1] slaveof $master_host $master_port - [lindex $slaves 2] slaveof $master_host $master_port - - # Wait for all the three slaves to reach the "online" - # state from the POV of the master. - set retry 500 - while {$retry} { - set info [r -3 info] - if {[string match {*slave0:*state=online*slave1:*state=online*slave2:*state=online*} $info]} { - break - } else { - incr retry -1 - after 100 - } - } - if {$retry == 0} { - error "assertion:Slaves not correctly synchronized" - } - - # Wait that slaves acknowledge they are online so - # we are sure that DBSIZE and DEBUG DIGEST will not - # fail because of timing issues. - wait_for_condition 500 100 { - [lindex [[lindex $slaves 0] role] 3] eq {connected} && - [lindex [[lindex $slaves 1] role] 3] eq {connected} && - [lindex [[lindex $slaves 2] role] 3] eq {connected} - } else { - fail "Slaves still not connected after some time" - } - - # Stop the write load - stop_bg_complex_data $load_handle0 - stop_bg_complex_data $load_handle1 - stop_bg_complex_data $load_handle2 - stop_write_load $load_handle3 - stop_write_load $load_handle4 - - # Make sure no more commands processed - wait_load_handlers_disconnected -3 - - wait_for_ofs_sync $master [lindex $slaves 0] - wait_for_ofs_sync $master [lindex $slaves 1] - wait_for_ofs_sync $master [lindex $slaves 2] - - # Check digests - set digest [$master debug digest] - set digest0 [[lindex $slaves 0] debug digest] - set digest1 [[lindex $slaves 1] debug digest] - set digest2 [[lindex $slaves 2] debug digest] - assert {$digest ne 0000000000000000000000000000000000000000} - assert {$digest eq $digest0} - assert {$digest eq $digest1} - assert {$digest eq $digest2} - } - } - } - } - } - } -} - -start_server {tags {"repl external:skip"} overrides {save {}}} { - set master [srv 0 client] - set master_host [srv 0 host] - set master_port [srv 0 port] - start_server {overrides {save {}}} { - test "Master stream is correctly processed while the replica has a script in -BUSY state" { - set load_handle0 [start_write_load $master_host $master_port 3] - set slave [srv 0 client] - $slave config set lua-time-limit 500 - $slave slaveof $master_host $master_port - - # Wait for the slave to be online - wait_for_condition 500 100 { - [lindex [$slave role] 3] eq {connected} - } else { - fail "Replica still not connected after some time" - } - - # Wait some time to make sure the master is sending data - # to the slave. - after 5000 - - # Stop the ability of the slave to process data by sendig - # a script that will put it in BUSY state. - $slave eval {for i=1,3000000000 do end} 0 - - # Wait some time again so that more master stream will - # be processed. - after 2000 - - # Stop the write load - stop_write_load $load_handle0 - - # number of keys - wait_for_condition 500 100 { - [$master debug digest] eq [$slave debug digest] - } else { - fail "Different datasets between replica and master" - } - } - } -} - -# Diskless load swapdb when NOT async_loading (different master replid) -foreach testType {Successful Aborted} rdbchannel {yes no} { - start_server {tags {"repl external:skip"}} { - set replica [srv 0 client] - set replica_host [srv 0 host] - set replica_port [srv 0 port] - set replica_log [srv 0 stdout] - start_server {} { - set master [srv 0 client] - set master_host [srv 0 host] - set master_port [srv 0 port] - - # Set master and replica to use diskless replication on swapdb mode - $master config set repl-diskless-sync yes - $master config set repl-diskless-sync-delay 0 - $master config set save "" - $master config set repl-rdb-channel $rdbchannel - $replica config set repl-diskless-load swapdb - $replica config set save "" - - # Put different data sets on the master and replica - # We need to put large keys on the master since the replica replies to info only once in 2mb - $replica debug populate 200 slave 10 - $master debug populate 1000 master 100000 - $master config set rdbcompression no - - # Set a key value on replica to check status on failure and after swapping db - $replica set mykey myvalue - - switch $testType { - "Aborted" { - # Set master with a slow rdb generation, so that we can easily intercept loading - # 10ms per key, with 1000 keys is 10 seconds - $master config set rdb-key-save-delay 10000 - - # Start the replication process - $replica replicaof $master_host $master_port - - test "Diskless load swapdb (different replid): replica enter loading rdbchannel=$rdbchannel" { - # Wait for the replica to start reading the rdb - wait_for_condition 100 100 { - [s -1 loading] eq 1 - } else { - fail "Replica didn't get into loading mode" - } - - assert_equal [s -1 async_loading] 0 - } - - # Make sure that next sync will not start immediately so that we can catch the replica in between syncs - $master config set repl-diskless-sync-delay 5 - - # Kill the replica connection on the master - set killed [$master client kill type replica] - - # Wait for loading to stop (fail) - wait_for_condition 100 100 { - [s -1 loading] eq 0 - } else { - fail "Replica didn't disconnect" - } - - test "Diskless load swapdb (different replid): old database is exposed after replication fails rdbchannel=$rdbchannel" { - # Ensure we see old values from replica - assert_equal [$replica get mykey] "myvalue" - - # Make sure amount of replica keys didn't change - assert_equal [$replica dbsize] 201 - } - - # Speed up shutdown - $master config set rdb-key-save-delay 0 - } - "Successful" { - # Start the replication process - $replica replicaof $master_host $master_port - - # Let replica finish sync with master - wait_for_condition 100 100 { - [s -1 master_link_status] eq "up" - } else { - fail "Master <-> Replica didn't finish sync" - } - - test {Diskless load swapdb (different replid): new database is exposed after swapping} { - # Ensure we don't see anymore the key that was stored only to replica and also that we don't get LOADING status - assert_equal [$replica GET mykey] "" - - # Make sure amount of keys matches master - assert_equal [$replica dbsize] 1000 - } - } - } - } - } -} - -# Diskless load swapdb when async_loading (matching master replid) -foreach testType {Successful Aborted} { - start_server {tags {"repl external:skip"}} { - set replica [srv 0 client] - set replica_host [srv 0 host] - set replica_port [srv 0 port] - set replica_log [srv 0 stdout] - start_server {} { - set master [srv 0 client] - set master_host [srv 0 host] - set master_port [srv 0 port] - - # Set master and replica to use diskless replication on swapdb mode - $master config set repl-diskless-sync yes - $master config set repl-diskless-sync-delay 0 - $master config set save "" - $replica config set repl-diskless-load swapdb - $replica config set save "" - - # Set replica writable so we can check that a key we manually added is served - # during replication and after failure, but disappears on success - $replica config set replica-read-only no - - # Initial sync to have matching replids between master and replica - $replica replicaof $master_host $master_port - - # Let replica finish initial sync with master - wait_for_condition 100 100 { - [s -1 master_link_status] eq "up" - } else { - fail "Master <-> Replica didn't finish sync" - } - - # Put different data sets on the master and replica - # We need to put large keys on the master since the replica replies to info only once in 2mb - $replica debug populate 2000 slave 10 - $master debug populate 2000 master 100000 - $master config set rdbcompression no - - # Set a key value on replica to check status during loading, on failure and after swapping db - $replica set mykey myvalue - - # Set a function value on replica to check status during loading, on failure and after swapping db - $replica function load {#!lua name=test - redis.register_function('test', function() return 'hello1' end) - } - - # Set a function value on master to check it reaches the replica when replication ends - $master function load {#!lua name=test - redis.register_function('test', function() return 'hello2' end) - } - - # Remember the sync_full stat before the client kill. - set sync_full [s 0 sync_full] - - if {$testType == "Aborted"} { - # Set master with a slow rdb generation, so that we can easily intercept loading - # 20ms per key, with 2000 keys is 40 seconds - $master config set rdb-key-save-delay 20000 - } - - # Force the replica to try another full sync (this time it will have matching master replid) - $master multi - $master client kill type replica - # Fill replication backlog with new content - $master config set repl-backlog-size 16384 - for {set keyid 0} {$keyid < 10} {incr keyid} { - $master set "$keyid string_$keyid" [string repeat A 16384] - } - $master exec - - # Wait for sync_full to get incremented from the previous value. - # After the client kill, make sure we do a reconnect, and do a FULL SYNC. - wait_for_condition 100 100 { - [s 0 sync_full] > $sync_full - } else { - fail "Master <-> Replica didn't start the full sync" - } - - switch $testType { - "Aborted" { - test {Diskless load swapdb (async_loading): replica enter async_loading} { - # Wait for the replica to start reading the rdb - wait_for_condition 100 100 { - [s -1 async_loading] eq 1 - } else { - fail "Replica didn't get into async_loading mode" - } - - assert_equal [s -1 loading] 0 - } - - test {Diskless load swapdb (async_loading): old database is exposed while async replication is in progress} { - # Ensure we still see old values while async_loading is in progress and also not LOADING status - assert_equal [$replica get mykey] "myvalue" - - # Ensure we still can call old function while async_loading is in progress - assert_equal [$replica fcall test 0] "hello1" - - # Make sure we're still async_loading to validate previous assertion - assert_equal [s -1 async_loading] 1 - - # Make sure amount of replica keys didn't change - assert_equal [$replica dbsize] 2001 - } - - test {Busy script during async loading} { - set rd_replica [redis_deferring_client -1] - $replica config set lua-time-limit 10 - $rd_replica eval {while true do end} 0 - after 200 - assert_error {BUSY*} {$replica ping} - $replica script kill - after 200 ; # Give some time to Lua to call the hook again... - assert_equal [$replica ping] "PONG" - $rd_replica close - } - - test {Blocked commands and configs during async-loading} { - assert_error {LOADING*} {$replica REPLICAOF no one} - } - - # Make sure that next sync will not start immediately so that we can catch the replica in between syncs - $master config set repl-diskless-sync-delay 5 - - # Kill the replica connection on the master - set killed [$master client kill type replica] - - # Wait for loading to stop (fail) - wait_for_condition 100 100 { - [s -1 async_loading] eq 0 - } else { - fail "Replica didn't disconnect" - } - - test {Diskless load swapdb (async_loading): old database is exposed after async replication fails} { - # Ensure we see old values from replica - assert_equal [$replica get mykey] "myvalue" - - # Ensure we still can call old function - assert_equal [$replica fcall test 0] "hello1" - - # Make sure amount of replica keys didn't change - assert_equal [$replica dbsize] 2001 - } - - # Speed up shutdown - $master config set rdb-key-save-delay 0 - } - "Successful" { - # Let replica finish sync with master - wait_for_condition 100 100 { - [s -1 master_link_status] eq "up" - } else { - fail "Master <-> Replica didn't finish sync" - } - - test {Diskless load swapdb (async_loading): new database is exposed after swapping} { - # Ensure we don't see anymore the key that was stored only to replica and also that we don't get LOADING status - assert_equal [$replica GET mykey] "" - - # Ensure we got the new function - assert_equal [$replica fcall test 0] "hello2" - - # Make sure amount of keys matches master - assert_equal [$replica dbsize] 2010 - } - } - } - } - } -} - -test {diskless loading short read} { - start_server {tags {"repl"} overrides {save ""}} { - set replica [srv 0 client] - set replica_host [srv 0 host] - set replica_port [srv 0 port] - start_server {overrides {save ""}} { - set master [srv 0 client] - set master_host [srv 0 host] - set master_port [srv 0 port] - - # Set master and replica to use diskless replication - $master config set repl-diskless-sync yes - $master config set rdbcompression no - $replica config set repl-diskless-load swapdb - $master config set hz 500 - $replica config set hz 500 - $master config set dynamic-hz no - $replica config set dynamic-hz no - # Try to fill the master with all types of data types / encodings - set start [clock clicks -milliseconds] - - # Set a function value to check short read handling on functions - r function load {#!lua name=test - redis.register_function('test', function() return 'hello1' end) - } - - set has_vector_sets [server_has_command vadd] - - for {set k 0} {$k < 3} {incr k} { - for {set i 0} {$i < 10} {incr i} { - r set "$k int_$i" [expr {int(rand()*10000)}] - r expire "$k int_$i" [expr {int(rand()*10000)}] - r set "$k string_$i" [string repeat A [expr {int(rand()*1000000)}]] - r hset "$k hash_small" [string repeat A [expr {int(rand()*10)}]] 0[string repeat A [expr {int(rand()*10)}]] - r hset "$k hash_large" [string repeat A [expr {int(rand()*10000)}]] [string repeat A [expr {int(rand()*1000000)}]] - r hsetex "$k hfe_small" EX [expr {int(rand()*100)}] FIELDS 1 [string repeat A [expr {int(rand()*10)}]] 0[string repeat A [expr {int(rand()*10)}]] - r hsetex "$k hfe_large" EX [expr {int(rand()*100)}] FIELDS 1 [string repeat A [expr {int(rand()*10000)}]] [string repeat A [expr {int(rand()*1000000)}]] - r sadd "$k set_small" [string repeat A [expr {int(rand()*10)}]] - r sadd "$k set_large" [string repeat A [expr {int(rand()*1000000)}]] - r zadd "$k zset_small" [expr {rand()}] [string repeat A [expr {int(rand()*10)}]] - r zadd "$k zset_large" [expr {rand()}] [string repeat A [expr {int(rand()*1000000)}]] - r lpush "$k list_small" [string repeat A [expr {int(rand()*10)}]] - r lpush "$k list_large" [string repeat A [expr {int(rand()*1000000)}]] - - if {$has_vector_sets} { - r vadd "$k vector_set" VALUES 3 [expr {rand()}] [expr {rand()}] [expr {rand()}] [string repeat A [expr {int(rand()*1000)}]] - } - - for {set j 0} {$j < 10} {incr j} { - r xadd "$k stream" * foo "asdf" bar "1234" - } - r xgroup create "$k stream" "mygroup_$i" 0 - r xreadgroup GROUP "mygroup_$i" Alice COUNT 1 STREAMS "$k stream" > - } - } - - if {$::verbose} { - set end [clock clicks -milliseconds] - set duration [expr $end - $start] - puts "filling took $duration ms (TODO: use pipeline)" - set start [clock clicks -milliseconds] - } - - # Start the replication process... - set loglines [count_log_lines -1] - $master config set repl-diskless-sync-delay 0 - $replica replicaof $master_host $master_port - - # kill the replication at various points - set attempts 100 - if {$::accurate} { set attempts 500 } - for {set i 0} {$i < $attempts} {incr i} { - # wait for the replica to start reading the rdb - # using the log file since the replica only responds to INFO once in 2mb - set res [wait_for_log_messages -1 {"*Loading DB in memory*"} $loglines 2000 1] - set loglines [lindex $res 1] - - # add some additional random sleep so that we kill the master on a different place each time - after [expr {int(rand()*50)}] - - # kill the replica connection on the master - set killed [$master client kill type replica] - - set res [wait_for_log_messages -1 {"*Internal error in RDB*" "*Finished with success*" "*Successful partial resynchronization*"} $loglines 500 10] - if {$::verbose} { puts $res } - set log_text [lindex $res 0] - set loglines [lindex $res 1] - if {![string match "*Internal error in RDB*" $log_text]} { - # force the replica to try another full sync - $master multi - $master client kill type replica - $master set asdf asdf - # fill replication backlog with new content - $master config set repl-backlog-size 16384 - for {set keyid 0} {$keyid < 10} {incr keyid} { - $master set "$keyid string_$keyid" [string repeat A 16384] - } - $master exec - } - - # wait for loading to stop (fail) - # After a loading successfully, next loop will enter `async_loading` - wait_for_condition 1000 1 { - [s -1 async_loading] eq 0 && - [s -1 loading] eq 0 - } else { - fail "Replica didn't disconnect" - } - } - if {$::verbose} { - set end [clock clicks -milliseconds] - set duration [expr $end - $start] - puts "test took $duration ms" - } - # enable fast shutdown - $master config set rdb-key-save-delay 0 - } - } -} {} {external:skip} - -# get current stime and utime metrics for a thread (since it's creation) -proc get_cpu_metrics { statfile } { - if { [ catch { - set fid [ open $statfile r ] - set data [ read $fid 1024 ] - ::close $fid - set data [ split $data ] - - ;## number of jiffies it has been scheduled... - set utime [ lindex $data 13 ] - set stime [ lindex $data 14 ] - } err ] } { - error "assertion:can't parse /proc: $err" - } - set mstime [clock milliseconds] - return [ list $mstime $utime $stime ] -} - -# compute %utime and %stime of a thread between two measurements -proc compute_cpu_usage {start end} { - set clock_ticks [exec getconf CLK_TCK] - # convert ms time to jiffies and calc delta - set dtime [ expr { ([lindex $end 0] - [lindex $start 0]) * double($clock_ticks) / 1000 } ] - set utime [ expr { [lindex $end 1] - [lindex $start 1] } ] - set stime [ expr { [lindex $end 2] - [lindex $start 2] } ] - set pucpu [ expr { ($utime / $dtime) * 100 } ] - set pscpu [ expr { ($stime / $dtime) * 100 } ] - return [ list $pucpu $pscpu ] -} - - -# test diskless rdb pipe with multiple replicas, which may drop half way -start_server {tags {"repl external:skip tsan:skip"} overrides {save ""}} { - set master [srv 0 client] - $master config set repl-diskless-sync yes - $master config set repl-diskless-sync-delay 5 - $master config set repl-diskless-sync-max-replicas 2 - set master_host [srv 0 host] - set master_port [srv 0 port] - set master_pid [srv 0 pid] - # put enough data in the db that the rdb file will be bigger than the socket buffers - # and since we'll have key-load-delay of 100, 20000 keys will take at least 2 seconds - # we also need the replica to process requests during transfer (which it does only once in 2mb) - $master debug populate 20000 test 10000 - $master config set rdbcompression no - $master config set repl-rdb-channel no - # If running on Linux, we also measure utime/stime to detect possible I/O handling issues - set os [catch {exec uname}] - set measure_time [expr {$os == "Linux"} ? 1 : 0] - foreach all_drop {no slow fast all timeout} { - test "diskless $all_drop replicas drop during rdb pipe" { - set replicas {} - set replicas_alive {} - # start one replica that will read the rdb fast, and one that will be slow - start_server {overrides {save ""}} { - lappend replicas [srv 0 client] - lappend replicas_alive [srv 0 client] - start_server {overrides {save ""}} { - lappend replicas [srv 0 client] - lappend replicas_alive [srv 0 client] - - # start replication - # it's enough for just one replica to be slow, and have it's write handler enabled - # so that the whole rdb generation process is bound to that - set loglines [count_log_lines -2] - [lindex $replicas 0] config set repl-diskless-load swapdb - [lindex $replicas 1] config set repl-diskless-load swapdb - [lindex $replicas 0] config set key-load-delay 100 ;# 20k keys and 100 microseconds sleep means at least 2 seconds - [lindex $replicas 0] replicaof $master_host $master_port - [lindex $replicas 1] replicaof $master_host $master_port - - # wait for the replicas to start reading the rdb - # using the log file since the replica only responds to INFO once in 2mb - wait_for_log_messages -1 {"*Loading DB in memory*"} 0 1500 10 - - if {$measure_time} { - set master_statfile "/proc/$master_pid/stat" - set master_start_metrics [get_cpu_metrics $master_statfile] - set start_time [clock seconds] - } - - # wait a while so that the pipe socket writer will be - # blocked on write (since replica 0 is slow to read from the socket) - after 500 - - # add some command to be present in the command stream after the rdb. - $master incr $all_drop - - # disconnect replicas depending on the current test - if {$all_drop == "all" || $all_drop == "fast"} { - exec kill [srv 0 pid] - set replicas_alive [lreplace $replicas_alive 1 1] - } - if {$all_drop == "all" || $all_drop == "slow"} { - exec kill [srv -1 pid] - set replicas_alive [lreplace $replicas_alive 0 0] - } - if {$all_drop == "timeout"} { - $master config set repl-timeout 2 - # we want the slow replica to hang on a key for very long so it'll reach repl-timeout - pause_process [srv -1 pid] - after 2000 - } - - # wait for rdb child to exit - wait_for_condition 500 100 { - [s -2 rdb_bgsave_in_progress] == 0 - } else { - fail "rdb child didn't terminate" - } - - # make sure we got what we were aiming for, by looking for the message in the log file - if {$all_drop == "all"} { - wait_for_log_messages -2 {"*Diskless rdb transfer, last replica dropped, killing fork child*"} $loglines 1 1 - } - if {$all_drop == "no"} { - wait_for_log_messages -2 {"*Diskless rdb transfer, done reading from pipe, 2 replicas still up*"} $loglines 1 1 - } - if {$all_drop == "slow" || $all_drop == "fast"} { - wait_for_log_messages -2 {"*Diskless rdb transfer, done reading from pipe, 1 replicas still up*"} $loglines 1 1 - } - if {$all_drop == "timeout"} { - wait_for_log_messages -2 {"*Disconnecting timedout replica (full sync)*"} $loglines 1 1 - wait_for_log_messages -2 {"*Diskless rdb transfer, done reading from pipe, 1 replicas still up*"} $loglines 1 1 - # master disconnected the slow replica, remove from array - set replicas_alive [lreplace $replicas_alive 0 0] - # release it - resume_process [srv -1 pid] - } - - # make sure we don't have a busy loop going thought epoll_wait - if {$measure_time} { - set master_end_metrics [get_cpu_metrics $master_statfile] - set time_elapsed [expr {[clock seconds]-$start_time}] - set master_cpu [compute_cpu_usage $master_start_metrics $master_end_metrics] - set master_utime [lindex $master_cpu 0] - set master_stime [lindex $master_cpu 1] - if {$::verbose} { - puts "elapsed: $time_elapsed" - puts "master utime: $master_utime" - puts "master stime: $master_stime" - } - if {!$::no_latency && ($all_drop == "all" || $all_drop == "slow" || $all_drop == "timeout")} { - assert {$master_utime < 70} - assert {$master_stime < 70} - } - if {!$::no_latency && ($all_drop == "none" || $all_drop == "fast")} { - assert {$master_utime < 15} - assert {$master_stime < 15} - } - } - - # verify the data integrity - foreach replica $replicas_alive { - # Wait that replicas acknowledge they are online so - # we are sure that DBSIZE and DEBUG DIGEST will not - # fail because of timing issues. - wait_for_condition 150 100 { - [lindex [$replica role] 3] eq {connected} - } else { - fail "replicas still not connected after some time" - } - - # Make sure that replicas and master have same - # number of keys - wait_for_condition 50 100 { - [$master dbsize] == [$replica dbsize] - } else { - fail "Different number of keys between master and replicas after too long time." - } - - # Check digests - set digest [$master debug digest] - set digest0 [$replica debug digest] - assert {$digest ne 0000000000000000000000000000000000000000} - assert {$digest eq $digest0} - } - } - } - } - } -} - -test "diskless replication child being killed is collected" { - # when diskless master is waiting for the replica to become writable - # it removes the read event from the rdb pipe so if the child gets killed - # the replica will hung. and the master may not collect the pid with waitpid - start_server {tags {"repl"} overrides {save ""}} { - set master [srv 0 client] - set master_host [srv 0 host] - set master_port [srv 0 port] - set master_pid [srv 0 pid] - $master config set repl-diskless-sync yes - $master config set repl-diskless-sync-delay 0 - $master config set repl-rdb-channel no - # put enough data in the db that the rdb file will be bigger than the socket buffers - $master debug populate 20000 test 10000 - $master config set rdbcompression no - start_server {overrides {save ""}} { - set replica [srv 0 client] - set loglines [count_log_lines 0] - $replica config set repl-diskless-load swapdb - $replica config set key-load-delay 1000000 - $replica config set loading-process-events-interval-bytes 1024 - $replica replicaof $master_host $master_port - - # wait for the replicas to start reading the rdb - wait_for_log_messages 0 {"*Loading DB in memory*"} $loglines 1500 10 - - # wait to be sure the replica is hung and the master is blocked on write - after 500 - - # simulate the OOM killer or anyone else kills the child - set fork_child_pid [get_child_pid -1] - exec kill -9 $fork_child_pid - - # wait for the parent to notice the child have exited - wait_for_condition 50 100 { - [s -1 rdb_bgsave_in_progress] == 0 - } else { - fail "rdb child didn't terminate" - } - - # Speed up shutdown - $replica config set key-load-delay 0 - } - } -} {} {external:skip} - -foreach mdl {yes no} { - test "replication child dies when parent is killed - diskless: $mdl" { - # when master is killed, make sure the fork child can detect that and exit - start_server {tags {"repl"} overrides {save ""}} { - set master [srv 0 client] - set master_host [srv 0 host] - set master_port [srv 0 port] - set master_pid [srv 0 pid] - $master config set repl-diskless-sync $mdl - $master config set repl-diskless-sync-delay 0 - # create keys that will take 10 seconds to save - $master config set rdb-key-save-delay 1000 - $master debug populate 10000 - start_server {overrides {save ""}} { - set replica [srv 0 client] - $replica replicaof $master_host $master_port - - # wait for rdb child to start - wait_for_condition 5000 10 { - [s -1 rdb_bgsave_in_progress] == 1 - } else { - fail "rdb child didn't start" - } - set fork_child_pid [get_child_pid -1] - - # simulate the OOM killer or anyone else kills the parent - exec kill -9 $master_pid - - # wait for the child to notice the parent died have exited - wait_for_condition 500 10 { - [process_is_alive $fork_child_pid] == 0 - } else { - fail "rdb child didn't terminate" - } - } - } - } {} {external:skip} -} - -test "diskless replication read pipe cleanup" { - # In diskless replication, we create a read pipe for the RDB, between the child and the parent. - # When we close this pipe (fd), the read handler also needs to be removed from the event loop (if it still registered). - # Otherwise, next time we will use the same fd, the registration will be fail (panic), because - # we will use EPOLL_CTL_MOD (the fd still register in the event loop), on fd that already removed from epoll_ctl - start_server {tags {"repl"} overrides {save ""}} { - set master [srv 0 client] - set master_host [srv 0 host] - set master_port [srv 0 port] - set master_pid [srv 0 pid] - $master config set repl-diskless-sync yes - $master config set repl-diskless-sync-delay 0 - - # put enough data in the db, and slowdown the save, to keep the parent busy at the read process - $master config set rdb-key-save-delay 100000 - $master debug populate 20000 test 10000 - $master config set rdbcompression no - start_server {overrides {save ""}} { - set replica [srv 0 client] - set loglines [count_log_lines 0] - $replica config set repl-diskless-load swapdb - $replica replicaof $master_host $master_port - - # wait for the replicas to start reading the rdb - wait_for_log_messages 0 {"*Loading DB in memory*"} $loglines 1500 10 - - set loglines [count_log_lines -1] - # send FLUSHALL so the RDB child will be killed - $master flushall - - # wait for another RDB child process to be started - wait_for_log_messages -1 {"*Background RDB transfer started by pid*"} $loglines 800 10 - - # make sure master is alive - $master ping - } - } -} {} {external:skip tsan:skip} - -test {replicaof right after disconnection} { - # this is a rare race condition that was reproduced sporadically by the psync2 unit. - # see details in #7205 - start_server {tags {"repl"} overrides {save ""}} { - set replica1 [srv 0 client] - set replica1_host [srv 0 host] - set replica1_port [srv 0 port] - set replica1_log [srv 0 stdout] - start_server {overrides {save ""}} { - set replica2 [srv 0 client] - set replica2_host [srv 0 host] - set replica2_port [srv 0 port] - set replica2_log [srv 0 stdout] - start_server {overrides {save ""}} { - set master [srv 0 client] - set master_host [srv 0 host] - set master_port [srv 0 port] - $replica1 replicaof $master_host $master_port - $replica2 replicaof $master_host $master_port - - wait_for_condition 50 100 { - [string match {*master_link_status:up*} [$replica1 info replication]] && - [string match {*master_link_status:up*} [$replica2 info replication]] - } else { - fail "Can't turn the instance into a replica" - } - - set rd [redis_deferring_client -1] - $rd debug sleep 1 - after 100 - - # when replica2 will wake up from the sleep it will find both disconnection - # from it's master and also a replicaof command at the same event loop - $master client kill type replica - $replica2 replicaof $replica1_host $replica1_port - $rd read - - wait_for_condition 50 100 { - [string match {*master_link_status:up*} [$replica2 info replication]] - } else { - fail "role change failed." - } - - # make sure psync succeeded, and there were no unexpected full syncs. - assert_equal [status $master sync_full] 2 - assert_equal [status $replica1 sync_full] 0 - assert_equal [status $replica2 sync_full] 0 - } - } - } -} {} {external:skip} - -test {Kill rdb child process if its dumping RDB is not useful} { - start_server {tags {"repl"}} { - set slave1 [srv 0 client] - start_server {} { - set slave2 [srv 0 client] - start_server {} { - set master [srv 0 client] - set master_host [srv 0 host] - set master_port [srv 0 port] - for {set i 0} {$i < 10} {incr i} { - $master set $i $i - } - # Generating RDB will cost 10s(10 * 1s) - $master config set rdb-key-save-delay 1000000 - $master config set repl-diskless-sync no - $master config set save "" - - $slave1 slaveof $master_host $master_port - $slave2 slaveof $master_host $master_port - - # Wait for starting child - wait_for_condition 50 100 { - ([s 0 rdb_bgsave_in_progress] == 1) && - ([string match "*wait_bgsave*" [s 0 slave0]]) && - ([string match "*wait_bgsave*" [s 0 slave1]]) - } else { - fail "rdb child didn't start" - } - - # Slave1 disconnect with master - $slave1 slaveof no one - # Shouldn't kill child since another slave wait for rdb - after 100 - assert {[s 0 rdb_bgsave_in_progress] == 1} - - # Slave2 disconnect with master - $slave2 slaveof no one - # Should kill child - wait_for_condition 1000 10 { - [s 0 rdb_bgsave_in_progress] eq 0 - } else { - fail "can't kill rdb child" - } - - # If have save parameters, won't kill child - $master config set save "900 1" - $slave1 slaveof $master_host $master_port - $slave2 slaveof $master_host $master_port - wait_for_condition 50 100 { - ([s 0 rdb_bgsave_in_progress] == 1) && - ([string match "*wait_bgsave*" [s 0 slave0]]) && - ([string match "*wait_bgsave*" [s 0 slave1]]) - } else { - fail "rdb child didn't start" - } - $slave1 slaveof no one - $slave2 slaveof no one - after 200 - assert {[s 0 rdb_bgsave_in_progress] == 1} - catch {$master shutdown nosave} - } - } - } -} {} {external:skip} - -start_server {tags {"repl external:skip"}} { - set master1_host [srv 0 host] - set master1_port [srv 0 port] - r set a b - - start_server {} { - set master2 [srv 0 client] - set master2_host [srv 0 host] - set master2_port [srv 0 port] - # Take 10s for dumping RDB - $master2 debug populate 10 master2 10 - $master2 config set rdb-key-save-delay 1000000 - - start_server {} { - set sub_replica [srv 0 client] - - start_server {} { - # Full sync with master1 - r slaveof $master1_host $master1_port - wait_for_sync r - assert_equal "b" [r get a] - - # Let sub replicas sync with me - $sub_replica slaveof [srv 0 host] [srv 0 port] - wait_for_sync $sub_replica - assert_equal "b" [$sub_replica get a] - - # Full sync with master2, and then kill master2 before finishing dumping RDB - r slaveof $master2_host $master2_port - wait_for_condition 50 100 { - ([s -2 rdb_bgsave_in_progress] == 1) && - ([string match "*wait_bgsave*" [s -2 slave0]] || - [string match "*send_bulk_and_stream*" [s -2 slave0]]) - } else { - fail "full sync didn't start" - } - catch {$master2 shutdown nosave} - - test {Don't disconnect with replicas before loading transferred RDB when full sync} { - assert ![log_file_matches [srv -1 stdout] "*Connection with master lost*"] - # The replication id is not changed in entire replication chain - assert_equal [s master_replid] [s -3 master_replid] - assert_equal [s master_replid] [s -1 master_replid] - } - - test {Discard cache master before loading transferred RDB when full sync} { - set full_sync [s -3 sync_full] - set partial_sync [s -3 sync_partial_ok] - # Partial sync with master1 - r slaveof $master1_host $master1_port - wait_for_sync r - # master1 accepts partial sync instead of full sync - assert_equal $full_sync [s -3 sync_full] - assert_equal [expr $partial_sync+1] [s -3 sync_partial_ok] - - # Since master only partially sync replica, and repl id is not changed, - # the replica doesn't disconnect with its sub-replicas - assert_equal [s master_replid] [s -3 master_replid] - assert_equal [s master_replid] [s -1 master_replid] - assert ![log_file_matches [srv -1 stdout] "*Connection with master lost*"] - # Sub replica just has one full sync, no partial resync. - assert_equal 1 [s sync_full] - assert_equal 0 [s sync_partial_ok] - } - } - } - } -} - -test {replica can handle EINTR if use diskless load} { - start_server {tags {"repl"}} { - set replica [srv 0 client] - set replica_log [srv 0 stdout] - start_server {} { - set master [srv 0 client] - set master_host [srv 0 host] - set master_port [srv 0 port] - - $master debug populate 100 master 100000 - $master config set rdbcompression no - $master config set repl-diskless-sync yes - $master config set repl-diskless-sync-delay 0 - $replica config set repl-diskless-load on-empty-db - # Construct EINTR error by using the built in watchdog - $replica config set watchdog-period 200 - # Block replica in read() - $master config set rdb-key-save-delay 10000 - # set speedy shutdown - $master config set save "" - # Start the replication process... - $replica replicaof $master_host $master_port - - # Wait for the replica to start reading the rdb - set res [wait_for_log_messages -1 {"*Loading DB in memory*"} 0 200 10] - set loglines [lindex $res 1] - - # Wait till we see the watchgod log line AFTER the loading started - wait_for_log_messages -1 {"*WATCHDOG TIMER EXPIRED*"} $loglines 200 10 - - # Make sure we're still loading, and that there was just one full sync attempt - assert ![log_file_matches [srv -1 stdout] "*Reconnecting to MASTER*"] - assert_equal 1 [s 0 sync_full] - assert_equal 1 [s -1 loading] - } - } -} {} {external:skip} - -start_server {tags {"repl" "external:skip"}} { - test "replica do not write the reply to the replication link - SYNC (_addReplyToBufferOrList)" { - set rd [redis_deferring_client] - set lines [count_log_lines 0] - - $rd sync - $rd ping - catch {$rd read} e - if {$::verbose} { puts "SYNC _addReplyToBufferOrList: $e" } - assert_equal "PONG" [r ping] - - # Check we got the warning logs about the PING command. - verify_log_message 0 "*Replica generated a reply to command 'ping', disconnecting it: *" $lines - - $rd close - waitForBgsave r - } - - test "replica do not write the reply to the replication link - SYNC (addReplyDeferredLen)" { - set rd [redis_deferring_client] - set lines [count_log_lines 0] - - $rd sync - $rd xinfo help - catch {$rd read} e - if {$::verbose} { puts "SYNC addReplyDeferredLen: $e" } - assert_equal "PONG" [r ping] - - # Check we got the warning logs about the XINFO HELP command. - verify_log_message 0 "*Replica generated a reply to command 'xinfo|help', disconnecting it: *" $lines - - $rd close - waitForBgsave r - } - - test "replica do not write the reply to the replication link - PSYNC (_addReplyToBufferOrList)" { - set rd [redis_deferring_client] - set lines [count_log_lines 0] - - $rd psync replicationid -1 - assert_match {FULLRESYNC * 0} [$rd read] - $rd get foo - catch {$rd read} e - if {$::verbose} { puts "PSYNC _addReplyToBufferOrList: $e" } - assert_equal "PONG" [r ping] - - # Check we got the warning logs about the GET command. - verify_log_message 0 "*Replica generated a reply to command 'get', disconnecting it: *" $lines - verify_log_message 0 "*== CRITICAL == This master is sending an error to its replica: *" $lines - verify_log_message 0 "*Replica can't interact with the keyspace*" $lines - - $rd close - waitForBgsave r - } - - test "replica do not write the reply to the replication link - PSYNC (addReplyDeferredLen)" { - set rd [redis_deferring_client] - set lines [count_log_lines 0] - - $rd psync replicationid -1 - assert_match {FULLRESYNC * 0} [$rd read] - $rd slowlog get - catch {$rd read} e - if {$::verbose} { puts "PSYNC addReplyDeferredLen: $e" } - assert_equal "PONG" [r ping] - - # Check we got the warning logs about the SLOWLOG GET command. - verify_log_message 0 "*Replica generated a reply to command 'slowlog|get', disconnecting it: *" $lines - - $rd close - waitForBgsave r - } - - test "PSYNC with wrong offset should throw error" { - # It used to accept the FULL SYNC, but also replied with an error. - assert_error {ERR value is not an integer or out of range} {r psync replicationid offset_str} - set logs [exec tail -n 100 < [srv 0 stdout]] - assert_match {*Replica * asks for synchronization but with a wrong offset} $logs - assert_equal "PONG" [r ping] - } -} - -start_server {tags {"repl external:skip"}} { - set master [srv 0 client] - set master_host [srv 0 host] - set master_port [srv 0 port] - $master debug SET-ACTIVE-EXPIRE 0 - start_server {} { - set slave [srv 0 client] - $slave debug SET-ACTIVE-EXPIRE 0 - $slave slaveof $master_host $master_port - - test "Test replication with lazy expire" { - # wait for replication to be in sync - wait_for_condition 50 100 { - [lindex [$slave role] 0] eq {slave} && - [string match {*master_link_status:up*} [$slave info replication]] - } else { - fail "Can't turn the instance into a replica" - } - - $master sadd s foo - $master pexpire s 1 - after 10 - $master sadd s foo - assert_equal 1 [$master wait 1 0] - - assert_equal "set" [$master type s] - assert_equal "set" [$slave type s] - } - } -} - -foreach disklessload {disabled on-empty-db} { - test "Replica should reply LOADING while flushing a large db (disklessload: $disklessload)" { - start_server {} { - set replica [srv 0 client] - start_server {} { - set master [srv 0 client] - set master_host [srv 0 host] - set master_port [srv 0 port] - - $replica config set repl-diskless-load $disklessload - - # Populate replica with many keys, master with a few keys. - $replica debug populate 4000000 - populate 3 master 10 - - # Start the replication process... - $replica replicaof $master_host $master_port - - wait_for_condition 100 100 { - [s -1 loading] eq 1 - } else { - fail "Replica didn't get into loading mode" - } - - # If replica has a large db, it may take some time to discard it - # after receiving new db from the master. In this case, replica - # should reply -LOADING. Replica may reply -LOADING while - # loading the new db as well. To test the first case, populated - # replica with large amount of keys and master with a few keys. - # Discarding old db will take a long time and loading new one - # will be quick. So, if we receive -LOADING, most probably it is - # when flushing the db. - wait_for_condition 1 10000 { - [catch {$replica ping} err] && - [string match *LOADING* $err] - } else { - # There is a chance that we may not catch LOADING response - # if flushing db happens too fast compared to test execution - # Then, we may consider increasing key count or introducing - # artificial delay to db flush. - fail "Replica did not reply LOADING." - } - - catch {$replica shutdown nosave} - } - } - } {} {repl external:skip debug_defrag:skip} -} - -start_server {tags {"repl external:skip"} overrides {save {}}} { - set master [srv 0 client] - set master_host [srv 0 host] - set master_port [srv 0 port] - populate 10000 master 10 - - start_server {overrides {save {} rdb-del-sync-files yes loading-process-events-interval-bytes 1024}} { - test "Allow appendonly config change while loading rdb on slave" { - set replica [srv 0 client] - - # While loading rdb on slave, verify appendonly config changes are allowed - # 1- Change appendonly config from no to yes - $replica config set appendonly no - $replica config set key-load-delay 100 - $replica debug populate 1000 - - # Start the replication process... - $replica replicaof $master_host $master_port - - wait_for_condition 10 1000 { - [s loading] eq 1 - } else { - fail "Replica didn't get into loading mode" - } - - # Change config while replica is loading data - $replica config set appendonly yes - assert_equal 1 [s loading] - - # Speed up loading and verify aof is enabled - $replica config set key-load-delay 0 - wait_done_loading $replica - assert_equal 1 [s aof_enabled] - - # Quick sanity for AOF - $replica replicaof no one - set prev [s aof_current_size] - $replica set x 100 - assert_morethan [s aof_current_size] $prev - - # 2- While loading rdb, change appendonly from yes to no - $replica config set appendonly yes - $replica config set key-load-delay 100 - $replica flushall - - # Start the replication process... - $replica replicaof $master_host $master_port - - wait_for_condition 10 1000 { - [s loading] eq 1 - } else { - fail "Replica didn't get into loading mode" - } - - # Change config while replica is loading data - $replica config set appendonly no - assert_equal 1 [s loading] - - # Speed up loading and verify aof is disabled - $replica config set key-load-delay 0 - wait_done_loading $replica - assert_equal 0 [s 0 aof_enabled] - } - } -} - -start_server {tags {"repl external:skip"}} { - set replica [srv 0 client] - start_server {} { - set master [srv 0 client] - set master_host [srv 0 host] - set master_port [srv 0 port] - - test "Replica flushes db lazily when replica-lazy-flush enabled" { - $replica config set replica-lazy-flush yes - $replica debug populate 1000 - populate 1 master 10 - - # Start the replication process... - $replica replicaof $master_host $master_port - - wait_for_condition 100 100 { - [s -1 lazyfreed_objects] >= 1000 && - [s -1 master_link_status] eq {up} - } else { - fail "Replica did not free db lazily" - } - } - } -} - -start_server {tags {"repl external:skip"}} { - set replica [srv 0 client] - start_server {} { - set master [srv 0 client] - set master_host [srv 0 host] - set master_port [srv 0 port] - - test "Test replication with functions when repl-diskless-load is set to on-empty-db" { - $replica config set repl-diskless-load on-empty-db - - populate 10 master 10 - $master function load {#!lua name=test - redis.register_function{function_name='func1', callback=function() return 'hello' end, flags={'no-writes'}} - } - - $replica replicaof $master_host $master_port - - # Wait until replication is completed - wait_for_sync $replica - wait_for_ofs_sync $master $replica - - # Sanity check - assert_equal [$replica fcall func1 0] "hello" - assert_morethan [$replica dbsize] 0 - assert_equal [$master debug digest] [$replica debug digest] - } - } -} - -start_server {tags {"repl external:skip"}} { - set master [srv 0 client] - set master_host [srv 0 host] - set master_port [srv 0 port] - - start_server {} { - set slave [srv 0 client] - $slave slaveof $master_host $master_port - - test "Accumulate repl_total_disconnect_time with delayed reconnection" { - wait_for_condition 50 100 { - [string match {*master_link_status:up*} [$slave info replication]] - } else { - fail "Initial replica setup failed" - } - - # Simulate disconnect by pointing to invalid master - $slave slaveof $master_host 0 - after 1000 - - $slave slaveof $master_host $master_port - - wait_for_condition 50 100 { - [string match {*master_link_status:up*} [$slave info replication]] - } else { - fail "Initial replica setup failed" - } - assert {[status $slave total_disconnect_time_sec] >= 1} - } - - test "Test the total_disconnect_time_sec incr after slaveof no one" { - $slave slaveof no one - after 1000 - $slave slaveof $master_host $master_port - wait_for_condition 50 100 { - [lindex [$slave role] 0] eq {slave} && - [string match {*master_link_status:up*} [$slave info replication]] - } else { - fail "Can't turn the instance into a replica" - } - assert {[status $slave total_disconnect_time_sec] >= 2} - } - - test "Test correct replication disconnection time counters behavior" { - # Simulate disconnection - $slave slaveof $master_host 0 - - after 1000 - - set total_disconnect_time [status $slave total_disconnect_time_sec] - set link_down_since [status $slave master_link_down_since_seconds] - - # Restore real master - $slave slaveof $master_host $master_port - wait_for_condition 50 100 { - [string match {*master_link_status:up*} [$slave info replication]] - } else { - fail "Replication did not reconnect" - } - # total_disconnect_time and link_down_since incer - assert {$total_disconnect_time >= 3} - assert {$link_down_since > 0} - assert {$total_disconnect_time > $link_down_since} - - # total_disconnect_time_reconnect can be up to 5 seconds more than total_disconnect_time due to reconnection time - set total_disconnect_time_reconnect [status $slave total_disconnect_time_sec] - assert {$total_disconnect_time_reconnect >= $total_disconnect_time && $total_disconnect_time_reconnect <= $total_disconnect_time + 5} - } - } -} - -start_server {tags {"repl external:skip"}} { - set master [srv 0 client] - set master_host [srv 0 host] - set master_port [srv 0 port] - - start_server {} { - set slave [srv 0 client] - $slave slaveof $master_host $master_port - - # Test: Normal establishment of the master link - test "Test normal establishment process of the master link" { - wait_for_condition 50 100 { - [lindex [$slave role] 0] eq {slave} && - [string match {*master_link_status:up*} [$slave info replication]] - } else { - fail "Can't turn the instance into a replica" - } - - assert_equal 1 [status $slave master_current_sync_attempts] - assert_equal 1 [status $slave master_total_sync_attempts] - } - - # Test: Sync attempts reset after 'slaveof no one' - test "Test sync attempts reset after slaveof no one" { - $slave slaveof no one - $slave slaveof $master_host $master_port - - wait_for_condition 50 100 { - [lindex [$slave role] 0] eq {slave} && - [string match {*master_link_status:up*} [$slave info replication]] - } else { - fail "Can't turn the instance into a replica" - } - - assert_equal 1 [status $slave master_current_sync_attempts] - assert_equal 1 [status $slave master_total_sync_attempts] - } - - # Test: Sync attempts reset on master reconnect - test "Test sync attempts reset on master reconnect" { - $slave client kill type master - - wait_for_condition 50 100 { - [lindex [$slave role] 0] eq {slave} && - [string match {*master_link_status:up*} [$slave info replication]] - } else { - fail "Can't turn the instance into a replica" - } - - assert_equal 1 [status $slave master_current_sync_attempts] - assert_equal 2 [status $slave master_total_sync_attempts] - } - - # Test: Sync attempts reset on master switch - test "Test sync attempts reset on master switch" { - start_server {} { - set new_master_host [srv 0 host] - set new_master_port [srv 0 port] - $slave slaveof $new_master_host $new_master_port - - wait_for_condition 50 100 { - [lindex [$slave role] 0] eq {slave} && - [string match {*master_link_status:up*} [$slave info replication]] - } else { - fail "Can't turn the instance into a replica" - } - - assert_equal 1 [status $slave master_current_sync_attempts] - assert_equal 1 [status $slave master_total_sync_attempts] - } - } - - # Test: Replication current attempts counter behavior - test "Replication current attempts counter behavior" { - $slave slaveof $master_host $master_port - - # Wait until replica state becomes "connected" - wait_for_condition 1000 50 { - [lindex [$slave role] 0] eq {slave} && - [string match {*master_link_status:up*} [$slave info replication]] - } else { - fail "slave did not connect to master." - } - - assert_equal 1 [status $slave master_current_sync_attempts] - - # Connect to an invalid master - $slave slaveof $master_host 0 - - # Expect current sync attempts to increase - wait_for_condition 100 50 { - [status $slave master_current_sync_attempts] >= 2 - } else { - fail "Timeout waiting for master_current_sync_attempts" - } - } - } -} |
