Add Redis source code for testing

1 files changed, 904 insertions, 0 deletions

diff --git a/examples/redis-unstable/tests/integration/replication-rdbchannel.tcl b/examples/redis-unstable/tests/integration/replication-rdbchannel.tcl
new file mode 100644
index 0000000..0fc955b
--- /dev/null
+++ b/examples/redis-unstable/tests/integration/replication-rdbchannel.tcl
@@ -0,0 +1,904 @@
+#
+# Copyright (c) 2009-Present, Redis Ltd.
+# All rights reserved.
+#
+# Copyright (c) 2024-present, Valkey contributors.
+# All rights reserved.
+#
+# Licensed under your choice of (a) the Redis Source Available License 2.0
+# (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the
+# GNU Affero General Public License v3 (AGPLv3).
+#
+# Portions of this file are available under BSD3 terms; see REDISCONTRIBUTIONS for more information.
+#
+
+# Returns either main or rdbchannel client id
+# Assumes there is one replica with two channels
+proc get_replica_client_id {master rdbchannel} {
+    set input [$master client list type replica]
+
+    foreach line [split $input "\n"] {
+        if {[regexp {id=(\d+).*flags=(\S+)} $line match id flags]} {
+            if {$rdbchannel == "yes"} {
+                # rdbchannel will have C flag
+                if {[string match *C* $flags]} {
+                    return $id
+                }
+            } else {
+                return $id
+            }
+        }
+    }
+
+    error "Replica not found"
+}
+
+start_server {tags {"repl external:skip"}} {
+    set replica1 [srv 0 client]
+
+    start_server {} {
+        set replica2 [srv 0 client]
+
+        start_server {} {
+            set master [srv 0 client]
+            set master_host [srv 0 host]
+            set master_port [srv 0 port]
+
+            $master config set repl-diskless-sync yes
+            $master config set repl-rdb-channel yes
+            populate 1000 master 10
+
+            test "Test replication with multiple replicas (rdbchannel enabled on both)" {
+                $replica1 config set repl-rdb-channel yes
+                $replica1 replicaof $master_host $master_port
+
+                $replica2 config set repl-rdb-channel yes
+                $replica2 replicaof $master_host $master_port
+
+                wait_replica_online $master 0
+                wait_replica_online $master 1
+
+                $master set x 1
+
+                # Wait until replicas catch master
+                wait_for_ofs_sync $master $replica1
+                wait_for_ofs_sync $master $replica2
+
+                # Verify db's are identical
+                assert_morethan [$master dbsize] 0
+                assert_equal [$master get x] 1
+                assert_equal [$master debug digest] [$replica1 debug digest]
+                assert_equal [$master debug digest] [$replica2 debug digest]
+            }
+
+            test "Test replication with multiple replicas (rdbchannel enabled on one of them)" {
+                # Allow both replicas to ask for sync
+                $master config set repl-diskless-sync-delay 5
+
+                $replica1 replicaof no one
+                $replica2 replicaof no one
+                $replica1 config set repl-rdb-channel yes
+                $replica2 config set repl-rdb-channel no
+
+                set loglines [count_log_lines 0]
+                set prev_forks [s 0 total_forks]
+                $master set x 2
+
+                # There will be two forks subsequently, one for rdbchannel
+                # replica another for the replica without rdbchannel config.
+                $replica1 replicaof $master_host $master_port
+                $replica2 replicaof $master_host $master_port
+
+                # There will be two forks subsequently, one for rdbchannel
+                # replica, another for the replica without rdbchannel config.
+                wait_for_log_messages 0 {"*Starting BGSAVE* replicas sockets (rdb-channel)*"} $loglines 300 100
+                wait_for_log_messages 0 {"*Starting BGSAVE* replicas sockets"} $loglines 300 100
+
+                wait_replica_online $master 0 100 100
+                wait_replica_online $master 1 100 100
+
+                # Verify two new forks.
+                assert_equal [s 0 total_forks] [expr $prev_forks + 2]
+
+                wait_for_ofs_sync $master $replica1
+                wait_for_ofs_sync $master $replica2
+
+                # Verify db's are identical
+                assert_equal [$replica1 get x] 2
+                assert_equal [$replica2 get x] 2
+                assert_equal [$master debug digest] [$replica1 debug digest]
+                assert_equal [$master debug digest] [$replica2 debug digest]
+            }
+
+            test "Test rdbchannel is not used if repl-diskless-sync config is disabled on master" {
+                $replica1 replicaof no one
+                $replica2 replicaof no one
+
+                $master config set repl-diskless-sync-delay 0
+                $master config set repl-diskless-sync no
+
+                $master set x 3
+                $replica1 replicaof $master_host $master_port
+
+                # Verify log message does not mention rdbchannel
+                wait_for_log_messages 0 {"*Starting BGSAVE for SYNC with target: disk*"} 0 2000 1
+
+                wait_replica_online $master 0
+                wait_for_ofs_sync $master $replica1
+
+                # Verify db's are identical
+                assert_equal [$replica1 get x] 3
+                assert_equal [$master debug digest] [$replica1 debug digest]
+            }
+        }
+    }
+}
+
+start_server {tags {"repl external:skip"}} {
+    set replica [srv 0 client]
+    set replica_pid [srv 0 pid]
+
+    start_server {} {
+        set master [srv 0 client]
+        set master_host [srv 0 host]
+        set master_port [srv 0 port]
+
+        $master config set repl-rdb-channel yes
+        $replica config set repl-rdb-channel yes
+
+        # Reuse this test to verify large key delivery
+        $master config set rdbcompression no
+        $master config set rdb-key-save-delay 3000
+        populate 1000 prefix1 10
+        populate 5 prefix2 3000000
+        populate 5 prefix3 2000000
+        populate 5 prefix4 1000000
+
+        # On master info output, we should see state transition in this order:
+        # 1. wait_bgsave: Replica receives psync error (+RDBCHANNELSYNC)
+        # 2. send_bulk_and_stream: Replica opens rdbchannel and delivery started
+        # 3. online: Sync is completed
+        test "Test replica state should start with wait_bgsave" {
+            $replica config set key-load-delay 100000
+            # Pause replica before opening rdb channel conn
+            $replica debug repl-pause before-rdb-channel
+            $replica replicaof $master_host $master_port
+
+            wait_for_condition 50 200 {
+                [s 0 connected_slaves] == 1 &&
+                [string match "*wait_bgsave*" [s 0 slave0]]
+            } else {
+                fail "replica failed"
+            }
+        }
+
+        test "Test replica state advances to send_bulk_and_stream when rdbchannel connects" {
+            $master set x 1
+            resume_process $replica_pid
+
+            wait_for_condition 50 200 {
+                [s 0 connected_slaves] == 1 &&
+                [s 0 rdb_bgsave_in_progress] == 1 &&
+                [string match "*send_bulk_and_stream*" [s 0 slave0]]
+            } else {
+                fail "replica failed"
+            }
+        }
+
+        test "Test replica rdbchannel client has SC flag on client list output" {
+            set input [$master client list type replica]
+
+            # There will two replicas, second one should be rdbchannel
+            set trimmed_input [string trimright $input]
+            set lines [split $trimmed_input "\n"]
+            if {[llength $lines] < 2} {
+                error "There is no second line in the input: $input"
+            }
+            set second_line [lindex $lines 1]
+
+            # Check if 'flags=SC' exists in the second line
+            if {![regexp {flags=SC} $second_line]} {
+                error "Flags are not 'SC' in the second line: $second_line"
+            }
+        }
+
+        test "Test replica state advances to online when fullsync is completed" {
+            # Speed up loading
+            $replica config set key-load-delay 0
+
+            wait_replica_online $master 0 100 1000
+            wait_for_ofs_sync $master $replica
+
+            wait_for_condition 50 200 {
+                [s 0 rdb_bgsave_in_progress] == 0 &&
+                [s 0 connected_slaves] == 1 &&
+                [string match "*online*" [s 0 slave0]]
+            } else {
+                fail "replica failed"
+            }
+
+            wait_replica_online $master 0 100 1000
+            wait_for_ofs_sync $master $replica
+
+            # Verify db's are identical
+            assert_morethan [$master dbsize] 0
+            assert_equal [$master debug digest] [$replica debug digest]
+        }
+    }
+}
+
+start_server {tags {"repl external:skip debug_defrag:skip"}} {
+    set replica [srv 0 client]
+
+    start_server {} {
+        set master [srv 0 client]
+        set master_host [srv 0 host]
+        set master_port [srv 0 port]
+
+        $master config set repl-rdb-channel yes
+        $replica config set repl-rdb-channel yes
+
+        test "Test master memory does not increase during replication" {
+            # Put some delay to rdb generation. If master doesn't forward
+            # incoming traffic to replica, master's replication buffer will grow
+            $master config set repl-diskless-sync-delay 0
+            $master config set rdb-key-save-delay 500 ;# 500us delay and 10k keys means at least 5 seconds replication
+            $master config set repl-backlog-size 5mb
+            $replica config set replica-full-sync-buffer-limit 200mb
+            populate 10000 master 10000 ;# 10k keys of 10k, means 100mb
+            $replica config set loading-process-events-interval-bytes 262144 ;# process events every 256kb of rdb or command stream
+
+            # Start write traffic
+            set load_handle [start_write_load $master_host $master_port 100 "key1" 5000 4]
+
+            set prev_used [s 0 used_memory]
+
+            $replica replicaof $master_host $master_port
+            set backlog_size [lindex [$master config get repl-backlog-size] 1]
+
+            # Verify used_memory stays low
+            set max_retry 1000
+            set peak_replica_buf_size 0
+            set peak_master_slave_buf_size 0
+            set peak_master_used_mem 0
+            set peak_master_rpl_buf 0
+            while {$max_retry} {
+                set replica_buf_size [s -1 replica_full_sync_buffer_size]
+                set master_slave_buf_size [s mem_clients_slaves]
+                set master_used_mem [s used_memory]
+                set master_rpl_buf [s mem_total_replication_buffers]
+                if {$replica_buf_size > $peak_replica_buf_size} {set peak_replica_buf_size $replica_buf_size}
+                if {$master_slave_buf_size > $peak_master_slave_buf_size} {set peak_master_slave_buf_size $master_slave_buf_size}
+                if {$master_used_mem > $peak_master_used_mem} {set peak_master_used_mem $master_used_mem}
+                if {$master_rpl_buf > $peak_master_rpl_buf} {set peak_master_rpl_buf $master_rpl_buf}
+                if {$::verbose} {
+                    puts "[clock format [clock seconds] -format %H:%M:%S] master: $master_slave_buf_size replica: $replica_buf_size"
+                }
+
+                # Wait for the replica to finish reading the rdb (also from the master's perspective), and also consume much of the replica buffer
+                if {[string match *slave0*state=online* [$master info]] &&
+                    [s -1 master_link_status] == "up" &&
+                    $replica_buf_size < 1000000} {
+                    break
+                } else {
+                    incr max_retry -1
+                    after 10
+                }
+            }
+            if {$max_retry == 0} {
+                error "assertion:Replica not in sync after 10 seconds"
+            }
+
+            if {$::verbose} {
+                puts "peak_master_used_mem $peak_master_used_mem"
+                puts "peak_master_rpl_buf $peak_master_rpl_buf"
+                puts "peak_master_slave_buf_size $peak_master_slave_buf_size"
+                puts "peak_replica_buf_size $peak_replica_buf_size"
+            }
+            # memory on the master is less than 1mb
+            assert_lessthan [expr $peak_master_used_mem - $prev_used - $backlog_size] 1000000
+            assert_lessthan $peak_master_rpl_buf [expr {$backlog_size + 1000000}]
+            assert_lessthan $peak_master_slave_buf_size 1000000
+            # buffers in the replica are more than 5mb
+            assert_morethan $peak_replica_buf_size 5000000
+
+            stop_write_load $load_handle
+        }
+    }
+}
+
+start_server {tags {"repl external:skip"}} {
+    set replica [srv 0 client]
+
+    start_server {} {
+        set master [srv 0 client]
+        set master_host [srv 0 host]
+        set master_port [srv 0 port]
+
+        $master config set repl-rdb-channel yes
+        $replica config set repl-rdb-channel yes
+
+        test "Test replication stream buffer becomes full on replica" {
+            # For replication stream accumulation, replica inherits slave output
+            # buffer limit as the size limit. In this test, we create traffic to
+            # fill the buffer fully. Once the limit is reached, accumulation
+            # will stop. This is not a failure scenario though. From that point,
+            # further accumulation may occur on master side. Replication should
+            # be completed successfully.
+
+            # Create some artificial delay for rdb delivery and load. We'll
+            # generate some traffic to fill the replication buffer.
+            $master config set rdb-key-save-delay 1000
+            $replica config set key-load-delay 1000
+            $replica config set client-output-buffer-limit "replica 64kb 64kb 0"
+            populate 2000 master 1
+
+            set prev_sync_full [s 0 sync_full]
+            $replica replicaof $master_host $master_port
+
+            # Wait for replica to establish psync using main channel
+            wait_for_condition 500 1000 {
+                [string match "*state=send_bulk_and_stream*" [s 0 slave0]]
+            } else {
+                fail "replica didn't start sync"
+            }
+
+            # Create some traffic on replication stream
+            populate 100 master 100000
+
+            # Wait for replica's buffer limit reached
+            wait_for_log_messages -1 {"*Replication buffer limit has been reached*"} 0 1000 10
+
+            # Speed up loading
+            $replica config set key-load-delay 0
+
+            # Wait until sync is successful
+            wait_for_condition 200 200 {
+                [status $master master_repl_offset] eq [status $replica master_repl_offset] &&
+                [status $master master_repl_offset] eq [status $replica slave_repl_offset]
+            } else {
+                fail "replica offsets didn't match in time"
+            }
+
+            # Verify sync was not interrupted.
+            assert_equal [s 0 sync_full] [expr $prev_sync_full + 1]
+
+            # Verify db's are identical
+            assert_morethan [$master dbsize] 0
+            assert_equal [$master debug digest] [$replica debug digest]
+        }
+
+        test "Test replication stream buffer config replica-full-sync-buffer-limit" {
+            # By default, replica inherits client-output-buffer-limit of replica
+            # to limit accumulated repl data during rdbchannel sync.
+            # replica-full-sync-buffer-limit should override it if it is set.
+            $replica replicaof no one
+
+            # Create some artificial delay for rdb delivery and load. We'll
+            # generate some traffic to fill the replication buffer.
+            $master config set rdb-key-save-delay 1000
+            $replica config set key-load-delay 1000
+            $replica config set client-output-buffer-limit "replica 1024 1024 0"
+            $replica config set replica-full-sync-buffer-limit 20mb
+            populate 2000 master 1
+
+            $replica replicaof $master_host $master_port
+
+            # Wait until replication starts
+            wait_for_condition 500 1000 {
+                [string match "*state=send_bulk_and_stream*" [s 0 slave0]]
+            } else {
+                fail "replica didn't start sync"
+            }
+
+            # Create some traffic on replication stream
+            populate 100 master 100000
+
+            # Make sure config is used, we accumulated more than
+            # client-output-buffer-limit
+            assert_morethan [s -1 replica_full_sync_buffer_size] 1024
+        }
+    }
+}
+
+start_server {tags {"repl external:skip debug_defrag:skip"}} {
+    set master [srv 0 client]
+    set master_host [srv 0 host]
+    set master_port [srv 0 port]
+    set master_pid  [srv 0 pid]
+    set loglines [count_log_lines 0]
+
+    $master config set repl-diskless-sync yes
+    $master config set repl-rdb-channel yes
+    $master config set repl-backlog-size 1mb
+    $master config set client-output-buffer-limit "replica 100k 0 0"
+    $master config set repl-diskless-sync-delay 3
+
+    start_server {} {
+        set replica [srv 0 client]
+        set replica_pid [srv 0 pid]
+
+        $replica config set repl-rdb-channel yes
+        $replica config set repl-timeout 10
+        $replica config set key-load-delay 10000
+        $replica config set loading-process-events-interval-bytes 1024
+
+        test "Test master disconnects replica when output buffer limit is reached" {
+            populate 20000 master 100 -1
+
+            $replica replicaof $master_host $master_port
+            wait_for_condition 100 200 {
+                [s 0 loading] == 1
+            } else {
+                fail "Replica did not start loading"
+            }
+
+            # Generate replication traffic of ~20mb to disconnect the slave on obuf limit
+            populate 20 master 1000000 -1
+
+            wait_for_log_messages -1 {"*Client * closed * for overcoming of output buffer limits.*"} $loglines 1000 10
+            $replica config set key-load-delay 0
+
+            # Wait until replica loads RDB
+            wait_for_log_messages 0 {"*Done loading RDB*"} 0 1000 10
+        }
+
+        test "Test replication recovers after output buffer failures" {
+            # Verify system is operational
+            $master set x 1
+
+            # Wait until replica catches up
+            wait_replica_online $master 0 1000 100
+            wait_for_ofs_sync $master $replica
+
+            # Verify db's are identical
+            assert_morethan [$master dbsize] 0
+            assert_equal [$replica get x] 1
+            assert_equal [$master debug digest] [$replica debug digest]
+        }
+    }
+}
+
+start_server {tags {"repl external:skip"}} {
+    set master [srv 0 client]
+    set master_host [srv 0 host]
+    set master_port [srv 0 port]
+
+    $master config set repl-diskless-sync yes
+    $master config set repl-rdb-channel yes
+    $master config set rdb-key-save-delay 300
+    $master config set client-output-buffer-limit "replica 0 0 0"
+    $master config set repl-diskless-sync-delay 5
+
+    populate 10000 master 1
+
+    start_server {} {
+        set replica1 [srv 0 client]
+        $replica1 config set repl-rdb-channel yes
+
+        start_server {} {
+            set replica2 [srv 0 client]
+            $replica2 config set repl-rdb-channel yes
+
+            set load_handle [start_write_load $master_host $master_port 100 "key"]
+
+            test "Test master continues RDB delivery if not all replicas are dropped" {
+                $replica1 replicaof $master_host $master_port
+                $replica2 replicaof $master_host $master_port
+
+                wait_for_condition 50 200 {
+                    [s -2 rdb_bgsave_in_progress] == 1
+                } else {
+                    fail "Sync did not start"
+                }
+
+                # Verify replicas are connected
+                wait_for_condition 500 100 {
+                    [s -2 connected_slaves] == 2
+                } else {
+                    fail "Replicas didn't connect: [s -2 connected_slaves]"
+                }
+
+                # kill one of the replicas
+                catch {$replica1 shutdown nosave}
+
+                # Wait until replica completes full sync
+                # Verify there is no other full sync attempt
+                wait_for_condition 50 1000 {
+                    [s 0 master_link_status] == "up" &&
+                    [s -2 sync_full] == 2 &&
+                    [s -2 connected_slaves] == 1
+                } else {
+                    fail "Sync session did not continue
+                          master_link_status: [s 0 master_link_status]
+                          sync_full:[s -2 sync_full]
+                          connected_slaves: [s -2 connected_slaves]"
+                }
+
+                # Wait until replica catches up
+                wait_replica_online $master 0 200 100
+                wait_for_condition 200 100 {
+                    [s 0 mem_replica_full_sync_buffer] == 0
+                } else {
+                    fail "Replica did not consume buffer in time"
+                }
+            }
+
+            test "Test master aborts rdb delivery if all replicas are dropped" {
+                $replica2 replicaof no one
+
+                # Start replication
+                $replica2 replicaof $master_host $master_port
+
+                wait_for_condition 50 1000 {
+                    [s -2 rdb_bgsave_in_progress] == 1
+                } else {
+                    fail "Sync did not start"
+                }
+                set loglines [count_log_lines -2]
+
+                # kill replica
+                catch {$replica2 shutdown nosave}
+
+                # Verify master aborts rdb save
+                wait_for_condition 50 1000 {
+                    [s -2 rdb_bgsave_in_progress] == 0 &&
+                    [s -2 connected_slaves] == 0
+                } else {
+                    fail "Master should abort the sync
+                          rdb_bgsave_in_progress:[s -2 rdb_bgsave_in_progress]
+                          connected_slaves: [s -2 connected_slaves]"
+                }
+                wait_for_log_messages -2 {"*Background transfer error*"} $loglines 1000 50
+            }
+
+            stop_write_load $load_handle
+        }
+    }
+}
+
+start_server {tags {"repl external:skip"}} {
+    set master [srv 0 client]
+    set master_host [srv 0 host]
+    set master_port [srv 0 port]
+
+    $master config set repl-diskless-sync yes
+    $master config set repl-rdb-channel yes
+    $master config set rdb-key-save-delay 1000
+
+    populate 3000 prefix1 1
+    populate 100 prefix2 100000
+
+    start_server {} {
+        set replica [srv 0 client]
+        set replica_pid [srv 0 pid]
+
+        $replica config set repl-rdb-channel yes
+        $replica config set repl-timeout 10
+
+        set load_handle [start_write_load $master_host $master_port 100 "key"]
+
+        test "Test replica recovers when rdb channel connection is killed" {
+            $replica replicaof $master_host $master_port
+
+            # Wait for sync session to start
+            wait_for_condition 500 200 {
+                [string match "*state=send_bulk_and_stream*" [s -1 slave0]] &&
+                [s -1 rdb_bgsave_in_progress] eq 1
+            } else {
+                fail "replica didn't start sync session in time"
+            }
+
+            set loglines [count_log_lines -1]
+
+            # Kill rdb channel client
+            set id [get_replica_client_id $master yes]
+            $master client kill id $id
+
+            wait_for_log_messages -1 {"*Background transfer error*"} $loglines 1000 10
+
+            # Verify master rejects main-ch-client-id after connection is killed
+            assert_error {*Unrecognized*} {$master replconf main-ch-client-id $id}
+
+            # Replica should retry
+            wait_for_condition 500 200 {
+                [string match "*state=send_bulk_and_stream*" [s -1 slave0]] &&
+                [s -1 rdb_bgsave_in_progress] eq 1
+            } else {
+                fail "replica didn't retry after connection close"
+            }
+        }
+
+        test "Test replica recovers when main channel connection is killed" {
+            set loglines [count_log_lines -1]
+
+            # Kill main channel client
+            set id [get_replica_client_id $master yes]
+            $master client kill id $id
+
+            wait_for_log_messages -1 {"*Background transfer error*"} $loglines 1000 20
+
+            # Replica should retry
+            wait_for_condition 500 2000 {
+                [string match "*state=send_bulk_and_stream*" [s -1 slave0]] &&
+                [s -1 rdb_bgsave_in_progress] eq 1
+            } else {
+                fail "replica didn't retry after connection close"
+            }
+        }
+
+        stop_write_load $load_handle
+
+        test "Test replica recovers connection failures" {
+            # Wait until replica catches up
+            wait_replica_online $master 0 1000 100
+            wait_for_ofs_sync $master $replica
+
+            # Verify db's are identical
+            assert_morethan [$master dbsize] 0
+            assert_equal [$master debug digest] [$replica debug digest]
+        }
+    }
+}
+
+start_server {tags {"repl external:skip tsan:skip"}} {
+    set replica [srv 0 client]
+    set replica_pid  [srv 0 pid]
+
+    start_server {} {
+        set master [srv 0 client]
+        set master_host [srv 0 host]
+        set master_port [srv 0 port]
+
+        test "Test master connection drops while streaming repl buffer into the db" {
+            # Just after replica loads RDB, it will stream repl buffer into the
+            # db. During streaming, we kill the master connection. Replica
+            # will abort streaming and then try another psync with master.
+            $master config set rdb-key-save-delay 1000
+            $master config set repl-rdb-channel yes
+            $master config set repl-diskless-sync yes
+            $replica config set repl-rdb-channel yes
+            $replica config set loading-process-events-interval-bytes 1024
+
+            # Populate db and start write traffic
+            populate 2000 master 1000
+            set load_handle [start_write_load $master_host $master_port 100 "key1"]
+
+            # Replica will pause in the loop of repl buffer streaming
+            $replica debug repl-pause on-streaming-repl-buf
+            $replica replicaof $master_host $master_port
+
+            # Check if repl stream accumulation is started.
+            wait_for_condition 50 1000 {
+                [s -1 replica_full_sync_buffer_size] > 0
+            } else {
+                fail "repl stream accumulation not started"
+            }
+
+            # Wait until replica starts streaming repl buffer
+            wait_for_log_messages -1 {"*Starting to stream replication buffer*"} 0 2000 10
+            stop_write_load $load_handle
+            $master config set rdb-key-save-delay 0
+
+            # Kill master connection and resume the process
+            $replica deferred 1
+            $replica client kill type master
+            $replica debug repl-pause clear
+            resume_process $replica_pid
+            $replica read
+            $replica read
+            $replica deferred 0
+
+            wait_for_log_messages -1 {"*Master client was freed while streaming*"} 0 500 10
+
+            # Quick check for stats test coverage
+            assert_morethan_equal [s -1 replica_full_sync_buffer_peak] [s -1 replica_full_sync_buffer_size]
+
+            # Wait until replica recovers and verify db's are identical
+            wait_replica_online $master 0 1000 10
+            wait_for_ofs_sync $master $replica
+
+            assert_morethan [$master dbsize] 0
+            assert_equal [$master debug digest] [$replica debug digest]
+        }
+    }
+}
+
+start_server {tags {"repl external:skip"}} {
+    set replica [srv 0 client]
+    set replica_pid  [srv 0 pid]
+
+    start_server {} {
+        set master [srv 0 client]
+        set master_host [srv 0 host]
+        set master_port [srv 0 port]
+
+        test "Test main channel connection drops while loading rdb (disk based)" {
+            # While loading rdb, we kill main channel connection.
+            # We expect replica to complete loading RDB and then try psync
+            # with the master.
+            $master config set repl-rdb-channel yes
+            $replica config set repl-rdb-channel yes
+            $replica config set repl-diskless-load disabled
+            $replica config set key-load-delay 10000
+            $replica config set loading-process-events-interval-bytes 1024
+
+            # Populate db and start write traffic
+            populate 10000 master 100
+            $replica replicaof $master_host $master_port
+
+            # Wait until replica starts loading
+            wait_for_condition 50 200 {
+                [s -1 loading] == 1
+            } else {
+                fail "replica did not start loading"
+            }
+
+            # Kill replica connections
+            $master client kill type replica
+            $master set x 1
+
+            # At this point, we expect replica to complete loading RDB. Then,
+            # it will try psync with master.
+            wait_for_log_messages -1 {"*Aborting rdb channel sync while loading the RDB*"} 0 2000 10
+            wait_for_log_messages -1 {"*After loading RDB, replica will try psync with master*"} 0 2000 10
+
+            # Speed up loading
+            $replica config set key-load-delay 0
+
+            # Wait until replica becomes online
+            wait_replica_online $master 0 100 100
+
+            # Verify there is another successful psync and no other full sync
+            wait_for_condition 50 200 {
+                [s 0 sync_full] == 1 &&
+                [s 0 sync_partial_ok] == 1
+            } else {
+                fail "psync was not successful [s 0 sync_full] [s 0 sync_partial_ok]"
+            }
+
+            # Verify db's are identical after recovery
+            wait_for_ofs_sync $master $replica
+            assert_morethan [$master dbsize] 0
+            assert_equal [$master debug digest] [$replica debug digest]
+        }
+    }
+}
+
+start_server {tags {"repl external:skip"}} {
+    set replica [srv 0 client]
+    set replica_pid  [srv 0 pid]
+
+    start_server {} {
+        set master [srv 0 client]
+        set master_host [srv 0 host]
+        set master_port [srv 0 port]
+
+        test "Test main channel connection drops while loading rdb (diskless)" {
+            # While loading rdb, kill both main and rdbchannel connections.
+            # We expect replica to abort sync and later retry again.
+            $master config set repl-rdb-channel yes
+            $replica config set repl-rdb-channel yes
+            $replica config set repl-diskless-load swapdb
+            $replica config set key-load-delay 10000
+            $replica config set loading-process-events-interval-bytes 1024
+
+            # Populate db and start write traffic
+            populate 10000 master 100
+
+            $replica replicaof $master_host $master_port
+
+            # Wait until replica starts loading
+            wait_for_condition 50 200 {
+                [s -1 loading] == 1
+            } else {
+                fail "replica did not start loading"
+            }
+
+            # Kill replica connections
+            $master client kill type replica
+            $master set x 1
+
+            # At this point, we expect replica to abort loading RDB.
+            wait_for_log_messages -1 {"*Aborting rdb channel sync while loading the RDB*"} 0 2000 10
+            wait_for_log_messages -1 {"*Failed trying to load the MASTER synchronization DB from socket*"} 0 2000 10
+
+            # Speed up loading
+            $replica config set key-load-delay 0
+
+            # Wait until replica recovers and becomes online
+            wait_replica_online $master 0 100 100
+
+            # Verify replica attempts another full sync
+            wait_for_condition 50 200 {
+                [s 0 sync_full] == 2 &&
+                [s 0 sync_partial_ok] == 0
+            } else {
+                fail "sync was not successful [s 0 sync_full] [s 0 sync_partial_ok]"
+            }
+
+            # Verify db's are identical after recovery
+            wait_for_ofs_sync $master $replica
+            assert_morethan [$master dbsize] 0
+            assert_equal [$master debug digest] [$replica debug digest]
+        }
+    }
+}
+
+start_server {tags {"repl external:skip tsan:skip"}} {
+    set master2 [srv 0 client]
+    set master2_host [srv 0 host]
+    set master2_port [srv 0 port]
+    start_server {tags {"repl external:skip"}} {
+        set replica [srv 0 client]
+        set replica_pid  [srv 0 pid]
+
+        start_server {} {
+            set master [srv 0 client]
+            set master_host [srv 0 host]
+            set master_port [srv 0 port]
+
+            test "Test replicaof command while streaming repl buffer into the db" {
+                # After replica loads the RDB, it will stream repl buffer into
+                # the db. During streaming, replica receives command
+                # "replicaof newmaster". Replica will abort streaming and then
+                # should be able to connect to the new master.
+                $master config set rdb-key-save-delay 1000
+                $master config set repl-rdb-channel yes
+                $master config set repl-diskless-sync yes
+                $replica config set repl-rdb-channel yes
+                $replica config set loading-process-events-interval-bytes 1024
+
+                # Populate db and start write traffic
+                populate 2000 master 1000
+                set load_handle [start_write_load $master_host $master_port 100 "key1"]
+
+                # Replica will pause in the loop of repl buffer streaming
+                $replica debug repl-pause on-streaming-repl-buf
+                $replica replicaof $master_host $master_port
+
+                # Check if repl stream accumulation is started.
+                wait_for_condition 50 1000 {
+                    [s -1 replica_full_sync_buffer_size] > 0
+                } else {
+                    fail "repl stream accumulation not started"
+                }
+
+                # Wait until replica starts streaming repl buffer
+                wait_for_log_messages -1 {"*Starting to stream replication buffer*"} 0 2000 10
+                stop_write_load $load_handle
+                $master config set rdb-key-save-delay 0
+
+                # Populate the other master
+                populate 100 master2 100 -2
+
+                # Send "replicaof newmaster" command and resume the process
+                $replica deferred 1
+                $replica replicaof $master2_host $master2_port
+                $replica debug repl-pause clear
+                resume_process $replica_pid
+                $replica read
+                $replica read
+                $replica deferred 0
+
+                wait_for_log_messages -1 {"*Master client was freed while streaming*"} 0 500 10
+
+                # Wait until replica recovers and verify db's are identical
+                wait_replica_online $master2 0 1000 10
+                wait_for_ofs_sync $master2 $replica
+                assert_morethan [$master2 dbsize] 0
+                assert_equal [$master2 debug digest] [$replica debug digest]
+
+                # Try replication once more to be sure everything is okay.
+                $replica replicaof no one
+                $master2 set x 100
+
+                $replica replicaof $master2_host $master2_port
+                wait_replica_online $master2 0 1000 10
+                wait_for_ofs_sync $master2 $replica
+                assert_morethan [$master2 dbsize] 0
+                assert_equal [$master2 debug digest] [$replica debug digest]
+            }
+        }
+    }
+}