Diff of the two buildlogs: -- --- b1/build.log 2024-01-08 00:29:45.675433524 +0000 +++ b2/build.log 2024-01-08 00:31:23.807212479 +0000 @@ -1,6 +1,6 @@ I: pbuilder: network access will be disabled during build -I: Current time: Sat Feb 8 18:51:37 -12 2025 -I: pbuilder-time-stamp: 1739083897 +I: Current time: Mon Jan 8 14:29:48 +14 2024 +I: pbuilder-time-stamp: 1704673788 I: Building the build Environment I: extracting base tarball [/var/cache/pbuilder/bookworm-reproducible-base.tgz] I: copying local configuration @@ -32,51 +32,83 @@ dpkg-source: info: applying t-skip--TestRaft_SnapshotRestore_PeerChange.patch I: using fakeroot in build. I: Installing the build-deps -I: user script /srv/workspace/pbuilder/849/tmp/hooks/D02_print_environment starting +I: user script /srv/workspace/pbuilder/15535/tmp/hooks/D01_modify_environment starting +debug: Running on ionos12-i386. +I: Changing host+domainname to test build reproducibility +I: Adding a custom variable just for the fun of it... +I: Changing /bin/sh to bash +'/bin/sh' -> '/bin/bash' +lrwxrwxrwx 1 root root 9 Jan 8 00:29 /bin/sh -> /bin/bash +I: Setting pbuilder2's login shell to /bin/bash +I: Setting pbuilder2's GECOS to second user,second room,second work-phone,second home-phone,second other +I: user script /srv/workspace/pbuilder/15535/tmp/hooks/D01_modify_environment finished +I: user script /srv/workspace/pbuilder/15535/tmp/hooks/D02_print_environment starting I: set - BUILDDIR='/build/reproducible-path' - BUILDUSERGECOS='first user,first room,first work-phone,first home-phone,first other' - BUILDUSERNAME='pbuilder1' - BUILD_ARCH='i386' - DEBIAN_FRONTEND='noninteractive' - DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all parallel=16 ' - DISTRIBUTION='bookworm' - HOME='/root' - HOST_ARCH='i386' + BASH=/bin/sh + BASHOPTS=checkwinsize:cmdhist:complete_fullquote:extquote:force_fignore:globasciiranges:globskipdots:hostcomplete:interactive_comments:patsub_replacement:progcomp:promptvars:sourcepath + BASH_ALIASES=() + BASH_ARGC=() + BASH_ARGV=() + BASH_CMDS=() + BASH_LINENO=([0]="12" [1]="0") + BASH_LOADABLES_PATH=/usr/local/lib/bash:/usr/lib/bash:/opt/local/lib/bash:/usr/pkg/lib/bash:/opt/pkg/lib/bash:. + BASH_SOURCE=([0]="/tmp/hooks/D02_print_environment" [1]="/tmp/hooks/D02_print_environment") + BASH_VERSINFO=([0]="5" [1]="2" [2]="15" [3]="1" [4]="release" [5]="i686-pc-linux-gnu") + BASH_VERSION='5.2.15(1)-release' + BUILDDIR=/build/reproducible-path + BUILDUSERGECOS='second user,second room,second work-phone,second home-phone,second other' + BUILDUSERNAME=pbuilder2 + BUILD_ARCH=i386 + DEBIAN_FRONTEND=noninteractive + DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all parallel=7 ' + DIRSTACK=() + DISTRIBUTION=bookworm + EUID=0 + FUNCNAME=([0]="Echo" [1]="main") + GROUPS=() + HOME=/root + HOSTNAME=i-capture-the-hostname + HOSTTYPE=i686 + HOST_ARCH=i386 IFS=' ' - INVOCATION_ID='bb79f038973c4466b89794a82c89bea8' - LANG='C' - LANGUAGE='en_US:en' - LC_ALL='C' - LD_LIBRARY_PATH='/usr/lib/libeatmydata' - LD_PRELOAD='libeatmydata.so' - MAIL='/var/mail/root' - OPTIND='1' - PATH='/usr/sbin:/usr/bin:/sbin:/bin:/usr/games' - PBCURRENTCOMMANDLINEOPERATION='build' - PBUILDER_OPERATION='build' - PBUILDER_PKGDATADIR='/usr/share/pbuilder' - PBUILDER_PKGLIBDIR='/usr/lib/pbuilder' - PBUILDER_SYSCONFDIR='/etc' - PPID='849' - PS1='# ' - PS2='> ' + INVOCATION_ID=e8464e7aae43441c8e8a943dc6b44349 + LANG=C + LANGUAGE=de_CH:de + LC_ALL=C + LD_LIBRARY_PATH=/usr/lib/libeatmydata + LD_PRELOAD=libeatmydata.so + MACHTYPE=i686-pc-linux-gnu + MAIL=/var/mail/root + OPTERR=1 + OPTIND=1 + OSTYPE=linux-gnu + PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path + PBCURRENTCOMMANDLINEOPERATION=build + PBUILDER_OPERATION=build + PBUILDER_PKGDATADIR=/usr/share/pbuilder + PBUILDER_PKGLIBDIR=/usr/lib/pbuilder + PBUILDER_SYSCONFDIR=/etc + PIPESTATUS=([0]="0") + POSIXLY_CORRECT=y + PPID=15535 PS4='+ ' - PWD='/' - SHELL='/bin/bash' - SHLVL='2' - SUDO_COMMAND='/usr/bin/timeout -k 18.1h 18h /usr/bin/ionice -c 3 /usr/bin/nice /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/r-b-build.F1G7eTIx/pbuilderrc_Ko24 --distribution bookworm --hookdir /etc/pbuilder/first-build-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/bookworm-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/r-b-build.F1G7eTIx/b1 --logfile b1/build.log golang-github-hashicorp-raft_1.1.2-1.dsc' - SUDO_GID='112' - SUDO_UID='107' - SUDO_USER='jenkins' - TERM='unknown' - TZ='/usr/share/zoneinfo/Etc/GMT+12' - USER='root' - _='/usr/bin/systemd-run' - http_proxy='http://85.184.249.68:3128' + PWD=/ + SHELL=/bin/bash + SHELLOPTS=braceexpand:errexit:hashall:interactive-comments:posix + SHLVL=3 + SUDO_COMMAND='/usr/bin/timeout -k 24.1h 24h /usr/bin/ionice -c 3 /usr/bin/nice -n 11 /usr/bin/unshare --uts -- /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/r-b-build.F1G7eTIx/pbuilderrc_L23S --distribution bookworm --hookdir /etc/pbuilder/rebuild-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/bookworm-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/r-b-build.F1G7eTIx/b2 --logfile b2/build.log golang-github-hashicorp-raft_1.1.2-1.dsc' + SUDO_GID=112 + SUDO_UID=107 + SUDO_USER=jenkins + TERM=unknown + TZ=/usr/share/zoneinfo/Etc/GMT-14 + UID=0 + USER=root + _='I: set' + http_proxy=http://78.137.99.97:3128 I: uname -a - Linux ionos6-i386 6.1.0-17-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.1.69-1 (2023-12-30) x86_64 GNU/Linux + Linux i-capture-the-hostname 6.1.0-17-686-pae #1 SMP PREEMPT_DYNAMIC Debian 6.1.69-1 (2023-12-30) i686 GNU/Linux I: ls -l /bin total 6036 -rwxr-xr-x 1 root root 1408088 Apr 23 2023 bash @@ -134,15 +166,15 @@ -rwxr-xr-x 1 root root 51080 Sep 20 2022 readlink -rwxr-xr-x 1 root root 75720 Sep 20 2022 rm -rwxr-xr-x 1 root root 51080 Sep 20 2022 rmdir - -rwxr-xr-x 1 root root 22308 Jul 28 2023 run-parts + -rwxr-xr-x 1 root root 22308 Jul 28 23:46 run-parts -rwxr-xr-x 1 root root 133224 Jan 5 2023 sed - lrwxrwxrwx 1 root root 4 Jan 5 2023 sh -> dash + lrwxrwxrwx 1 root root 9 Jan 8 00:29 sh -> /bin/bash -rwxr-xr-x 1 root root 38760 Sep 20 2022 sleep -rwxr-xr-x 1 root root 87976 Sep 20 2022 stty -rwsr-xr-x 1 root root 83492 Mar 23 2023 su -rwxr-xr-x 1 root root 38792 Sep 20 2022 sync -rwxr-xr-x 1 root root 598456 Apr 6 2023 tar - -rwxr-xr-x 1 root root 13860 Jul 28 2023 tempfile + -rwxr-xr-x 1 root root 13860 Jul 28 23:46 tempfile -rwxr-xr-x 1 root root 120776 Sep 20 2022 touch -rwxr-xr-x 1 root root 34664 Sep 20 2022 true -rwxr-xr-x 1 root root 17892 Mar 23 2023 ulockmgr_server @@ -162,7 +194,7 @@ -rwxr-xr-x 1 root root 2206 Apr 10 2022 zless -rwxr-xr-x 1 root root 1842 Apr 10 2022 zmore -rwxr-xr-x 1 root root 4577 Apr 10 2022 znew -I: user script /srv/workspace/pbuilder/849/tmp/hooks/D02_print_environment finished +I: user script /srv/workspace/pbuilder/15535/tmp/hooks/D02_print_environment finished -> Attempting to satisfy build-dependencies -> Creating pbuilder-satisfydepends-dummy package Package: pbuilder-satisfydepends-dummy @@ -306,7 +338,7 @@ Get: 96 http://deb.debian.org/debian bookworm/main i386 golang-gopkg-check.v1-dev all 0.0+git20200902.038fdea-1 [32.1 kB] Get: 97 http://deb.debian.org/debian bookworm/main i386 golang-gopkg-mgo.v2-dev all 2016.08.01-7 [316 kB] Get: 98 http://deb.debian.org/debian bookworm/main i386 golang-github-hashicorp-go-msgpack-dev all 0.5.5-1 [43.3 kB] -Fetched 117 MB in 1s (92.3 MB/s) +Fetched 117 MB in 5s (23.0 MB/s) debconf: delaying package configuration, since apt-utils is not installed Selecting previously unselected package sensible-utils. (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 18156 files and directories currently installed.) @@ -718,7 +750,11 @@ fakeroot is already the newest version (1.31-1.2). 0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded. I: Building the package -I: Running cd /build/reproducible-path/golang-github-hashicorp-raft-1.1.2/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-genchanges -S > ../golang-github-hashicorp-raft_1.1.2-1_source.changes +I: user script /srv/workspace/pbuilder/15535/tmp/hooks/A99_set_merged_usr starting +Not re-configuring usrmerge for bookworm +I: user script /srv/workspace/pbuilder/15535/tmp/hooks/A99_set_merged_usr finished +hostname: Name or service not known +I: Running cd /build/reproducible-path/golang-github-hashicorp-raft-1.1.2/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-genchanges -S > ../golang-github-hashicorp-raft_1.1.2-1_source.changes dpkg-buildpackage: info: source package golang-github-hashicorp-raft dpkg-buildpackage: info: source version 1.1.2-1 dpkg-buildpackage: info: source distribution unstable @@ -736,148 +772,148 @@ dh_autoreconf -O--buildsystem=golang dh_auto_configure -O--buildsystem=golang dh_auto_build -O--buildsystem=golang - cd obj-i686-linux-gnu && go install -trimpath -v -p 16 github.com/hashicorp/raft github.com/hashicorp/raft/bench -internal/goarch + cd obj-i686-linux-gnu && go install -trimpath -v -p 7 github.com/hashicorp/raft github.com/hashicorp/raft/bench internal/unsafeheader -unicode/utf8 -internal/goos -container/list +internal/goarch internal/goexperiment -internal/itoa +internal/race +internal/cpu +internal/goos runtime/internal/atomic -crypto/subtle runtime/internal/syscall -internal/cpu -internal/race +internal/abi +runtime/internal/math +sync/atomic +runtime/internal/sys +unicode/utf8 unicode -crypto/internal/subtle +container/list +internal/itoa math/bits -sync/atomic +crypto/internal/subtle +crypto/subtle crypto/internal/boring/sig encoding -internal/abi -runtime/internal/math +internal/bytealg unicode/utf16 -runtime/internal/sys internal/nettrace vendor/golang.org/x/crypto/cryptobyte/asn1 vendor/golang.org/x/crypto/internal/subtle math -internal/bytealg runtime internal/reflectlite sync internal/testlog -internal/sysinfo internal/singleflight +internal/sysinfo math/rand runtime/cgo errors sort internal/oserror io -github.com/hashicorp/golang-lru/simplelru internal/safefilepath -vendor/golang.org/x/net/dns/dnsmessage path strconv -crypto/internal/nistec/fiat syscall -hash -crypto/internal/randutil +github.com/hashicorp/golang-lru/simplelru +vendor/golang.org/x/net/dns/dnsmessage +crypto/internal/nistec/fiat bytes strings -hash/crc32 -hash/crc64 +hash +reflect +bufio crypto +crypto/internal/randutil +github.com/hashicorp/go-immutable-radix +hash/crc32 crypto/rc4 -reflect vendor/golang.org/x/text/transform -bufio net/http/internal/ascii -github.com/hashicorp/go-immutable-radix +time +internal/syscall/unix +internal/syscall/execenv regexp/syntax html -internal/syscall/execenv -internal/syscall/unix -time -regexp +hash/crc64 crypto/internal/nistec +regexp context io/fs internal/poll +os internal/fmtsort encoding/binary -os encoding/base64 crypto/cipher crypto/internal/edwards25519/field -vendor/golang.org/x/crypto/internal/poly1305 crypto/md5 +vendor/golang.org/x/crypto/internal/poly1305 vendor/golang.org/x/crypto/curve25519/internal/field -golang.org/x/sys/unix encoding/pem +golang.org/x/sys/unix crypto/internal/edwards25519 +fmt +internal/godebug crypto/internal/boring crypto/des -vendor/golang.org/x/crypto/chacha20 -crypto/hmac -crypto/sha512 +internal/intern +path/filepath crypto/aes +net/netip +crypto/sha512 +crypto/hmac crypto/sha1 crypto/sha256 -internal/godebug -path/filepath -fmt -os/signal +vendor/golang.org/x/crypto/chacha20 +vendor/golang.org/x/crypto/hkdf io/ioutil -internal/intern +os/signal vendor/golang.org/x/crypto/chacha20poly1305 -vendor/golang.org/x/crypto/hkdf -net/netip net +encoding/json log +math/big +compress/flate encoding/hex vendor/golang.org/x/crypto/curve25519 -net/http/internal net/url -encoding/json -compress/flate -mime -vendor/golang.org/x/net/http2/hpack -go/token -mime/quotedprintable -encoding/gob -math/big -vendor/golang.org/x/text/unicode/norm -text/template/parse -flag -runtime/debug vendor/golang.org/x/text/unicode/bidi -runtime/trace +vendor/golang.org/x/text/unicode/norm compress/gzip -testing +vendor/golang.org/x/net/http2/hpack +mime vendor/golang.org/x/text/secure/bidirule +mime/quotedprintable +net/http/internal github.com/mattn/go-isatty -text/template +encoding/gob +go/token +text/template/parse github.com/mattn/go-colorable -github.com/fatih/color -github.com/hashicorp/go-hclog -vendor/golang.org/x/net/idna -crypto/internal/boring/bbig -crypto/elliptic crypto/rand -crypto/dsa +crypto/elliptic +crypto/internal/boring/bbig encoding/asn1 -html/template crypto/ed25519 crypto/rsa -crypto/x509/pkix +crypto/dsa +vendor/golang.org/x/net/idna +github.com/fatih/color +flag +github.com/hashicorp/go-hclog vendor/golang.org/x/crypto/cryptobyte +crypto/x509/pkix +text/template +runtime/debug +runtime/trace +testing crypto/ecdsa +html/template vendor/golang.org/x/net/http/httpproxy -net/textproto crypto/x509 +net/textproto vendor/golang.org/x/net/http/httpguts mime/multipart crypto/tls @@ -889,7 +925,7 @@ github.com/hashicorp/raft github.com/hashicorp/raft/bench dh_auto_test -O--buildsystem=golang - cd obj-i686-linux-gnu && go test -vet=off -v -p 16 github.com/hashicorp/raft github.com/hashicorp/raft/bench + cd obj-i686-linux-gnu && go test -vet=off -v -p 7 github.com/hashicorp/raft github.com/hashicorp/raft/bench === RUN TestCommitment_setVoters --- PASS: TestCommitment_setVoters (0.00s) === RUN TestCommitment_match_max @@ -931,33 +967,33 @@ === RUN TestFileSnapshotSinkImpl --- PASS: TestFileSnapshotSinkImpl (0.00s) === RUN TestFileSS_CreateSnapshotMissingParentDir - testing.go:153: 2025-02-09T06:52:19.024Z [INFO] creating new snapshot: path=/tmp/raft2354081483/raft3054436695/snapshots/3-10-1739083939024.tmp + testing.go:153: 2024-01-08T00:30:53.958Z [INFO] creating new snapshot: path=/tmp/raft1326029728/raft553529258/snapshots/3-10-1704673853958.tmp --- PASS: TestFileSS_CreateSnapshotMissingParentDir (0.00s) === RUN TestFileSS_CreateSnapshot - testing.go:153: 2025-02-09T06:52:19.027Z [INFO] creating new snapshot: path=/tmp/raft3098021935/snapshots/3-10-1739083939027.tmp - testing.go:153: 2025-02-09T06:52:19.029Z [WARN] found temporary snapshot: name=3-10-1739083939027.tmp ---- PASS: TestFileSS_CreateSnapshot (0.00s) + testing.go:153: 2024-01-08T00:30:53.962Z [INFO] creating new snapshot: path=/tmp/raft2195550648/snapshots/3-10-1704673853962.tmp + testing.go:153: 2024-01-08T00:30:53.964Z [WARN] found temporary snapshot: name=3-10-1704673853962.tmp +--- PASS: TestFileSS_CreateSnapshot (0.01s) === RUN TestFileSS_CancelSnapshot - testing.go:153: 2025-02-09T06:52:19.031Z [INFO] creating new snapshot: path=/tmp/raft1571085551/snapshots/3-10-1739083939031.tmp + testing.go:153: 2024-01-08T00:30:53.972Z [INFO] creating new snapshot: path=/tmp/raft2000699288/snapshots/3-10-1704673853972.tmp --- PASS: TestFileSS_CancelSnapshot (0.00s) === RUN TestFileSS_Retention - testing.go:153: 2025-02-09T06:52:19.033Z [INFO] creating new snapshot: path=/tmp/raft2011223678/snapshots/3-10-1739083939033.tmp - testing.go:153: 2025-02-09T06:52:19.036Z [INFO] creating new snapshot: path=/tmp/raft2011223678/snapshots/3-11-1739083939036.tmp - testing.go:153: 2025-02-09T06:52:19.039Z [INFO] creating new snapshot: path=/tmp/raft2011223678/snapshots/3-12-1739083939039.tmp - testing.go:153: 2025-02-09T06:52:19.042Z [INFO] reaping snapshot: path=/tmp/raft2011223678/snapshots/3-10-1739083939033 - testing.go:153: 2025-02-09T06:52:19.042Z [INFO] creating new snapshot: path=/tmp/raft2011223678/snapshots/3-13-1739083939042.tmp - testing.go:153: 2025-02-09T06:52:19.045Z [INFO] reaping snapshot: path=/tmp/raft2011223678/snapshots/3-11-1739083939036 - testing.go:153: 2025-02-09T06:52:19.045Z [INFO] creating new snapshot: path=/tmp/raft2011223678/snapshots/3-14-1739083939045.tmp - testing.go:153: 2025-02-09T06:52:19.048Z [INFO] reaping snapshot: path=/tmp/raft2011223678/snapshots/3-12-1739083939039 ---- PASS: TestFileSS_Retention (0.02s) + testing.go:153: 2024-01-08T00:30:53.976Z [INFO] creating new snapshot: path=/tmp/raft2965010602/snapshots/3-10-1704673853976.tmp + testing.go:153: 2024-01-08T00:30:53.984Z [INFO] creating new snapshot: path=/tmp/raft2965010602/snapshots/3-11-1704673853984.tmp + testing.go:153: 2024-01-08T00:30:53.992Z [INFO] creating new snapshot: path=/tmp/raft2965010602/snapshots/3-12-1704673853992.tmp + testing.go:153: 2024-01-08T00:30:53.999Z [INFO] reaping snapshot: path=/tmp/raft2965010602/snapshots/3-10-1704673853976 + testing.go:153: 2024-01-08T00:30:53.999Z [INFO] creating new snapshot: path=/tmp/raft2965010602/snapshots/3-13-1704673853999.tmp + testing.go:153: 2024-01-08T00:30:54.006Z [INFO] reaping snapshot: path=/tmp/raft2965010602/snapshots/3-11-1704673853984 + testing.go:153: 2024-01-08T00:30:54.007Z [INFO] creating new snapshot: path=/tmp/raft2965010602/snapshots/3-14-1704673854007.tmp + testing.go:153: 2024-01-08T00:30:54.014Z [INFO] reaping snapshot: path=/tmp/raft2965010602/snapshots/3-12-1704673853992 +--- PASS: TestFileSS_Retention (0.04s) === RUN TestFileSS_BadPerm --- PASS: TestFileSS_BadPerm (0.00s) === RUN TestFileSS_MissingParentDir --- PASS: TestFileSS_MissingParentDir (0.00s) === RUN TestFileSS_Ordering - testing.go:153: 2025-02-09T06:52:19.049Z [INFO] creating new snapshot: path=/tmp/raft3649208096/snapshots/5-130350-1739083939049.tmp - testing.go:153: 2025-02-09T06:52:19.052Z [INFO] creating new snapshot: path=/tmp/raft3649208096/snapshots/36-204917-1739083939052.tmp ---- PASS: TestFileSS_Ordering (0.01s) + testing.go:153: 2024-01-08T00:30:54.015Z [INFO] creating new snapshot: path=/tmp/raft1498870585/snapshots/5-130350-1704673854015.tmp + testing.go:153: 2024-01-08T00:30:54.023Z [INFO] creating new snapshot: path=/tmp/raft1498870585/snapshots/36-204917-1704673854023.tmp +--- PASS: TestFileSS_Ordering (0.02s) === RUN TestDeferFutureSuccess --- PASS: TestDeferFutureSuccess (0.00s) === RUN TestDeferFutureError @@ -981,7 +1017,7 @@ === RUN TestLogCache --- PASS: TestLogCache (0.00s) === RUN TestNetworkTransport_CloseStreams ---- PASS: TestNetworkTransport_CloseStreams (0.00s) +--- PASS: TestNetworkTransport_CloseStreams (0.01s) === RUN TestNetworkTransport_StartStop --- PASS: TestNetworkTransport_StartStop (0.00s) === RUN TestNetworkTransport_Heartbeat_FastPath @@ -1002,29 +1038,29 @@ === RUN TestNetworkTransport_EncodeDecode_AddressProvider --- PASS: TestNetworkTransport_EncodeDecode_AddressProvider (0.00s) === RUN TestNetworkTransport_PooledConn ---- PASS: TestNetworkTransport_PooledConn (0.00s) +--- PASS: TestNetworkTransport_PooledConn (0.03s) === RUN TestNetworkTransport_ListenBackoff - net_transport_test.go:718: countingWriter: 2025-02-09T06:52:19.073Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-01-08T00:30:54.089Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:718: countingWriter: 2025-02-09T06:52:19.078Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-01-08T00:30:54.094Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:718: countingWriter: 2025-02-09T06:52:19.088Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-01-08T00:30:54.105Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:718: countingWriter: 2025-02-09T06:52:19.108Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-01-08T00:30:54.125Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:718: countingWriter: 2025-02-09T06:52:19.149Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-01-08T00:30:54.165Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:718: countingWriter: 2025-02-09T06:52:19.229Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-01-08T00:30:54.246Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:718: countingWriter: 2025-02-09T06:52:19.389Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-01-08T00:30:54.407Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:718: countingWriter: 2025-02-09T06:52:19.710Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-01-08T00:30:54.728Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:718: countingWriter: 2025-02-09T06:52:20.350Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-01-08T00:30:55.368Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:718: countingWriter: 2025-02-09T06:52:21.355Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-01-08T00:30:56.372Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:718: countingWriter: 2025-02-09T06:52:22.355Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-01-08T00:30:57.376Z [ERROR] test: failed to accept connection: error="intentional error in test" --- PASS: TestNetworkTransport_ListenBackoff (4.00s) === RUN TestPeersJSON_BadConfiguration @@ -1034,1119 +1070,1137 @@ === RUN TestPeersJSON_ReadConfigJSON --- PASS: TestPeersJSON_ReadConfigJSON (0.00s) === RUN TestRaft_StartStop -testLoggerAdapter verbose: server-30f4cbd2-56a4-ef70-b693-1c18a9118dea: 2025-02-09T06:52:23.077Z [INFO] server-30f4cbd2-56a4-ef70-b693-1c18a9118dea: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-30f4cbd2-56a4-ef70-b693-1c18a9118dea Address:30f4cbd2-56a4-ef70-b693-1c18a9118dea}]" - testing.go:149: server-30f4cbd2-56a4-ef70-b693-1c18a9118dea: 2025-02-09T06:52:23.077Z [INFO] server-30f4cbd2-56a4-ef70-b693-1c18a9118dea: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-30f4cbd2-56a4-ef70-b693-1c18a9118dea Address:30f4cbd2-56a4-ef70-b693-1c18a9118dea}]" +testLoggerAdapter verbose: server-43a6be72-e193-8e23-a74a-0da1252ae32c: 2024-01-08T00:30:58.096Z [INFO] server-43a6be72-e193-8e23-a74a-0da1252ae32c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-43a6be72-e193-8e23-a74a-0da1252ae32c Address:43a6be72-e193-8e23-a74a-0da1252ae32c}]" + testing.go:149: server-43a6be72-e193-8e23-a74a-0da1252ae32c: 2024-01-08T00:30:58.096Z [INFO] server-43a6be72-e193-8e23-a74a-0da1252ae32c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-43a6be72-e193-8e23-a74a-0da1252ae32c Address:43a6be72-e193-8e23-a74a-0da1252ae32c}]" --- PASS: TestRaft_StartStop (0.00s) === RUN TestRaft_AfterShutdown -testLoggerAdapter verbose: server-576fcbf2-f959-d3e1-c712-8860ff5d9db4: 2025-02-09T06:52:23.077Z [INFO] server-576fcbf2-f959-d3e1-c712-8860ff5d9db4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-576fcbf2-f959-d3e1-c712-8860ff5d9db4 Address:576fcbf2-f959-d3e1-c712-8860ff5d9db4}]" - testing.go:149: server-576fcbf2-f959-d3e1-c712-8860ff5d9db4: 2025-02-09T06:52:23.077Z [INFO] server-576fcbf2-f959-d3e1-c712-8860ff5d9db4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-576fcbf2-f959-d3e1-c712-8860ff5d9db4 Address:576fcbf2-f959-d3e1-c712-8860ff5d9db4}]" +testLoggerAdapter verbose: server-a683a6f6-84c9-0ab9-b8dc-a5b3b6309898: 2024-01-08T00:30:58.098Z [INFO] server-a683a6f6-84c9-0ab9-b8dc-a5b3b6309898: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a683a6f6-84c9-0ab9-b8dc-a5b3b6309898 Address:a683a6f6-84c9-0ab9-b8dc-a5b3b6309898}]" + testing.go:149: server-a683a6f6-84c9-0ab9-b8dc-a5b3b6309898: 2024-01-08T00:30:58.098Z [INFO] server-a683a6f6-84c9-0ab9-b8dc-a5b3b6309898: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a683a6f6-84c9-0ab9-b8dc-a5b3b6309898 Address:a683a6f6-84c9-0ab9-b8dc-a5b3b6309898}]" --- PASS: TestRaft_AfterShutdown (0.00s) === RUN TestRaft_LiveBootstrap -testLoggerAdapter verbose: server-cbc143bc-f7b2-63e9-78b9-2d875509c874: 2025-02-09T06:52:23.078Z [INFO] server-cbc143bc-f7b2-63e9-78b9-2d875509c874: initial configuration: index=0 servers=[] - testing.go:149: server-cbc143bc-f7b2-63e9-78b9-2d875509c874: 2025-02-09T06:52:23.078Z [INFO] server-cbc143bc-f7b2-63e9-78b9-2d875509c874: initial configuration: index=0 servers=[] -testLoggerAdapter verbose: server-eb255dfb-b4e4-1c0d-8958-0b5f42c242cf: 2025-02-09T06:52:23.078Z [INFO] server-eb255dfb-b4e4-1c0d-8958-0b5f42c242cf: initial configuration: index=0 servers=[] - testing.go:149: server-eb255dfb-b4e4-1c0d-8958-0b5f42c242cf: 2025-02-09T06:52:23.078Z [INFO] server-eb255dfb-b4e4-1c0d-8958-0b5f42c242cf: initial configuration: index=0 servers=[] -testLoggerAdapter verbose: server-1fb07c3f-e017-89bb-8d6b-bc1027e97eec: 2025-02-09T06:52:23.078Z [INFO] server-1fb07c3f-e017-89bb-8d6b-bc1027e97eec: initial configuration: index=0 servers=[] -testLoggerAdapter verbose: server-cbc143bc-f7b2-63e9-78b9-2d875509c874: 2025-02-09T06:52:23.078Z [INFO] server-cbc143bc-f7b2-63e9-78b9-2d875509c874: entering follower state: follower="Node at cbc143bc-f7b2-63e9-78b9-2d875509c874 [Follower]" leader= -testLoggerAdapter verbose: server-eb255dfb-b4e4-1c0d-8958-0b5f42c242cf: 2025-02-09T06:52:23.078Z [INFO] server-eb255dfb-b4e4-1c0d-8958-0b5f42c242cf: entering follower state: follower="Node at eb255dfb-b4e4-1c0d-8958-0b5f42c242cf [Follower]" leader= - testing.go:149: server-1fb07c3f-e017-89bb-8d6b-bc1027e97eec: 2025-02-09T06:52:23.078Z [INFO] server-1fb07c3f-e017-89bb-8d6b-bc1027e97eec: initial configuration: index=0 servers=[] - testing.go:149: server-cbc143bc-f7b2-63e9-78b9-2d875509c874: 2025-02-09T06:52:23.078Z [INFO] server-cbc143bc-f7b2-63e9-78b9-2d875509c874: entering follower state: follower="Node at cbc143bc-f7b2-63e9-78b9-2d875509c874 [Follower]" leader= -testLoggerAdapter verbose: server-1fb07c3f-e017-89bb-8d6b-bc1027e97eec: 2025-02-09T06:52:23.078Z [INFO] server-1fb07c3f-e017-89bb-8d6b-bc1027e97eec: entering follower state: follower="Node at 1fb07c3f-e017-89bb-8d6b-bc1027e97eec [Follower]" leader= - testing.go:149: server-eb255dfb-b4e4-1c0d-8958-0b5f42c242cf: 2025-02-09T06:52:23.078Z [INFO] server-eb255dfb-b4e4-1c0d-8958-0b5f42c242cf: entering follower state: follower="Node at eb255dfb-b4e4-1c0d-8958-0b5f42c242cf [Follower]" leader= - testing.go:149: server-1fb07c3f-e017-89bb-8d6b-bc1027e97eec: 2025-02-09T06:52:23.078Z [INFO] server-1fb07c3f-e017-89bb-8d6b-bc1027e97eec: entering follower state: follower="Node at 1fb07c3f-e017-89bb-8d6b-bc1027e97eec [Follower]" leader= -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:23.079Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T06:52:23.079Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-1fb07c3f-e017-89bb-8d6b-bc1027e97eec: 2025-02-09T06:52:23.129Z [WARN] server-1fb07c3f-e017-89bb-8d6b-bc1027e97eec: no known peers, aborting election - testing.go:149: server-1fb07c3f-e017-89bb-8d6b-bc1027e97eec: 2025-02-09T06:52:23.129Z [WARN] server-1fb07c3f-e017-89bb-8d6b-bc1027e97eec: no known peers, aborting election -testLoggerAdapter verbose: server-eb255dfb-b4e4-1c0d-8958-0b5f42c242cf: 2025-02-09T06:52:23.140Z [WARN] server-eb255dfb-b4e4-1c0d-8958-0b5f42c242cf: no known peers, aborting election - testing.go:149: server-eb255dfb-b4e4-1c0d-8958-0b5f42c242cf: 2025-02-09T06:52:23.140Z [WARN] server-eb255dfb-b4e4-1c0d-8958-0b5f42c242cf: no known peers, aborting election -testLoggerAdapter verbose: server-cbc143bc-f7b2-63e9-78b9-2d875509c874: 2025-02-09T06:52:23.158Z [WARN] server-cbc143bc-f7b2-63e9-78b9-2d875509c874: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-cbc143bc-f7b2-63e9-78b9-2d875509c874: 2025-02-09T06:52:23.158Z [WARN] server-cbc143bc-f7b2-63e9-78b9-2d875509c874: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-cbc143bc-f7b2-63e9-78b9-2d875509c874: 2025-02-09T06:52:23.158Z [INFO] server-cbc143bc-f7b2-63e9-78b9-2d875509c874: entering candidate state: node="Node at cbc143bc-f7b2-63e9-78b9-2d875509c874 [Candidate]" term=2 - testing.go:149: server-cbc143bc-f7b2-63e9-78b9-2d875509c874: 2025-02-09T06:52:23.158Z [INFO] server-cbc143bc-f7b2-63e9-78b9-2d875509c874: entering candidate state: node="Node at cbc143bc-f7b2-63e9-78b9-2d875509c874 [Candidate]" term=2 -testLoggerAdapter verbose: server-cbc143bc-f7b2-63e9-78b9-2d875509c874: 2025-02-09T06:52:23.158Z [INFO] server-cbc143bc-f7b2-63e9-78b9-2d875509c874: election won: tally=2 - testing.go:149: server-cbc143bc-f7b2-63e9-78b9-2d875509c874: 2025-02-09T06:52:23.158Z [INFO] server-cbc143bc-f7b2-63e9-78b9-2d875509c874: election won: tally=2 -testLoggerAdapter verbose: server-cbc143bc-f7b2-63e9-78b9-2d875509c874: 2025-02-09T06:52:23.158Z [INFO] server-cbc143bc-f7b2-63e9-78b9-2d875509c874: entering leader state: leader="Node at cbc143bc-f7b2-63e9-78b9-2d875509c874 [Leader]" - testing.go:149: server-cbc143bc-f7b2-63e9-78b9-2d875509c874: 2025-02-09T06:52:23.158Z [INFO] server-cbc143bc-f7b2-63e9-78b9-2d875509c874: entering leader state: leader="Node at cbc143bc-f7b2-63e9-78b9-2d875509c874 [Leader]" -testLoggerAdapter verbose: server-cbc143bc-f7b2-63e9-78b9-2d875509c874: 2025-02-09T06:52:23.158Z [INFO] server-cbc143bc-f7b2-63e9-78b9-2d875509c874: added peer, starting replication: peer=server-eb255dfb-b4e4-1c0d-8958-0b5f42c242cf - testing.go:149: server-cbc143bc-f7b2-63e9-78b9-2d875509c874: 2025-02-09T06:52:23.158Z [INFO] server-cbc143bc-f7b2-63e9-78b9-2d875509c874: added peer, starting replication: peer=server-eb255dfb-b4e4-1c0d-8958-0b5f42c242cf -testLoggerAdapter verbose: server-cbc143bc-f7b2-63e9-78b9-2d875509c874: 2025-02-09T06:52:23.158Z [INFO] server-cbc143bc-f7b2-63e9-78b9-2d875509c874: added peer, starting replication: peer=server-1fb07c3f-e017-89bb-8d6b-bc1027e97eec - testing.go:149: server-cbc143bc-f7b2-63e9-78b9-2d875509c874: 2025-02-09T06:52:23.158Z [INFO] server-cbc143bc-f7b2-63e9-78b9-2d875509c874: added peer, starting replication: peer=server-1fb07c3f-e017-89bb-8d6b-bc1027e97eec -testLoggerAdapter verbose: server-1fb07c3f-e017-89bb-8d6b-bc1027e97eec: 2025-02-09T06:52:23.158Z [WARN] server-1fb07c3f-e017-89bb-8d6b-bc1027e97eec: failed to get previous log: previous-index=1 last-index=0 error="log not found" - testing.go:149: server-1fb07c3f-e017-89bb-8d6b-bc1027e97eec: 2025-02-09T06:52:23.158Z [WARN] server-1fb07c3f-e017-89bb-8d6b-bc1027e97eec: failed to get previous log: previous-index=1 last-index=0 error="log not found" -testLoggerAdapter verbose: server-cbc143bc-f7b2-63e9-78b9-2d875509c874: 2025-02-09T06:52:23.159Z [WARN] server-cbc143bc-f7b2-63e9-78b9-2d875509c874: appendEntries rejected, sending older logs: peer="{Voter server-1fb07c3f-e017-89bb-8d6b-bc1027e97eec 1fb07c3f-e017-89bb-8d6b-bc1027e97eec}" next=1 - testing.go:149: server-cbc143bc-f7b2-63e9-78b9-2d875509c874: 2025-02-09T06:52:23.159Z [WARN] server-cbc143bc-f7b2-63e9-78b9-2d875509c874: appendEntries rejected, sending older logs: peer="{Voter server-1fb07c3f-e017-89bb-8d6b-bc1027e97eec 1fb07c3f-e017-89bb-8d6b-bc1027e97eec}" next=1 -testLoggerAdapter verbose: server-eb255dfb-b4e4-1c0d-8958-0b5f42c242cf: 2025-02-09T06:52:23.158Z [WARN] server-eb255dfb-b4e4-1c0d-8958-0b5f42c242cf: failed to get previous log: previous-index=1 last-index=0 error="log not found" - testing.go:149: server-eb255dfb-b4e4-1c0d-8958-0b5f42c242cf: 2025-02-09T06:52:23.158Z [WARN] server-eb255dfb-b4e4-1c0d-8958-0b5f42c242cf: failed to get previous log: previous-index=1 last-index=0 error="log not found" -testLoggerAdapter verbose: server-cbc143bc-f7b2-63e9-78b9-2d875509c874: 2025-02-09T06:52:23.159Z [INFO] server-cbc143bc-f7b2-63e9-78b9-2d875509c874: pipelining replication: peer="{Voter server-1fb07c3f-e017-89bb-8d6b-bc1027e97eec 1fb07c3f-e017-89bb-8d6b-bc1027e97eec}" - testing.go:149: server-cbc143bc-f7b2-63e9-78b9-2d875509c874: 2025-02-09T06:52:23.159Z [INFO] server-cbc143bc-f7b2-63e9-78b9-2d875509c874: pipelining replication: peer="{Voter server-1fb07c3f-e017-89bb-8d6b-bc1027e97eec 1fb07c3f-e017-89bb-8d6b-bc1027e97eec}" -testLoggerAdapter verbose: server-cbc143bc-f7b2-63e9-78b9-2d875509c874: 2025-02-09T06:52:23.159Z [WARN] server-cbc143bc-f7b2-63e9-78b9-2d875509c874: appendEntries rejected, sending older logs: peer="{Voter server-eb255dfb-b4e4-1c0d-8958-0b5f42c242cf eb255dfb-b4e4-1c0d-8958-0b5f42c242cf}" next=1 - testing.go:149: server-cbc143bc-f7b2-63e9-78b9-2d875509c874: 2025-02-09T06:52:23.159Z [WARN] server-cbc143bc-f7b2-63e9-78b9-2d875509c874: appendEntries rejected, sending older logs: peer="{Voter server-eb255dfb-b4e4-1c0d-8958-0b5f42c242cf eb255dfb-b4e4-1c0d-8958-0b5f42c242cf}" next=1 -testLoggerAdapter verbose: server-cbc143bc-f7b2-63e9-78b9-2d875509c874: 2025-02-09T06:52:23.159Z [INFO] server-cbc143bc-f7b2-63e9-78b9-2d875509c874: pipelining replication: peer="{Voter server-eb255dfb-b4e4-1c0d-8958-0b5f42c242cf eb255dfb-b4e4-1c0d-8958-0b5f42c242cf}" - testing.go:149: server-cbc143bc-f7b2-63e9-78b9-2d875509c874: 2025-02-09T06:52:23.159Z [INFO] server-cbc143bc-f7b2-63e9-78b9-2d875509c874: pipelining replication: peer="{Voter server-eb255dfb-b4e4-1c0d-8958-0b5f42c242cf eb255dfb-b4e4-1c0d-8958-0b5f42c242cf}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:23.263Z [INFO] stable state for Follower reached at 2025-02-09 06:52:23.158854082 +0000 UTC m=+4.137662847 (2 nodes), 79.67914ms from start of poll, 80.319386ms from cluster start. Timeout at 2025-02-09 06:52:23.26395573 +0000 UTC m=+4.242764488, 105.101641ms after stability - testing.go:149: cluster: 2025-02-09T06:52:23.263Z [INFO] stable state for Follower reached at 2025-02-09 06:52:23.158854082 +0000 UTC m=+4.137662847 (2 nodes), 79.67914ms from start of poll, 80.319386ms from cluster start. Timeout at 2025-02-09 06:52:23.26395573 +0000 UTC m=+4.242764488, 105.101641ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:23.264Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:23.264Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:23.369Z [INFO] stable state for Leader reached at 2025-02-09 06:52:23.264039303 +0000 UTC m=+4.242848066 (1 nodes), 834ns from start of poll, 185.504605ms from cluster start. Timeout at 2025-02-09 06:52:23.369226607 +0000 UTC m=+4.348035364, 105.187298ms after stability - testing.go:149: cluster: 2025-02-09T06:52:23.369Z [INFO] stable state for Leader reached at 2025-02-09 06:52:23.264039303 +0000 UTC m=+4.242848066 (1 nodes), 834ns from start of poll, 185.504605ms from cluster start. Timeout at 2025-02-09 06:52:23.369226607 +0000 UTC m=+4.348035364, 105.187298ms after stability -testLoggerAdapter verbose: server-cbc143bc-f7b2-63e9-78b9-2d875509c874: 2025-02-09T06:52:23.380Z [INFO] server-cbc143bc-f7b2-63e9-78b9-2d875509c874: aborting pipeline replication: peer="{Voter server-1fb07c3f-e017-89bb-8d6b-bc1027e97eec 1fb07c3f-e017-89bb-8d6b-bc1027e97eec}" - testing.go:149: server-cbc143bc-f7b2-63e9-78b9-2d875509c874: 2025-02-09T06:52:23.380Z [INFO] server-cbc143bc-f7b2-63e9-78b9-2d875509c874: aborting pipeline replication: peer="{Voter server-1fb07c3f-e017-89bb-8d6b-bc1027e97eec 1fb07c3f-e017-89bb-8d6b-bc1027e97eec}" -testLoggerAdapter verbose: server-cbc143bc-f7b2-63e9-78b9-2d875509c874: 2025-02-09T06:52:23.380Z [INFO] server-cbc143bc-f7b2-63e9-78b9-2d875509c874: aborting pipeline replication: peer="{Voter server-eb255dfb-b4e4-1c0d-8958-0b5f42c242cf eb255dfb-b4e4-1c0d-8958-0b5f42c242cf}" - testing.go:149: server-cbc143bc-f7b2-63e9-78b9-2d875509c874: 2025-02-09T06:52:23.380Z [INFO] server-cbc143bc-f7b2-63e9-78b9-2d875509c874: aborting pipeline replication: peer="{Voter server-eb255dfb-b4e4-1c0d-8958-0b5f42c242cf eb255dfb-b4e4-1c0d-8958-0b5f42c242cf}" -testLoggerAdapter verbose: server-cbc143bc-f7b2-63e9-78b9-2d875509c874: 2025-02-09T06:52:23.430Z [ERROR] server-cbc143bc-f7b2-63e9-78b9-2d875509c874: failed to heartbeat to: peer=1fb07c3f-e017-89bb-8d6b-bc1027e97eec error="command timed out" - testing.go:149: server-cbc143bc-f7b2-63e9-78b9-2d875509c874: 2025-02-09T06:52:23.430Z [ERROR] server-cbc143bc-f7b2-63e9-78b9-2d875509c874: failed to heartbeat to: peer=1fb07c3f-e017-89bb-8d6b-bc1027e97eec error="command timed out" ---- PASS: TestRaft_LiveBootstrap (0.35s) +testLoggerAdapter verbose: server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: 2024-01-08T00:30:58.099Z [INFO] server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: initial configuration: index=0 servers=[] + testing.go:149: server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: 2024-01-08T00:30:58.099Z [INFO] server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: initial configuration: index=0 servers=[] +testLoggerAdapter verbose: server-973d02ab-ccbc-3aa3-dfdd-7014fee58249: 2024-01-08T00:30:58.099Z [INFO] server-973d02ab-ccbc-3aa3-dfdd-7014fee58249: initial configuration: index=0 servers=[] + testing.go:149: server-973d02ab-ccbc-3aa3-dfdd-7014fee58249: 2024-01-08T00:30:58.099Z [INFO] server-973d02ab-ccbc-3aa3-dfdd-7014fee58249: initial configuration: index=0 servers=[] +testLoggerAdapter verbose: server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: 2024-01-08T00:30:58.099Z [INFO] server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: entering follower state: follower="Node at a2bf7e73-f626-d3e2-e4ac-44b433be2cbb [Follower]" leader= + testing.go:149: server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: 2024-01-08T00:30:58.099Z [INFO] server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: entering follower state: follower="Node at a2bf7e73-f626-d3e2-e4ac-44b433be2cbb [Follower]" leader= +testLoggerAdapter verbose: server-973d02ab-ccbc-3aa3-dfdd-7014fee58249: 2024-01-08T00:30:58.099Z [INFO] server-973d02ab-ccbc-3aa3-dfdd-7014fee58249: entering follower state: follower="Node at 973d02ab-ccbc-3aa3-dfdd-7014fee58249 [Follower]" leader= + testing.go:149: server-973d02ab-ccbc-3aa3-dfdd-7014fee58249: 2024-01-08T00:30:58.099Z [INFO] server-973d02ab-ccbc-3aa3-dfdd-7014fee58249: entering follower state: follower="Node at 973d02ab-ccbc-3aa3-dfdd-7014fee58249 [Follower]" leader= +testLoggerAdapter verbose: server-a29e7003-fbdb-813d-959e-b1862c65567c: 2024-01-08T00:30:58.099Z [INFO] server-a29e7003-fbdb-813d-959e-b1862c65567c: initial configuration: index=0 servers=[] + testing.go:149: server-a29e7003-fbdb-813d-959e-b1862c65567c: 2024-01-08T00:30:58.099Z [INFO] server-a29e7003-fbdb-813d-959e-b1862c65567c: initial configuration: index=0 servers=[] +testLoggerAdapter verbose: server-a29e7003-fbdb-813d-959e-b1862c65567c: 2024-01-08T00:30:58.099Z [INFO] server-a29e7003-fbdb-813d-959e-b1862c65567c: entering follower state: follower="Node at a29e7003-fbdb-813d-959e-b1862c65567c [Follower]" leader= + testing.go:149: server-a29e7003-fbdb-813d-959e-b1862c65567c: 2024-01-08T00:30:58.099Z [INFO] server-a29e7003-fbdb-813d-959e-b1862c65567c: entering follower state: follower="Node at a29e7003-fbdb-813d-959e-b1862c65567c [Follower]" leader= +testLoggerAdapter verbose: cluster: 2024-01-08T00:30:58.099Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T00:30:58.099Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: server-973d02ab-ccbc-3aa3-dfdd-7014fee58249: 2024-01-08T00:30:58.164Z [WARN] server-973d02ab-ccbc-3aa3-dfdd-7014fee58249: no known peers, aborting election + testing.go:149: server-973d02ab-ccbc-3aa3-dfdd-7014fee58249: 2024-01-08T00:30:58.164Z [WARN] server-973d02ab-ccbc-3aa3-dfdd-7014fee58249: no known peers, aborting election +testLoggerAdapter verbose: server-a29e7003-fbdb-813d-959e-b1862c65567c: 2024-01-08T00:30:58.174Z [WARN] server-a29e7003-fbdb-813d-959e-b1862c65567c: no known peers, aborting election + testing.go:149: server-a29e7003-fbdb-813d-959e-b1862c65567c: 2024-01-08T00:30:58.174Z [WARN] server-a29e7003-fbdb-813d-959e-b1862c65567c: no known peers, aborting election +testLoggerAdapter verbose: server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: 2024-01-08T00:30:58.181Z [WARN] server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: 2024-01-08T00:30:58.181Z [WARN] server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: 2024-01-08T00:30:58.181Z [INFO] server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: entering candidate state: node="Node at a2bf7e73-f626-d3e2-e4ac-44b433be2cbb [Candidate]" term=2 + testing.go:149: server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: 2024-01-08T00:30:58.181Z [INFO] server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: entering candidate state: node="Node at a2bf7e73-f626-d3e2-e4ac-44b433be2cbb [Candidate]" term=2 +testLoggerAdapter verbose: server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: 2024-01-08T00:30:58.181Z [INFO] server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: election won: tally=2 + testing.go:149: server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: 2024-01-08T00:30:58.181Z [INFO] server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: election won: tally=2 +testLoggerAdapter verbose: server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: 2024-01-08T00:30:58.181Z [INFO] server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: entering leader state: leader="Node at a2bf7e73-f626-d3e2-e4ac-44b433be2cbb [Leader]" + testing.go:149: server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: 2024-01-08T00:30:58.181Z [INFO] server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: entering leader state: leader="Node at a2bf7e73-f626-d3e2-e4ac-44b433be2cbb [Leader]" +testLoggerAdapter verbose: server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: 2024-01-08T00:30:58.181Z [INFO] server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: added peer, starting replication: peer=server-973d02ab-ccbc-3aa3-dfdd-7014fee58249 + testing.go:149: server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: 2024-01-08T00:30:58.181Z [INFO] server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: added peer, starting replication: peer=server-973d02ab-ccbc-3aa3-dfdd-7014fee58249 +testLoggerAdapter verbose: server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: 2024-01-08T00:30:58.181Z [INFO] server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: added peer, starting replication: peer=server-a29e7003-fbdb-813d-959e-b1862c65567c + testing.go:149: server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: 2024-01-08T00:30:58.181Z [INFO] server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: added peer, starting replication: peer=server-a29e7003-fbdb-813d-959e-b1862c65567c +testLoggerAdapter verbose: server-a29e7003-fbdb-813d-959e-b1862c65567c: 2024-01-08T00:30:58.182Z [WARN] server-a29e7003-fbdb-813d-959e-b1862c65567c: failed to get previous log: previous-index=1 last-index=0 error="log not found" + testing.go:149: server-a29e7003-fbdb-813d-959e-b1862c65567c: 2024-01-08T00:30:58.182Z [WARN] server-a29e7003-fbdb-813d-959e-b1862c65567c: failed to get previous log: previous-index=1 last-index=0 error="log not found" +testLoggerAdapter verbose: server-973d02ab-ccbc-3aa3-dfdd-7014fee58249: 2024-01-08T00:30:58.182Z [WARN] server-973d02ab-ccbc-3aa3-dfdd-7014fee58249: failed to get previous log: previous-index=1 last-index=0 error="log not found" + testing.go:149: server-973d02ab-ccbc-3aa3-dfdd-7014fee58249: 2024-01-08T00:30:58.182Z [WARN] server-973d02ab-ccbc-3aa3-dfdd-7014fee58249: failed to get previous log: previous-index=1 last-index=0 error="log not found" +testLoggerAdapter verbose: server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: 2024-01-08T00:30:58.182Z [WARN] server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: appendEntries rejected, sending older logs: peer="{Voter server-a29e7003-fbdb-813d-959e-b1862c65567c a29e7003-fbdb-813d-959e-b1862c65567c}" next=1 + testing.go:149: server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: 2024-01-08T00:30:58.182Z [WARN] server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: appendEntries rejected, sending older logs: peer="{Voter server-a29e7003-fbdb-813d-959e-b1862c65567c a29e7003-fbdb-813d-959e-b1862c65567c}" next=1 +testLoggerAdapter verbose: server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: 2024-01-08T00:30:58.182Z [WARN] server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: appendEntries rejected, sending older logs: peer="{Voter server-973d02ab-ccbc-3aa3-dfdd-7014fee58249 973d02ab-ccbc-3aa3-dfdd-7014fee58249}" next=1 + testing.go:149: server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: 2024-01-08T00:30:58.182Z [WARN] server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: appendEntries rejected, sending older logs: peer="{Voter server-973d02ab-ccbc-3aa3-dfdd-7014fee58249 973d02ab-ccbc-3aa3-dfdd-7014fee58249}" next=1 +testLoggerAdapter verbose: server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: 2024-01-08T00:30:58.182Z [INFO] server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: pipelining replication: peer="{Voter server-a29e7003-fbdb-813d-959e-b1862c65567c a29e7003-fbdb-813d-959e-b1862c65567c}" + testing.go:149: server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: 2024-01-08T00:30:58.182Z [INFO] server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: pipelining replication: peer="{Voter server-a29e7003-fbdb-813d-959e-b1862c65567c a29e7003-fbdb-813d-959e-b1862c65567c}" +testLoggerAdapter verbose: server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: 2024-01-08T00:30:58.182Z [INFO] server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: pipelining replication: peer="{Voter server-973d02ab-ccbc-3aa3-dfdd-7014fee58249 973d02ab-ccbc-3aa3-dfdd-7014fee58249}" + testing.go:149: server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: 2024-01-08T00:30:58.182Z [INFO] server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: pipelining replication: peer="{Voter server-973d02ab-ccbc-3aa3-dfdd-7014fee58249 973d02ab-ccbc-3aa3-dfdd-7014fee58249}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:30:58.287Z [INFO] stable state for Follower reached at 2024-01-08 00:30:58.181969687 +0000 UTC m=+4.228858754 (2 nodes), 82.086071ms from start of poll, 82.684371ms from cluster start. Timeout at 2024-01-08 00:30:58.286983493 +0000 UTC m=+4.333872527, 105.013773ms after stability + testing.go:149: cluster: 2024-01-08T00:30:58.287Z [INFO] stable state for Follower reached at 2024-01-08 00:30:58.181969687 +0000 UTC m=+4.228858754 (2 nodes), 82.086071ms from start of poll, 82.684371ms from cluster start. Timeout at 2024-01-08 00:30:58.286983493 +0000 UTC m=+4.333872527, 105.013773ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T00:30:58.287Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:30:58.287Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T00:30:58.392Z [INFO] stable state for Leader reached at 2024-01-08 00:30:58.287143559 +0000 UTC m=+4.334032597 (1 nodes), 2.047µs from start of poll, 187.858214ms from cluster start. Timeout at 2024-01-08 00:30:58.39280871 +0000 UTC m=+4.439697752, 105.665155ms after stability + testing.go:149: cluster: 2024-01-08T00:30:58.392Z [INFO] stable state for Leader reached at 2024-01-08 00:30:58.287143559 +0000 UTC m=+4.334032597 (1 nodes), 2.047µs from start of poll, 187.858214ms from cluster start. Timeout at 2024-01-08 00:30:58.39280871 +0000 UTC m=+4.439697752, 105.665155ms after stability +testLoggerAdapter verbose: server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: 2024-01-08T00:30:58.404Z [INFO] server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: aborting pipeline replication: peer="{Voter server-a29e7003-fbdb-813d-959e-b1862c65567c a29e7003-fbdb-813d-959e-b1862c65567c}" + testing.go:149: server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: 2024-01-08T00:30:58.404Z [INFO] server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: aborting pipeline replication: peer="{Voter server-a29e7003-fbdb-813d-959e-b1862c65567c a29e7003-fbdb-813d-959e-b1862c65567c}" +testLoggerAdapter verbose: server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: 2024-01-08T00:30:58.404Z [INFO] server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: aborting pipeline replication: peer="{Voter server-973d02ab-ccbc-3aa3-dfdd-7014fee58249 973d02ab-ccbc-3aa3-dfdd-7014fee58249}" + testing.go:149: server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: 2024-01-08T00:30:58.404Z [INFO] server-a2bf7e73-f626-d3e2-e4ac-44b433be2cbb: aborting pipeline replication: peer="{Voter server-973d02ab-ccbc-3aa3-dfdd-7014fee58249 973d02ab-ccbc-3aa3-dfdd-7014fee58249}" +--- PASS: TestRaft_LiveBootstrap (0.31s) === RUN TestRaft_RecoverCluster_NoState -testLoggerAdapter verbose: server-ec4ff957-b81b-9581-93be-6df83e75d18d: 2025-02-09T06:52:23.431Z [INFO] server-ec4ff957-b81b-9581-93be-6df83e75d18d: initial configuration: index=0 servers=[] - testing.go:149: server-ec4ff957-b81b-9581-93be-6df83e75d18d: 2025-02-09T06:52:23.431Z [INFO] server-ec4ff957-b81b-9581-93be-6df83e75d18d: initial configuration: index=0 servers=[] -testLoggerAdapter verbose: server-ec4ff957-b81b-9581-93be-6df83e75d18d: 2025-02-09T06:52:23.431Z [INFO] server-ec4ff957-b81b-9581-93be-6df83e75d18d: entering follower state: follower="Node at ec4ff957-b81b-9581-93be-6df83e75d18d [Shutdown]" leader= - testing.go:149: server-ec4ff957-b81b-9581-93be-6df83e75d18d: 2025-02-09T06:52:23.431Z [INFO] server-ec4ff957-b81b-9581-93be-6df83e75d18d: entering follower state: follower="Node at ec4ff957-b81b-9581-93be-6df83e75d18d [Shutdown]" leader= +testLoggerAdapter verbose: server-2b6174a5-f346-120a-ff5e-ae3a9dbffd11: 2024-01-08T00:30:58.405Z [INFO] server-2b6174a5-f346-120a-ff5e-ae3a9dbffd11: initial configuration: index=0 servers=[] + testing.go:149: server-2b6174a5-f346-120a-ff5e-ae3a9dbffd11: 2024-01-08T00:30:58.405Z [INFO] server-2b6174a5-f346-120a-ff5e-ae3a9dbffd11: initial configuration: index=0 servers=[] +testLoggerAdapter verbose: server-2b6174a5-f346-120a-ff5e-ae3a9dbffd11: 2024-01-08T00:30:58.405Z [INFO] server-2b6174a5-f346-120a-ff5e-ae3a9dbffd11: entering follower state: follower="Node at 2b6174a5-f346-120a-ff5e-ae3a9dbffd11 [Shutdown]" leader= + testing.go:149: server-2b6174a5-f346-120a-ff5e-ae3a9dbffd11: 2024-01-08T00:30:58.405Z [INFO] server-2b6174a5-f346-120a-ff5e-ae3a9dbffd11: entering follower state: follower="Node at 2b6174a5-f346-120a-ff5e-ae3a9dbffd11 [Shutdown]" leader= --- PASS: TestRaft_RecoverCluster_NoState (0.00s) === RUN TestRaft_RecoverCluster raft_test.go:119: Disabled unreliable test --- SKIP: TestRaft_RecoverCluster (0.00s) === RUN TestRaft_HasExistingState -testLoggerAdapter verbose: server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: 2025-02-09T06:52:23.432Z [INFO] server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4cfcbcd0-981b-e060-8e39-69dec5f5017f Address:4cfcbcd0-981b-e060-8e39-69dec5f5017f} {Suffrage:Voter ID:server-83070580-0ca6-9a61-3d95-daeb3820ce41 Address:83070580-0ca6-9a61-3d95-daeb3820ce41}]" - testing.go:149: server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: 2025-02-09T06:52:23.432Z [INFO] server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4cfcbcd0-981b-e060-8e39-69dec5f5017f Address:4cfcbcd0-981b-e060-8e39-69dec5f5017f} {Suffrage:Voter ID:server-83070580-0ca6-9a61-3d95-daeb3820ce41 Address:83070580-0ca6-9a61-3d95-daeb3820ce41}]" -testLoggerAdapter verbose: server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: 2025-02-09T06:52:23.432Z [INFO] server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: entering follower state: follower="Node at 4cfcbcd0-981b-e060-8e39-69dec5f5017f [Follower]" leader= - testing.go:149: server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: 2025-02-09T06:52:23.432Z [INFO] server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: entering follower state: follower="Node at 4cfcbcd0-981b-e060-8e39-69dec5f5017f [Follower]" leader= -testLoggerAdapter verbose: server-83070580-0ca6-9a61-3d95-daeb3820ce41: 2025-02-09T06:52:23.432Z [INFO] server-83070580-0ca6-9a61-3d95-daeb3820ce41: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4cfcbcd0-981b-e060-8e39-69dec5f5017f Address:4cfcbcd0-981b-e060-8e39-69dec5f5017f} {Suffrage:Voter ID:server-83070580-0ca6-9a61-3d95-daeb3820ce41 Address:83070580-0ca6-9a61-3d95-daeb3820ce41}]" - testing.go:149: server-83070580-0ca6-9a61-3d95-daeb3820ce41: 2025-02-09T06:52:23.432Z [INFO] server-83070580-0ca6-9a61-3d95-daeb3820ce41: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4cfcbcd0-981b-e060-8e39-69dec5f5017f Address:4cfcbcd0-981b-e060-8e39-69dec5f5017f} {Suffrage:Voter ID:server-83070580-0ca6-9a61-3d95-daeb3820ce41 Address:83070580-0ca6-9a61-3d95-daeb3820ce41}]" -testLoggerAdapter verbose: server-83070580-0ca6-9a61-3d95-daeb3820ce41: 2025-02-09T06:52:23.432Z [INFO] server-83070580-0ca6-9a61-3d95-daeb3820ce41: entering follower state: follower="Node at 83070580-0ca6-9a61-3d95-daeb3820ce41 [Follower]" leader= - testing.go:149: server-83070580-0ca6-9a61-3d95-daeb3820ce41: 2025-02-09T06:52:23.432Z [INFO] server-83070580-0ca6-9a61-3d95-daeb3820ce41: entering follower state: follower="Node at 83070580-0ca6-9a61-3d95-daeb3820ce41 [Follower]" leader= -testLoggerAdapter verbose: server-09135aeb-9b38-9316-19a7-b432e5f60d23: 2025-02-09T06:52:23.432Z [INFO] server-09135aeb-9b38-9316-19a7-b432e5f60d23: initial configuration: index=0 servers=[] - testing.go:149: server-09135aeb-9b38-9316-19a7-b432e5f60d23: 2025-02-09T06:52:23.432Z [INFO] server-09135aeb-9b38-9316-19a7-b432e5f60d23: initial configuration: index=0 servers=[] -testLoggerAdapter verbose: server-09135aeb-9b38-9316-19a7-b432e5f60d23: 2025-02-09T06:52:23.432Z [INFO] server-09135aeb-9b38-9316-19a7-b432e5f60d23: entering follower state: follower="Node at 09135aeb-9b38-9316-19a7-b432e5f60d23 [Follower]" leader= - testing.go:149: server-09135aeb-9b38-9316-19a7-b432e5f60d23: 2025-02-09T06:52:23.432Z [INFO] server-09135aeb-9b38-9316-19a7-b432e5f60d23: entering follower state: follower="Node at 09135aeb-9b38-9316-19a7-b432e5f60d23 [Follower]" leader= -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:23.432Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:23.432Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-09135aeb-9b38-9316-19a7-b432e5f60d23: 2025-02-09T06:52:23.496Z [WARN] server-09135aeb-9b38-9316-19a7-b432e5f60d23: no known peers, aborting election - testing.go:149: server-09135aeb-9b38-9316-19a7-b432e5f60d23: 2025-02-09T06:52:23.496Z [WARN] server-09135aeb-9b38-9316-19a7-b432e5f60d23: no known peers, aborting election -testLoggerAdapter verbose: server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: 2025-02-09T06:52:23.509Z [WARN] server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: 2025-02-09T06:52:23.509Z [WARN] server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: 2025-02-09T06:52:23.509Z [INFO] server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: entering candidate state: node="Node at 4cfcbcd0-981b-e060-8e39-69dec5f5017f [Candidate]" term=2 - testing.go:149: server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: 2025-02-09T06:52:23.509Z [INFO] server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: entering candidate state: node="Node at 4cfcbcd0-981b-e060-8e39-69dec5f5017f [Candidate]" term=2 -testLoggerAdapter verbose: server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: 2025-02-09T06:52:23.509Z [INFO] server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: election won: tally=2 - testing.go:149: server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: 2025-02-09T06:52:23.509Z [INFO] server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: election won: tally=2 -testLoggerAdapter verbose: server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: 2025-02-09T06:52:23.509Z [INFO] server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: entering leader state: leader="Node at 4cfcbcd0-981b-e060-8e39-69dec5f5017f [Leader]" - testing.go:149: server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: 2025-02-09T06:52:23.509Z [INFO] server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: entering leader state: leader="Node at 4cfcbcd0-981b-e060-8e39-69dec5f5017f [Leader]" -testLoggerAdapter verbose: server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: 2025-02-09T06:52:23.509Z [INFO] server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: added peer, starting replication: peer=server-83070580-0ca6-9a61-3d95-daeb3820ce41 - testing.go:149: server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: 2025-02-09T06:52:23.509Z [INFO] server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: added peer, starting replication: peer=server-83070580-0ca6-9a61-3d95-daeb3820ce41 -testLoggerAdapter verbose: server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: 2025-02-09T06:52:23.509Z [INFO] server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: pipelining replication: peer="{Voter server-83070580-0ca6-9a61-3d95-daeb3820ce41 83070580-0ca6-9a61-3d95-daeb3820ce41}" - testing.go:149: server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: 2025-02-09T06:52:23.509Z [INFO] server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: pipelining replication: peer="{Voter server-83070580-0ca6-9a61-3d95-daeb3820ce41 83070580-0ca6-9a61-3d95-daeb3820ce41}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:23.615Z [INFO] stable state for Leader reached at 2025-02-09 06:52:23.509806819 +0000 UTC m=+4.488615584 (1 nodes), 76.910087ms from start of poll, 77.565441ms from cluster start. Timeout at 2025-02-09 06:52:23.615558569 +0000 UTC m=+4.594367329, 105.751745ms after stability - testing.go:149: cluster: 2025-02-09T06:52:23.615Z [INFO] stable state for Leader reached at 2025-02-09 06:52:23.509806819 +0000 UTC m=+4.488615584 (1 nodes), 76.910087ms from start of poll, 77.565441ms from cluster start. Timeout at 2025-02-09 06:52:23.615558569 +0000 UTC m=+4.594367329, 105.751745ms after stability -testLoggerAdapter verbose: server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: 2025-02-09T06:52:23.615Z [INFO] server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: updating configuration: command=AddStaging server-id=server-09135aeb-9b38-9316-19a7-b432e5f60d23 server-addr=09135aeb-9b38-9316-19a7-b432e5f60d23 servers="[{Suffrage:Voter ID:server-4cfcbcd0-981b-e060-8e39-69dec5f5017f Address:4cfcbcd0-981b-e060-8e39-69dec5f5017f} {Suffrage:Voter ID:server-83070580-0ca6-9a61-3d95-daeb3820ce41 Address:83070580-0ca6-9a61-3d95-daeb3820ce41} {Suffrage:Voter ID:server-09135aeb-9b38-9316-19a7-b432e5f60d23 Address:09135aeb-9b38-9316-19a7-b432e5f60d23}]" - testing.go:149: server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: 2025-02-09T06:52:23.615Z [INFO] server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: updating configuration: command=AddStaging server-id=server-09135aeb-9b38-9316-19a7-b432e5f60d23 server-addr=09135aeb-9b38-9316-19a7-b432e5f60d23 servers="[{Suffrage:Voter ID:server-4cfcbcd0-981b-e060-8e39-69dec5f5017f Address:4cfcbcd0-981b-e060-8e39-69dec5f5017f} {Suffrage:Voter ID:server-83070580-0ca6-9a61-3d95-daeb3820ce41 Address:83070580-0ca6-9a61-3d95-daeb3820ce41} {Suffrage:Voter ID:server-09135aeb-9b38-9316-19a7-b432e5f60d23 Address:09135aeb-9b38-9316-19a7-b432e5f60d23}]" -testLoggerAdapter verbose: server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: 2025-02-09T06:52:23.615Z [INFO] server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: added peer, starting replication: peer=server-09135aeb-9b38-9316-19a7-b432e5f60d23 - testing.go:149: server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: 2025-02-09T06:52:23.615Z [INFO] server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: added peer, starting replication: peer=server-09135aeb-9b38-9316-19a7-b432e5f60d23 -testLoggerAdapter verbose: server-09135aeb-9b38-9316-19a7-b432e5f60d23: 2025-02-09T06:52:23.615Z [WARN] server-09135aeb-9b38-9316-19a7-b432e5f60d23: failed to get previous log: previous-index=3 last-index=0 error="log not found" - testing.go:149: server-09135aeb-9b38-9316-19a7-b432e5f60d23: 2025-02-09T06:52:23.615Z [WARN] server-09135aeb-9b38-9316-19a7-b432e5f60d23: failed to get previous log: previous-index=3 last-index=0 error="log not found" -testLoggerAdapter verbose: server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: 2025-02-09T06:52:23.615Z [WARN] server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: appendEntries rejected, sending older logs: peer="{Voter server-09135aeb-9b38-9316-19a7-b432e5f60d23 09135aeb-9b38-9316-19a7-b432e5f60d23}" next=1 - testing.go:149: server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: 2025-02-09T06:52:23.615Z [WARN] server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: appendEntries rejected, sending older logs: peer="{Voter server-09135aeb-9b38-9316-19a7-b432e5f60d23 09135aeb-9b38-9316-19a7-b432e5f60d23}" next=1 -testLoggerAdapter verbose: server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: 2025-02-09T06:52:23.615Z [INFO] server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: pipelining replication: peer="{Voter server-09135aeb-9b38-9316-19a7-b432e5f60d23 09135aeb-9b38-9316-19a7-b432e5f60d23}" - testing.go:149: server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: 2025-02-09T06:52:23.615Z [INFO] server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: pipelining replication: peer="{Voter server-09135aeb-9b38-9316-19a7-b432e5f60d23 09135aeb-9b38-9316-19a7-b432e5f60d23}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:23.621Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:23.621Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:23.727Z [INFO] stable state for Leader reached at 2025-02-09 06:52:23.621956981 +0000 UTC m=+4.600765752 (1 nodes), 506ns from start of poll, 189.715609ms from cluster start. Timeout at 2025-02-09 06:52:23.72709286 +0000 UTC m=+4.705901620, 105.135868ms after stability - testing.go:149: cluster: 2025-02-09T06:52:23.727Z [INFO] stable state for Leader reached at 2025-02-09 06:52:23.621956981 +0000 UTC m=+4.600765752 (1 nodes), 506ns from start of poll, 189.715609ms from cluster start. Timeout at 2025-02-09 06:52:23.72709286 +0000 UTC m=+4.705901620, 105.135868ms after stability -testLoggerAdapter verbose: server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: 2025-02-09T06:52:23.727Z [INFO] server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: aborting pipeline replication: peer="{Voter server-83070580-0ca6-9a61-3d95-daeb3820ce41 83070580-0ca6-9a61-3d95-daeb3820ce41}" - testing.go:149: server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: 2025-02-09T06:52:23.727Z [INFO] server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: aborting pipeline replication: peer="{Voter server-83070580-0ca6-9a61-3d95-daeb3820ce41 83070580-0ca6-9a61-3d95-daeb3820ce41}" -testLoggerAdapter verbose: server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: 2025-02-09T06:52:23.727Z [INFO] server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: aborting pipeline replication: peer="{Voter server-09135aeb-9b38-9316-19a7-b432e5f60d23 09135aeb-9b38-9316-19a7-b432e5f60d23}" - testing.go:149: server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: 2025-02-09T06:52:23.727Z [INFO] server-4cfcbcd0-981b-e060-8e39-69dec5f5017f: aborting pipeline replication: peer="{Voter server-09135aeb-9b38-9316-19a7-b432e5f60d23 09135aeb-9b38-9316-19a7-b432e5f60d23}" ---- PASS: TestRaft_HasExistingState (0.30s) +testLoggerAdapter verbose: server-8e70590a-d0ae-be79-3970-60fa015aafee: 2024-01-08T00:30:58.406Z [INFO] server-8e70590a-d0ae-be79-3970-60fa015aafee: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8e70590a-d0ae-be79-3970-60fa015aafee Address:8e70590a-d0ae-be79-3970-60fa015aafee} {Suffrage:Voter ID:server-f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c Address:f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c}]" + testing.go:149: server-8e70590a-d0ae-be79-3970-60fa015aafee: 2024-01-08T00:30:58.406Z [INFO] server-8e70590a-d0ae-be79-3970-60fa015aafee: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8e70590a-d0ae-be79-3970-60fa015aafee Address:8e70590a-d0ae-be79-3970-60fa015aafee} {Suffrage:Voter ID:server-f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c Address:f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c}]" +testLoggerAdapter verbose: server-8e70590a-d0ae-be79-3970-60fa015aafee: 2024-01-08T00:30:58.406Z [INFO] server-8e70590a-d0ae-be79-3970-60fa015aafee: entering follower state: follower="Node at 8e70590a-d0ae-be79-3970-60fa015aafee [Follower]" leader= + testing.go:149: server-8e70590a-d0ae-be79-3970-60fa015aafee: 2024-01-08T00:30:58.406Z [INFO] server-8e70590a-d0ae-be79-3970-60fa015aafee: entering follower state: follower="Node at 8e70590a-d0ae-be79-3970-60fa015aafee [Follower]" leader= +testLoggerAdapter verbose: server-f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c: 2024-01-08T00:30:58.406Z [INFO] server-f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8e70590a-d0ae-be79-3970-60fa015aafee Address:8e70590a-d0ae-be79-3970-60fa015aafee} {Suffrage:Voter ID:server-f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c Address:f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c}]" + testing.go:149: server-f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c: 2024-01-08T00:30:58.406Z [INFO] server-f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8e70590a-d0ae-be79-3970-60fa015aafee Address:8e70590a-d0ae-be79-3970-60fa015aafee} {Suffrage:Voter ID:server-f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c Address:f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c}]" +testLoggerAdapter verbose: server-f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c: 2024-01-08T00:30:58.406Z [INFO] server-f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c: entering follower state: follower="Node at f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c [Follower]" leader= + testing.go:149: server-f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c: 2024-01-08T00:30:58.406Z [INFO] server-f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c: entering follower state: follower="Node at f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c [Follower]" leader= +testLoggerAdapter verbose: server-58d4ba3f-7492-7c34-c358-c714c349e806: 2024-01-08T00:30:58.407Z [INFO] server-58d4ba3f-7492-7c34-c358-c714c349e806: initial configuration: index=0 servers=[] + testing.go:149: server-58d4ba3f-7492-7c34-c358-c714c349e806: 2024-01-08T00:30:58.407Z [INFO] server-58d4ba3f-7492-7c34-c358-c714c349e806: initial configuration: index=0 servers=[] +testLoggerAdapter verbose: cluster: 2024-01-08T00:30:58.407Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-58d4ba3f-7492-7c34-c358-c714c349e806: 2024-01-08T00:30:58.407Z [INFO] server-58d4ba3f-7492-7c34-c358-c714c349e806: entering follower state: follower="Node at 58d4ba3f-7492-7c34-c358-c714c349e806 [Follower]" leader= + testing.go:149: cluster: 2024-01-08T00:30:58.407Z [INFO] starting stability test: raft-state=Leader + testing.go:149: server-58d4ba3f-7492-7c34-c358-c714c349e806: 2024-01-08T00:30:58.407Z [INFO] server-58d4ba3f-7492-7c34-c358-c714c349e806: entering follower state: follower="Node at 58d4ba3f-7492-7c34-c358-c714c349e806 [Follower]" leader= +testLoggerAdapter verbose: server-8e70590a-d0ae-be79-3970-60fa015aafee: 2024-01-08T00:30:58.460Z [WARN] server-8e70590a-d0ae-be79-3970-60fa015aafee: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-8e70590a-d0ae-be79-3970-60fa015aafee: 2024-01-08T00:30:58.460Z [WARN] server-8e70590a-d0ae-be79-3970-60fa015aafee: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-8e70590a-d0ae-be79-3970-60fa015aafee: 2024-01-08T00:30:58.460Z [INFO] server-8e70590a-d0ae-be79-3970-60fa015aafee: entering candidate state: node="Node at 8e70590a-d0ae-be79-3970-60fa015aafee [Candidate]" term=2 + testing.go:149: server-8e70590a-d0ae-be79-3970-60fa015aafee: 2024-01-08T00:30:58.460Z [INFO] server-8e70590a-d0ae-be79-3970-60fa015aafee: entering candidate state: node="Node at 8e70590a-d0ae-be79-3970-60fa015aafee [Candidate]" term=2 +testLoggerAdapter verbose: server-8e70590a-d0ae-be79-3970-60fa015aafee: 2024-01-08T00:30:58.460Z [INFO] server-8e70590a-d0ae-be79-3970-60fa015aafee: election won: tally=2 + testing.go:149: server-8e70590a-d0ae-be79-3970-60fa015aafee: 2024-01-08T00:30:58.460Z [INFO] server-8e70590a-d0ae-be79-3970-60fa015aafee: election won: tally=2 +testLoggerAdapter verbose: server-8e70590a-d0ae-be79-3970-60fa015aafee: 2024-01-08T00:30:58.461Z [INFO] server-8e70590a-d0ae-be79-3970-60fa015aafee: entering leader state: leader="Node at 8e70590a-d0ae-be79-3970-60fa015aafee [Leader]" + testing.go:149: server-8e70590a-d0ae-be79-3970-60fa015aafee: 2024-01-08T00:30:58.461Z [INFO] server-8e70590a-d0ae-be79-3970-60fa015aafee: entering leader state: leader="Node at 8e70590a-d0ae-be79-3970-60fa015aafee [Leader]" +testLoggerAdapter verbose: server-8e70590a-d0ae-be79-3970-60fa015aafee: 2024-01-08T00:30:58.461Z [INFO] server-8e70590a-d0ae-be79-3970-60fa015aafee: added peer, starting replication: peer=server-f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c + testing.go:149: server-8e70590a-d0ae-be79-3970-60fa015aafee: 2024-01-08T00:30:58.461Z [INFO] server-8e70590a-d0ae-be79-3970-60fa015aafee: added peer, starting replication: peer=server-f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c +testLoggerAdapter verbose: server-8e70590a-d0ae-be79-3970-60fa015aafee: 2024-01-08T00:30:58.461Z [INFO] server-8e70590a-d0ae-be79-3970-60fa015aafee: pipelining replication: peer="{Voter server-f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c}" + testing.go:149: server-8e70590a-d0ae-be79-3970-60fa015aafee: 2024-01-08T00:30:58.461Z [INFO] server-8e70590a-d0ae-be79-3970-60fa015aafee: pipelining replication: peer="{Voter server-f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c}" +testLoggerAdapter verbose: server-58d4ba3f-7492-7c34-c358-c714c349e806: 2024-01-08T00:30:58.489Z [WARN] server-58d4ba3f-7492-7c34-c358-c714c349e806: no known peers, aborting election + testing.go:149: server-58d4ba3f-7492-7c34-c358-c714c349e806: 2024-01-08T00:30:58.489Z [WARN] server-58d4ba3f-7492-7c34-c358-c714c349e806: no known peers, aborting election +testLoggerAdapter verbose: cluster: 2024-01-08T00:30:58.567Z [INFO] stable state for Leader reached at 2024-01-08 00:30:58.461016792 +0000 UTC m=+4.507905846 (1 nodes), 53.682203ms from start of poll, 54.524866ms from cluster start. Timeout at 2024-01-08 00:30:58.567001104 +0000 UTC m=+4.613890312, 105.984466ms after stability + testing.go:149: cluster: 2024-01-08T00:30:58.567Z [INFO] stable state for Leader reached at 2024-01-08 00:30:58.461016792 +0000 UTC m=+4.507905846 (1 nodes), 53.682203ms from start of poll, 54.524866ms from cluster start. Timeout at 2024-01-08 00:30:58.567001104 +0000 UTC m=+4.613890312, 105.984466ms after stability +testLoggerAdapter verbose: server-8e70590a-d0ae-be79-3970-60fa015aafee: 2024-01-08T00:30:58.567Z [INFO] server-8e70590a-d0ae-be79-3970-60fa015aafee: updating configuration: command=AddStaging server-id=server-58d4ba3f-7492-7c34-c358-c714c349e806 server-addr=58d4ba3f-7492-7c34-c358-c714c349e806 servers="[{Suffrage:Voter ID:server-8e70590a-d0ae-be79-3970-60fa015aafee Address:8e70590a-d0ae-be79-3970-60fa015aafee} {Suffrage:Voter ID:server-f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c Address:f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c} {Suffrage:Voter ID:server-58d4ba3f-7492-7c34-c358-c714c349e806 Address:58d4ba3f-7492-7c34-c358-c714c349e806}]" + testing.go:149: server-8e70590a-d0ae-be79-3970-60fa015aafee: 2024-01-08T00:30:58.567Z [INFO] server-8e70590a-d0ae-be79-3970-60fa015aafee: updating configuration: command=AddStaging server-id=server-58d4ba3f-7492-7c34-c358-c714c349e806 server-addr=58d4ba3f-7492-7c34-c358-c714c349e806 servers="[{Suffrage:Voter ID:server-8e70590a-d0ae-be79-3970-60fa015aafee Address:8e70590a-d0ae-be79-3970-60fa015aafee} {Suffrage:Voter ID:server-f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c Address:f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c} {Suffrage:Voter ID:server-58d4ba3f-7492-7c34-c358-c714c349e806 Address:58d4ba3f-7492-7c34-c358-c714c349e806}]" +testLoggerAdapter verbose: server-8e70590a-d0ae-be79-3970-60fa015aafee: 2024-01-08T00:30:58.567Z [INFO] server-8e70590a-d0ae-be79-3970-60fa015aafee: added peer, starting replication: peer=server-58d4ba3f-7492-7c34-c358-c714c349e806 + testing.go:149: server-8e70590a-d0ae-be79-3970-60fa015aafee: 2024-01-08T00:30:58.567Z [INFO] server-8e70590a-d0ae-be79-3970-60fa015aafee: added peer, starting replication: peer=server-58d4ba3f-7492-7c34-c358-c714c349e806 +testLoggerAdapter verbose: server-58d4ba3f-7492-7c34-c358-c714c349e806: 2024-01-08T00:30:58.567Z [WARN] server-58d4ba3f-7492-7c34-c358-c714c349e806: failed to get previous log: previous-index=3 last-index=0 error="log not found" + testing.go:149: server-58d4ba3f-7492-7c34-c358-c714c349e806: 2024-01-08T00:30:58.567Z [WARN] server-58d4ba3f-7492-7c34-c358-c714c349e806: failed to get previous log: previous-index=3 last-index=0 error="log not found" +testLoggerAdapter verbose: server-8e70590a-d0ae-be79-3970-60fa015aafee: 2024-01-08T00:30:58.567Z [WARN] server-8e70590a-d0ae-be79-3970-60fa015aafee: appendEntries rejected, sending older logs: peer="{Voter server-58d4ba3f-7492-7c34-c358-c714c349e806 58d4ba3f-7492-7c34-c358-c714c349e806}" next=1 + testing.go:149: server-8e70590a-d0ae-be79-3970-60fa015aafee: 2024-01-08T00:30:58.567Z [WARN] server-8e70590a-d0ae-be79-3970-60fa015aafee: appendEntries rejected, sending older logs: peer="{Voter server-58d4ba3f-7492-7c34-c358-c714c349e806 58d4ba3f-7492-7c34-c358-c714c349e806}" next=1 +testLoggerAdapter verbose: server-8e70590a-d0ae-be79-3970-60fa015aafee: 2024-01-08T00:30:58.567Z [INFO] server-8e70590a-d0ae-be79-3970-60fa015aafee: pipelining replication: peer="{Voter server-58d4ba3f-7492-7c34-c358-c714c349e806 58d4ba3f-7492-7c34-c358-c714c349e806}" + testing.go:149: server-8e70590a-d0ae-be79-3970-60fa015aafee: 2024-01-08T00:30:58.567Z [INFO] server-8e70590a-d0ae-be79-3970-60fa015aafee: pipelining replication: peer="{Voter server-58d4ba3f-7492-7c34-c358-c714c349e806 58d4ba3f-7492-7c34-c358-c714c349e806}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:30:58.573Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:30:58.573Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T00:30:58.679Z [INFO] stable state for Leader reached at 2024-01-08 00:30:58.573465699 +0000 UTC m=+4.620354750 (1 nodes), 2.015µs from start of poll, 166.97377ms from cluster start. Timeout at 2024-01-08 00:30:58.678984501 +0000 UTC m=+4.725873640, 105.51889ms after stability + testing.go:149: cluster: 2024-01-08T00:30:58.679Z [INFO] stable state for Leader reached at 2024-01-08 00:30:58.573465699 +0000 UTC m=+4.620354750 (1 nodes), 2.015µs from start of poll, 166.97377ms from cluster start. Timeout at 2024-01-08 00:30:58.678984501 +0000 UTC m=+4.725873640, 105.51889ms after stability +testLoggerAdapter verbose: server-8e70590a-d0ae-be79-3970-60fa015aafee: 2024-01-08T00:30:58.679Z [INFO] server-8e70590a-d0ae-be79-3970-60fa015aafee: aborting pipeline replication: peer="{Voter server-58d4ba3f-7492-7c34-c358-c714c349e806 58d4ba3f-7492-7c34-c358-c714c349e806}" + testing.go:149: server-8e70590a-d0ae-be79-3970-60fa015aafee: 2024-01-08T00:30:58.679Z [INFO] server-8e70590a-d0ae-be79-3970-60fa015aafee: aborting pipeline replication: peer="{Voter server-58d4ba3f-7492-7c34-c358-c714c349e806 58d4ba3f-7492-7c34-c358-c714c349e806}" +testLoggerAdapter verbose: server-8e70590a-d0ae-be79-3970-60fa015aafee: 2024-01-08T00:30:58.679Z [INFO] server-8e70590a-d0ae-be79-3970-60fa015aafee: aborting pipeline replication: peer="{Voter server-f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c}" + testing.go:149: server-8e70590a-d0ae-be79-3970-60fa015aafee: 2024-01-08T00:30:58.679Z [INFO] server-8e70590a-d0ae-be79-3970-60fa015aafee: aborting pipeline replication: peer="{Voter server-f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c f03a1c4e-3b0d-ee16-9c68-5bc28760cd1c}" +--- PASS: TestRaft_HasExistingState (0.27s) === RUN TestRaft_SingleNode -testLoggerAdapter verbose: server-be2792e7-5695-74ef-abcc-17f6cf32c26e: 2025-02-09T06:52:23.727Z [INFO] server-be2792e7-5695-74ef-abcc-17f6cf32c26e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-be2792e7-5695-74ef-abcc-17f6cf32c26e Address:be2792e7-5695-74ef-abcc-17f6cf32c26e}]" - testing.go:149: server-be2792e7-5695-74ef-abcc-17f6cf32c26e: 2025-02-09T06:52:23.727Z [INFO] server-be2792e7-5695-74ef-abcc-17f6cf32c26e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-be2792e7-5695-74ef-abcc-17f6cf32c26e Address:be2792e7-5695-74ef-abcc-17f6cf32c26e}]" -testLoggerAdapter verbose: server-be2792e7-5695-74ef-abcc-17f6cf32c26e: 2025-02-09T06:52:23.728Z [INFO] server-be2792e7-5695-74ef-abcc-17f6cf32c26e: entering follower state: follower="Node at be2792e7-5695-74ef-abcc-17f6cf32c26e [Follower]" leader= - testing.go:149: server-be2792e7-5695-74ef-abcc-17f6cf32c26e: 2025-02-09T06:52:23.728Z [INFO] server-be2792e7-5695-74ef-abcc-17f6cf32c26e: entering follower state: follower="Node at be2792e7-5695-74ef-abcc-17f6cf32c26e [Follower]" leader= -testLoggerAdapter verbose: server-be2792e7-5695-74ef-abcc-17f6cf32c26e: 2025-02-09T06:52:23.818Z [WARN] server-be2792e7-5695-74ef-abcc-17f6cf32c26e: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-be2792e7-5695-74ef-abcc-17f6cf32c26e: 2025-02-09T06:52:23.818Z [WARN] server-be2792e7-5695-74ef-abcc-17f6cf32c26e: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-be2792e7-5695-74ef-abcc-17f6cf32c26e: 2025-02-09T06:52:23.818Z [INFO] server-be2792e7-5695-74ef-abcc-17f6cf32c26e: entering candidate state: node="Node at be2792e7-5695-74ef-abcc-17f6cf32c26e [Candidate]" term=2 - testing.go:149: server-be2792e7-5695-74ef-abcc-17f6cf32c26e: 2025-02-09T06:52:23.818Z [INFO] server-be2792e7-5695-74ef-abcc-17f6cf32c26e: entering candidate state: node="Node at be2792e7-5695-74ef-abcc-17f6cf32c26e [Candidate]" term=2 -testLoggerAdapter verbose: server-be2792e7-5695-74ef-abcc-17f6cf32c26e: 2025-02-09T06:52:23.818Z [INFO] server-be2792e7-5695-74ef-abcc-17f6cf32c26e: election won: tally=1 - testing.go:149: server-be2792e7-5695-74ef-abcc-17f6cf32c26e: 2025-02-09T06:52:23.818Z [INFO] server-be2792e7-5695-74ef-abcc-17f6cf32c26e: election won: tally=1 -testLoggerAdapter verbose: server-be2792e7-5695-74ef-abcc-17f6cf32c26e: 2025-02-09T06:52:23.818Z [INFO] server-be2792e7-5695-74ef-abcc-17f6cf32c26e: entering leader state: leader="Node at be2792e7-5695-74ef-abcc-17f6cf32c26e [Leader]" - testing.go:149: server-be2792e7-5695-74ef-abcc-17f6cf32c26e: 2025-02-09T06:52:23.818Z [INFO] server-be2792e7-5695-74ef-abcc-17f6cf32c26e: entering leader state: leader="Node at be2792e7-5695-74ef-abcc-17f6cf32c26e [Leader]" +testLoggerAdapter verbose: server-75d84fa3-09ba-8ebe-0fa9-ffd50de8294b: 2024-01-08T00:30:58.680Z [INFO] server-75d84fa3-09ba-8ebe-0fa9-ffd50de8294b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-75d84fa3-09ba-8ebe-0fa9-ffd50de8294b Address:75d84fa3-09ba-8ebe-0fa9-ffd50de8294b}]" + testing.go:149: server-75d84fa3-09ba-8ebe-0fa9-ffd50de8294b: 2024-01-08T00:30:58.680Z [INFO] server-75d84fa3-09ba-8ebe-0fa9-ffd50de8294b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-75d84fa3-09ba-8ebe-0fa9-ffd50de8294b Address:75d84fa3-09ba-8ebe-0fa9-ffd50de8294b}]" +testLoggerAdapter verbose: server-75d84fa3-09ba-8ebe-0fa9-ffd50de8294b: 2024-01-08T00:30:58.680Z [INFO] server-75d84fa3-09ba-8ebe-0fa9-ffd50de8294b: entering follower state: follower="Node at 75d84fa3-09ba-8ebe-0fa9-ffd50de8294b [Follower]" leader= + testing.go:149: server-75d84fa3-09ba-8ebe-0fa9-ffd50de8294b: 2024-01-08T00:30:58.680Z [INFO] server-75d84fa3-09ba-8ebe-0fa9-ffd50de8294b: entering follower state: follower="Node at 75d84fa3-09ba-8ebe-0fa9-ffd50de8294b [Follower]" leader= +testLoggerAdapter verbose: server-75d84fa3-09ba-8ebe-0fa9-ffd50de8294b: 2024-01-08T00:30:58.769Z [WARN] server-75d84fa3-09ba-8ebe-0fa9-ffd50de8294b: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-75d84fa3-09ba-8ebe-0fa9-ffd50de8294b: 2024-01-08T00:30:58.769Z [WARN] server-75d84fa3-09ba-8ebe-0fa9-ffd50de8294b: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-75d84fa3-09ba-8ebe-0fa9-ffd50de8294b: 2024-01-08T00:30:58.769Z [INFO] server-75d84fa3-09ba-8ebe-0fa9-ffd50de8294b: entering candidate state: node="Node at 75d84fa3-09ba-8ebe-0fa9-ffd50de8294b [Candidate]" term=2 + testing.go:149: server-75d84fa3-09ba-8ebe-0fa9-ffd50de8294b: 2024-01-08T00:30:58.769Z [INFO] server-75d84fa3-09ba-8ebe-0fa9-ffd50de8294b: entering candidate state: node="Node at 75d84fa3-09ba-8ebe-0fa9-ffd50de8294b [Candidate]" term=2 +testLoggerAdapter verbose: server-75d84fa3-09ba-8ebe-0fa9-ffd50de8294b: 2024-01-08T00:30:58.769Z [INFO] server-75d84fa3-09ba-8ebe-0fa9-ffd50de8294b: election won: tally=1 + testing.go:149: server-75d84fa3-09ba-8ebe-0fa9-ffd50de8294b: 2024-01-08T00:30:58.769Z [INFO] server-75d84fa3-09ba-8ebe-0fa9-ffd50de8294b: election won: tally=1 +testLoggerAdapter verbose: server-75d84fa3-09ba-8ebe-0fa9-ffd50de8294b: 2024-01-08T00:30:58.769Z [INFO] server-75d84fa3-09ba-8ebe-0fa9-ffd50de8294b: entering leader state: leader="Node at 75d84fa3-09ba-8ebe-0fa9-ffd50de8294b [Leader]" + testing.go:149: server-75d84fa3-09ba-8ebe-0fa9-ffd50de8294b: 2024-01-08T00:30:58.769Z [INFO] server-75d84fa3-09ba-8ebe-0fa9-ffd50de8294b: entering leader state: leader="Node at 75d84fa3-09ba-8ebe-0fa9-ffd50de8294b [Leader]" --- PASS: TestRaft_SingleNode (0.09s) === RUN TestRaft_TripleNode -testLoggerAdapter verbose: server-7b2e7087-dcf9-edad-2fab-5170f13283d3: 2025-02-09T06:52:23.819Z [INFO] server-7b2e7087-dcf9-edad-2fab-5170f13283d3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b2e7087-dcf9-edad-2fab-5170f13283d3 Address:7b2e7087-dcf9-edad-2fab-5170f13283d3} {Suffrage:Voter ID:server-c72960bd-5d54-05fa-7da6-ed70508d3e8a Address:c72960bd-5d54-05fa-7da6-ed70508d3e8a} {Suffrage:Voter ID:server-ea541217-9cc8-5c19-a099-dae1392cf9b3 Address:ea541217-9cc8-5c19-a099-dae1392cf9b3}]" - testing.go:149: server-7b2e7087-dcf9-edad-2fab-5170f13283d3: 2025-02-09T06:52:23.819Z [INFO] server-7b2e7087-dcf9-edad-2fab-5170f13283d3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b2e7087-dcf9-edad-2fab-5170f13283d3 Address:7b2e7087-dcf9-edad-2fab-5170f13283d3} {Suffrage:Voter ID:server-c72960bd-5d54-05fa-7da6-ed70508d3e8a Address:c72960bd-5d54-05fa-7da6-ed70508d3e8a} {Suffrage:Voter ID:server-ea541217-9cc8-5c19-a099-dae1392cf9b3 Address:ea541217-9cc8-5c19-a099-dae1392cf9b3}]" -testLoggerAdapter verbose: server-7b2e7087-dcf9-edad-2fab-5170f13283d3: 2025-02-09T06:52:23.819Z [INFO] server-7b2e7087-dcf9-edad-2fab-5170f13283d3: entering follower state: follower="Node at 7b2e7087-dcf9-edad-2fab-5170f13283d3 [Follower]" leader= - testing.go:149: server-7b2e7087-dcf9-edad-2fab-5170f13283d3: 2025-02-09T06:52:23.819Z [INFO] server-7b2e7087-dcf9-edad-2fab-5170f13283d3: entering follower state: follower="Node at 7b2e7087-dcf9-edad-2fab-5170f13283d3 [Follower]" leader= -testLoggerAdapter verbose: server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: 2025-02-09T06:52:23.819Z [INFO] server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b2e7087-dcf9-edad-2fab-5170f13283d3 Address:7b2e7087-dcf9-edad-2fab-5170f13283d3} {Suffrage:Voter ID:server-c72960bd-5d54-05fa-7da6-ed70508d3e8a Address:c72960bd-5d54-05fa-7da6-ed70508d3e8a} {Suffrage:Voter ID:server-ea541217-9cc8-5c19-a099-dae1392cf9b3 Address:ea541217-9cc8-5c19-a099-dae1392cf9b3}]" - testing.go:149: server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: 2025-02-09T06:52:23.819Z [INFO] server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b2e7087-dcf9-edad-2fab-5170f13283d3 Address:7b2e7087-dcf9-edad-2fab-5170f13283d3} {Suffrage:Voter ID:server-c72960bd-5d54-05fa-7da6-ed70508d3e8a Address:c72960bd-5d54-05fa-7da6-ed70508d3e8a} {Suffrage:Voter ID:server-ea541217-9cc8-5c19-a099-dae1392cf9b3 Address:ea541217-9cc8-5c19-a099-dae1392cf9b3}]" -testLoggerAdapter verbose: server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: 2025-02-09T06:52:23.819Z [INFO] server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: entering follower state: follower="Node at c72960bd-5d54-05fa-7da6-ed70508d3e8a [Follower]" leader= - testing.go:149: server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: 2025-02-09T06:52:23.819Z [INFO] server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: entering follower state: follower="Node at c72960bd-5d54-05fa-7da6-ed70508d3e8a [Follower]" leader= -testLoggerAdapter verbose: server-ea541217-9cc8-5c19-a099-dae1392cf9b3: 2025-02-09T06:52:23.819Z [INFO] server-ea541217-9cc8-5c19-a099-dae1392cf9b3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b2e7087-dcf9-edad-2fab-5170f13283d3 Address:7b2e7087-dcf9-edad-2fab-5170f13283d3} {Suffrage:Voter ID:server-c72960bd-5d54-05fa-7da6-ed70508d3e8a Address:c72960bd-5d54-05fa-7da6-ed70508d3e8a} {Suffrage:Voter ID:server-ea541217-9cc8-5c19-a099-dae1392cf9b3 Address:ea541217-9cc8-5c19-a099-dae1392cf9b3}]" - testing.go:149: server-ea541217-9cc8-5c19-a099-dae1392cf9b3: 2025-02-09T06:52:23.819Z [INFO] server-ea541217-9cc8-5c19-a099-dae1392cf9b3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b2e7087-dcf9-edad-2fab-5170f13283d3 Address:7b2e7087-dcf9-edad-2fab-5170f13283d3} {Suffrage:Voter ID:server-c72960bd-5d54-05fa-7da6-ed70508d3e8a Address:c72960bd-5d54-05fa-7da6-ed70508d3e8a} {Suffrage:Voter ID:server-ea541217-9cc8-5c19-a099-dae1392cf9b3 Address:ea541217-9cc8-5c19-a099-dae1392cf9b3}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:23.820Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T06:52:23.820Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-ea541217-9cc8-5c19-a099-dae1392cf9b3: 2025-02-09T06:52:23.820Z [INFO] server-ea541217-9cc8-5c19-a099-dae1392cf9b3: entering follower state: follower="Node at ea541217-9cc8-5c19-a099-dae1392cf9b3 [Follower]" leader= - testing.go:149: server-ea541217-9cc8-5c19-a099-dae1392cf9b3: 2025-02-09T06:52:23.820Z [INFO] server-ea541217-9cc8-5c19-a099-dae1392cf9b3: entering follower state: follower="Node at ea541217-9cc8-5c19-a099-dae1392cf9b3 [Follower]" leader= -testLoggerAdapter verbose: server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: 2025-02-09T06:52:23.871Z [WARN] server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: 2025-02-09T06:52:23.871Z [WARN] server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: 2025-02-09T06:52:23.871Z [INFO] server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: entering candidate state: node="Node at c72960bd-5d54-05fa-7da6-ed70508d3e8a [Candidate]" term=2 - testing.go:149: server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: 2025-02-09T06:52:23.871Z [INFO] server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: entering candidate state: node="Node at c72960bd-5d54-05fa-7da6-ed70508d3e8a [Candidate]" term=2 -testLoggerAdapter verbose: server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: 2025-02-09T06:52:23.871Z [INFO] server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: election won: tally=2 - testing.go:149: server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: 2025-02-09T06:52:23.871Z [INFO] server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: election won: tally=2 -testLoggerAdapter verbose: server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: 2025-02-09T06:52:23.871Z [INFO] server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: entering leader state: leader="Node at c72960bd-5d54-05fa-7da6-ed70508d3e8a [Leader]" - testing.go:149: server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: 2025-02-09T06:52:23.871Z [INFO] server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: entering leader state: leader="Node at c72960bd-5d54-05fa-7da6-ed70508d3e8a [Leader]" -testLoggerAdapter verbose: server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: 2025-02-09T06:52:23.871Z [INFO] server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: added peer, starting replication: peer=server-7b2e7087-dcf9-edad-2fab-5170f13283d3 - testing.go:149: server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: 2025-02-09T06:52:23.871Z [INFO] server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: added peer, starting replication: peer=server-7b2e7087-dcf9-edad-2fab-5170f13283d3 -testLoggerAdapter verbose: server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: 2025-02-09T06:52:23.871Z [INFO] server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: added peer, starting replication: peer=server-ea541217-9cc8-5c19-a099-dae1392cf9b3 - testing.go:149: server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: 2025-02-09T06:52:23.871Z [INFO] server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: added peer, starting replication: peer=server-ea541217-9cc8-5c19-a099-dae1392cf9b3 -testLoggerAdapter verbose: server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: 2025-02-09T06:52:23.871Z [INFO] server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: pipelining replication: peer="{Voter server-ea541217-9cc8-5c19-a099-dae1392cf9b3 ea541217-9cc8-5c19-a099-dae1392cf9b3}" - testing.go:149: server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: 2025-02-09T06:52:23.871Z [INFO] server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: pipelining replication: peer="{Voter server-ea541217-9cc8-5c19-a099-dae1392cf9b3 ea541217-9cc8-5c19-a099-dae1392cf9b3}" -testLoggerAdapter verbose: server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: 2025-02-09T06:52:23.871Z [INFO] server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: pipelining replication: peer="{Voter server-7b2e7087-dcf9-edad-2fab-5170f13283d3 7b2e7087-dcf9-edad-2fab-5170f13283d3}" - testing.go:149: server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: 2025-02-09T06:52:23.871Z [INFO] server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: pipelining replication: peer="{Voter server-7b2e7087-dcf9-edad-2fab-5170f13283d3 7b2e7087-dcf9-edad-2fab-5170f13283d3}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:23.977Z [INFO] stable state for Follower reached at 2025-02-09 06:52:23.871708584 +0000 UTC m=+4.850517347 (2 nodes), 51.622837ms from start of poll, 52.076696ms from cluster start. Timeout at 2025-02-09 06:52:23.977218918 +0000 UTC m=+4.956027687, 105.51034ms after stability - testing.go:149: cluster: 2025-02-09T06:52:23.977Z [INFO] stable state for Follower reached at 2025-02-09 06:52:23.871708584 +0000 UTC m=+4.850517347 (2 nodes), 51.622837ms from start of poll, 52.076696ms from cluster start. Timeout at 2025-02-09 06:52:23.977218918 +0000 UTC m=+4.956027687, 105.51034ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:23.977Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:23.977Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:24.082Z [INFO] stable state for Leader reached at 2025-02-09 06:52:23.977291405 +0000 UTC m=+4.956100170 (1 nodes), 468ns from start of poll, 157.659519ms from cluster start. Timeout at 2025-02-09 06:52:24.08242389 +0000 UTC m=+5.061232675, 105.132505ms after stability - testing.go:149: cluster: 2025-02-09T06:52:24.082Z [INFO] stable state for Leader reached at 2025-02-09 06:52:23.977291405 +0000 UTC m=+4.956100170 (1 nodes), 468ns from start of poll, 157.659519ms from cluster start. Timeout at 2025-02-09 06:52:24.08242389 +0000 UTC m=+5.061232675, 105.132505ms after stability -testLoggerAdapter verbose: server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: 2025-02-09T06:52:24.093Z [INFO] server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: aborting pipeline replication: peer="{Voter server-ea541217-9cc8-5c19-a099-dae1392cf9b3 ea541217-9cc8-5c19-a099-dae1392cf9b3}" - testing.go:149: server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: 2025-02-09T06:52:24.093Z [INFO] server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: aborting pipeline replication: peer="{Voter server-ea541217-9cc8-5c19-a099-dae1392cf9b3 ea541217-9cc8-5c19-a099-dae1392cf9b3}" -testLoggerAdapter verbose: server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: 2025-02-09T06:52:24.093Z [INFO] server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: aborting pipeline replication: peer="{Voter server-7b2e7087-dcf9-edad-2fab-5170f13283d3 7b2e7087-dcf9-edad-2fab-5170f13283d3}" - testing.go:149: server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: 2025-02-09T06:52:24.093Z [INFO] server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: aborting pipeline replication: peer="{Voter server-7b2e7087-dcf9-edad-2fab-5170f13283d3 7b2e7087-dcf9-edad-2fab-5170f13283d3}" -testLoggerAdapter verbose: server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: 2025-02-09T06:52:24.143Z [ERROR] server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: failed to heartbeat to: peer=ea541217-9cc8-5c19-a099-dae1392cf9b3 error="command timed out" - testing.go:149: server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: 2025-02-09T06:52:24.143Z [ERROR] server-c72960bd-5d54-05fa-7da6-ed70508d3e8a: failed to heartbeat to: peer=ea541217-9cc8-5c19-a099-dae1392cf9b3 error="command timed out" ---- PASS: TestRaft_TripleNode (0.33s) +testLoggerAdapter verbose: server-27979bdb-2e77-d2c5-e95a-fcc3441736e7: 2024-01-08T00:30:58.770Z [INFO] server-27979bdb-2e77-d2c5-e95a-fcc3441736e7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-27979bdb-2e77-d2c5-e95a-fcc3441736e7 Address:27979bdb-2e77-d2c5-e95a-fcc3441736e7} {Suffrage:Voter ID:server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28 Address:cb8bea42-e1e6-d359-7b3f-f3bde7859a28} {Suffrage:Voter ID:server-183056d4-fff1-d1ea-0306-8d317b8fc38d Address:183056d4-fff1-d1ea-0306-8d317b8fc38d}]" + testing.go:149: server-27979bdb-2e77-d2c5-e95a-fcc3441736e7: 2024-01-08T00:30:58.770Z [INFO] server-27979bdb-2e77-d2c5-e95a-fcc3441736e7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-27979bdb-2e77-d2c5-e95a-fcc3441736e7 Address:27979bdb-2e77-d2c5-e95a-fcc3441736e7} {Suffrage:Voter ID:server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28 Address:cb8bea42-e1e6-d359-7b3f-f3bde7859a28} {Suffrage:Voter ID:server-183056d4-fff1-d1ea-0306-8d317b8fc38d Address:183056d4-fff1-d1ea-0306-8d317b8fc38d}]" +testLoggerAdapter verbose: server-27979bdb-2e77-d2c5-e95a-fcc3441736e7: 2024-01-08T00:30:58.770Z [INFO] server-27979bdb-2e77-d2c5-e95a-fcc3441736e7: entering follower state: follower="Node at 27979bdb-2e77-d2c5-e95a-fcc3441736e7 [Follower]" leader= + testing.go:149: server-27979bdb-2e77-d2c5-e95a-fcc3441736e7: 2024-01-08T00:30:58.770Z [INFO] server-27979bdb-2e77-d2c5-e95a-fcc3441736e7: entering follower state: follower="Node at 27979bdb-2e77-d2c5-e95a-fcc3441736e7 [Follower]" leader= +testLoggerAdapter verbose: server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28: 2024-01-08T00:30:58.770Z [INFO] server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-27979bdb-2e77-d2c5-e95a-fcc3441736e7 Address:27979bdb-2e77-d2c5-e95a-fcc3441736e7} {Suffrage:Voter ID:server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28 Address:cb8bea42-e1e6-d359-7b3f-f3bde7859a28} {Suffrage:Voter ID:server-183056d4-fff1-d1ea-0306-8d317b8fc38d Address:183056d4-fff1-d1ea-0306-8d317b8fc38d}]" + testing.go:149: server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28: 2024-01-08T00:30:58.770Z [INFO] server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-27979bdb-2e77-d2c5-e95a-fcc3441736e7 Address:27979bdb-2e77-d2c5-e95a-fcc3441736e7} {Suffrage:Voter ID:server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28 Address:cb8bea42-e1e6-d359-7b3f-f3bde7859a28} {Suffrage:Voter ID:server-183056d4-fff1-d1ea-0306-8d317b8fc38d Address:183056d4-fff1-d1ea-0306-8d317b8fc38d}]" +testLoggerAdapter verbose: server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28: 2024-01-08T00:30:58.771Z [INFO] server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28: entering follower state: follower="Node at cb8bea42-e1e6-d359-7b3f-f3bde7859a28 [Follower]" leader= + testing.go:149: server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28: 2024-01-08T00:30:58.771Z [INFO] server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28: entering follower state: follower="Node at cb8bea42-e1e6-d359-7b3f-f3bde7859a28 [Follower]" leader= +testLoggerAdapter verbose: server-183056d4-fff1-d1ea-0306-8d317b8fc38d: 2024-01-08T00:30:58.771Z [INFO] server-183056d4-fff1-d1ea-0306-8d317b8fc38d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-27979bdb-2e77-d2c5-e95a-fcc3441736e7 Address:27979bdb-2e77-d2c5-e95a-fcc3441736e7} {Suffrage:Voter ID:server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28 Address:cb8bea42-e1e6-d359-7b3f-f3bde7859a28} {Suffrage:Voter ID:server-183056d4-fff1-d1ea-0306-8d317b8fc38d Address:183056d4-fff1-d1ea-0306-8d317b8fc38d}]" + testing.go:149: server-183056d4-fff1-d1ea-0306-8d317b8fc38d: 2024-01-08T00:30:58.771Z [INFO] server-183056d4-fff1-d1ea-0306-8d317b8fc38d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-27979bdb-2e77-d2c5-e95a-fcc3441736e7 Address:27979bdb-2e77-d2c5-e95a-fcc3441736e7} {Suffrage:Voter ID:server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28 Address:cb8bea42-e1e6-d359-7b3f-f3bde7859a28} {Suffrage:Voter ID:server-183056d4-fff1-d1ea-0306-8d317b8fc38d Address:183056d4-fff1-d1ea-0306-8d317b8fc38d}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:30:58.771Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T00:30:58.771Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: server-183056d4-fff1-d1ea-0306-8d317b8fc38d: 2024-01-08T00:30:58.771Z [INFO] server-183056d4-fff1-d1ea-0306-8d317b8fc38d: entering follower state: follower="Node at 183056d4-fff1-d1ea-0306-8d317b8fc38d [Follower]" leader= + testing.go:149: server-183056d4-fff1-d1ea-0306-8d317b8fc38d: 2024-01-08T00:30:58.771Z [INFO] server-183056d4-fff1-d1ea-0306-8d317b8fc38d: entering follower state: follower="Node at 183056d4-fff1-d1ea-0306-8d317b8fc38d [Follower]" leader= +testLoggerAdapter verbose: server-183056d4-fff1-d1ea-0306-8d317b8fc38d: 2024-01-08T00:30:58.839Z [WARN] server-183056d4-fff1-d1ea-0306-8d317b8fc38d: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-183056d4-fff1-d1ea-0306-8d317b8fc38d: 2024-01-08T00:30:58.839Z [WARN] server-183056d4-fff1-d1ea-0306-8d317b8fc38d: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-183056d4-fff1-d1ea-0306-8d317b8fc38d: 2024-01-08T00:30:58.839Z [INFO] server-183056d4-fff1-d1ea-0306-8d317b8fc38d: entering candidate state: node="Node at 183056d4-fff1-d1ea-0306-8d317b8fc38d [Candidate]" term=2 + testing.go:149: server-183056d4-fff1-d1ea-0306-8d317b8fc38d: 2024-01-08T00:30:58.839Z [INFO] server-183056d4-fff1-d1ea-0306-8d317b8fc38d: entering candidate state: node="Node at 183056d4-fff1-d1ea-0306-8d317b8fc38d [Candidate]" term=2 +testLoggerAdapter verbose: server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28: 2024-01-08T00:30:58.839Z [WARN] server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28: 2024-01-08T00:30:58.839Z [WARN] server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28: 2024-01-08T00:30:58.839Z [INFO] server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28: entering candidate state: node="Node at cb8bea42-e1e6-d359-7b3f-f3bde7859a28 [Candidate]" term=2 + testing.go:149: server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28: 2024-01-08T00:30:58.839Z [INFO] server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28: entering candidate state: node="Node at cb8bea42-e1e6-d359-7b3f-f3bde7859a28 [Candidate]" term=2 +testLoggerAdapter verbose: server-183056d4-fff1-d1ea-0306-8d317b8fc38d: 2024-01-08T00:30:58.839Z [INFO] server-183056d4-fff1-d1ea-0306-8d317b8fc38d: election won: tally=2 + testing.go:149: server-183056d4-fff1-d1ea-0306-8d317b8fc38d: 2024-01-08T00:30:58.839Z [INFO] server-183056d4-fff1-d1ea-0306-8d317b8fc38d: election won: tally=2 +testLoggerAdapter verbose: server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28: 2024-01-08T00:30:58.839Z [INFO] server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28: duplicate requestVote for same term: term=2 +testLoggerAdapter verbose: server-183056d4-fff1-d1ea-0306-8d317b8fc38d: 2024-01-08T00:30:58.839Z [INFO] server-183056d4-fff1-d1ea-0306-8d317b8fc38d: entering leader state: leader="Node at 183056d4-fff1-d1ea-0306-8d317b8fc38d [Leader]" + testing.go:149: server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28: 2024-01-08T00:30:58.839Z [INFO] server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28: duplicate requestVote for same term: term=2 + testing.go:149: server-183056d4-fff1-d1ea-0306-8d317b8fc38d: 2024-01-08T00:30:58.839Z [INFO] server-183056d4-fff1-d1ea-0306-8d317b8fc38d: entering leader state: leader="Node at 183056d4-fff1-d1ea-0306-8d317b8fc38d [Leader]" +testLoggerAdapter verbose: server-27979bdb-2e77-d2c5-e95a-fcc3441736e7: 2024-01-08T00:30:58.839Z [INFO] server-27979bdb-2e77-d2c5-e95a-fcc3441736e7: duplicate requestVote for same term: term=2 +testLoggerAdapter verbose: server-183056d4-fff1-d1ea-0306-8d317b8fc38d: 2024-01-08T00:30:58.839Z [INFO] server-183056d4-fff1-d1ea-0306-8d317b8fc38d: added peer, starting replication: peer=server-27979bdb-2e77-d2c5-e95a-fcc3441736e7 + testing.go:149: server-27979bdb-2e77-d2c5-e95a-fcc3441736e7: 2024-01-08T00:30:58.839Z [INFO] server-27979bdb-2e77-d2c5-e95a-fcc3441736e7: duplicate requestVote for same term: term=2 + testing.go:149: server-183056d4-fff1-d1ea-0306-8d317b8fc38d: 2024-01-08T00:30:58.839Z [INFO] server-183056d4-fff1-d1ea-0306-8d317b8fc38d: added peer, starting replication: peer=server-27979bdb-2e77-d2c5-e95a-fcc3441736e7 +testLoggerAdapter verbose: server-183056d4-fff1-d1ea-0306-8d317b8fc38d: 2024-01-08T00:30:58.839Z [INFO] server-183056d4-fff1-d1ea-0306-8d317b8fc38d: added peer, starting replication: peer=server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28 + testing.go:149: server-183056d4-fff1-d1ea-0306-8d317b8fc38d: 2024-01-08T00:30:58.839Z [INFO] server-183056d4-fff1-d1ea-0306-8d317b8fc38d: added peer, starting replication: peer=server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28 +testLoggerAdapter verbose: server-183056d4-fff1-d1ea-0306-8d317b8fc38d: 2024-01-08T00:30:58.839Z [WARN] server-183056d4-fff1-d1ea-0306-8d317b8fc38d: rejecting vote request since we have a leader: from=cb8bea42-e1e6-d359-7b3f-f3bde7859a28 leader=183056d4-fff1-d1ea-0306-8d317b8fc38d + testing.go:149: server-183056d4-fff1-d1ea-0306-8d317b8fc38d: 2024-01-08T00:30:58.839Z [WARN] server-183056d4-fff1-d1ea-0306-8d317b8fc38d: rejecting vote request since we have a leader: from=cb8bea42-e1e6-d359-7b3f-f3bde7859a28 leader=183056d4-fff1-d1ea-0306-8d317b8fc38d +testLoggerAdapter verbose: server-183056d4-fff1-d1ea-0306-8d317b8fc38d: 2024-01-08T00:30:58.839Z [INFO] server-183056d4-fff1-d1ea-0306-8d317b8fc38d: pipelining replication: peer="{Voter server-27979bdb-2e77-d2c5-e95a-fcc3441736e7 27979bdb-2e77-d2c5-e95a-fcc3441736e7}" +testLoggerAdapter verbose: server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28: 2024-01-08T00:30:58.839Z [INFO] server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28: entering follower state: follower="Node at cb8bea42-e1e6-d359-7b3f-f3bde7859a28 [Follower]" leader=183056d4-fff1-d1ea-0306-8d317b8fc38d + testing.go:149: server-183056d4-fff1-d1ea-0306-8d317b8fc38d: 2024-01-08T00:30:58.839Z [INFO] server-183056d4-fff1-d1ea-0306-8d317b8fc38d: pipelining replication: peer="{Voter server-27979bdb-2e77-d2c5-e95a-fcc3441736e7 27979bdb-2e77-d2c5-e95a-fcc3441736e7}" +testLoggerAdapter verbose: server-183056d4-fff1-d1ea-0306-8d317b8fc38d: 2024-01-08T00:30:58.839Z [INFO] server-183056d4-fff1-d1ea-0306-8d317b8fc38d: pipelining replication: peer="{Voter server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28 cb8bea42-e1e6-d359-7b3f-f3bde7859a28}" + testing.go:149: server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28: 2024-01-08T00:30:58.839Z [INFO] server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28: entering follower state: follower="Node at cb8bea42-e1e6-d359-7b3f-f3bde7859a28 [Follower]" leader=183056d4-fff1-d1ea-0306-8d317b8fc38d + testing.go:149: server-183056d4-fff1-d1ea-0306-8d317b8fc38d: 2024-01-08T00:30:58.839Z [INFO] server-183056d4-fff1-d1ea-0306-8d317b8fc38d: pipelining replication: peer="{Voter server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28 cb8bea42-e1e6-d359-7b3f-f3bde7859a28}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:30:58.945Z [INFO] stable state for Follower reached at 2024-01-08 00:30:58.839812924 +0000 UTC m=+4.886702045 (2 nodes), 68.471664ms from start of poll, 69.285934ms from cluster start. Timeout at 2024-01-08 00:30:58.944968828 +0000 UTC m=+4.991857920, 105.155875ms after stability + testing.go:149: cluster: 2024-01-08T00:30:58.945Z [INFO] stable state for Follower reached at 2024-01-08 00:30:58.839812924 +0000 UTC m=+4.886702045 (2 nodes), 68.471664ms from start of poll, 69.285934ms from cluster start. Timeout at 2024-01-08 00:30:58.944968828 +0000 UTC m=+4.991857920, 105.155875ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T00:30:58.945Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:30:58.945Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T00:30:59.050Z [INFO] stable state for Leader reached at 2024-01-08 00:30:58.945100214 +0000 UTC m=+4.991989249 (1 nodes), 2.209µs from start of poll, 174.573138ms from cluster start. Timeout at 2024-01-08 00:30:59.050841434 +0000 UTC m=+5.097730509, 105.74126ms after stability + testing.go:149: cluster: 2024-01-08T00:30:59.050Z [INFO] stable state for Leader reached at 2024-01-08 00:30:58.945100214 +0000 UTC m=+4.991989249 (1 nodes), 2.209µs from start of poll, 174.573138ms from cluster start. Timeout at 2024-01-08 00:30:59.050841434 +0000 UTC m=+5.097730509, 105.74126ms after stability +testLoggerAdapter verbose: server-183056d4-fff1-d1ea-0306-8d317b8fc38d: 2024-01-08T00:30:59.062Z [INFO] server-183056d4-fff1-d1ea-0306-8d317b8fc38d: aborting pipeline replication: peer="{Voter server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28 cb8bea42-e1e6-d359-7b3f-f3bde7859a28}" + testing.go:149: server-183056d4-fff1-d1ea-0306-8d317b8fc38d: 2024-01-08T00:30:59.062Z [INFO] server-183056d4-fff1-d1ea-0306-8d317b8fc38d: aborting pipeline replication: peer="{Voter server-cb8bea42-e1e6-d359-7b3f-f3bde7859a28 cb8bea42-e1e6-d359-7b3f-f3bde7859a28}" +testLoggerAdapter verbose: server-183056d4-fff1-d1ea-0306-8d317b8fc38d: 2024-01-08T00:30:59.062Z [INFO] server-183056d4-fff1-d1ea-0306-8d317b8fc38d: aborting pipeline replication: peer="{Voter server-27979bdb-2e77-d2c5-e95a-fcc3441736e7 27979bdb-2e77-d2c5-e95a-fcc3441736e7}" + testing.go:149: server-183056d4-fff1-d1ea-0306-8d317b8fc38d: 2024-01-08T00:30:59.062Z [INFO] server-183056d4-fff1-d1ea-0306-8d317b8fc38d: aborting pipeline replication: peer="{Voter server-27979bdb-2e77-d2c5-e95a-fcc3441736e7 27979bdb-2e77-d2c5-e95a-fcc3441736e7}" +testLoggerAdapter verbose: server-183056d4-fff1-d1ea-0306-8d317b8fc38d: 2024-01-08T00:30:59.112Z [ERROR] server-183056d4-fff1-d1ea-0306-8d317b8fc38d: failed to heartbeat to: peer=27979bdb-2e77-d2c5-e95a-fcc3441736e7 error="command timed out" + testing.go:149: server-183056d4-fff1-d1ea-0306-8d317b8fc38d: 2024-01-08T00:30:59.112Z [ERROR] server-183056d4-fff1-d1ea-0306-8d317b8fc38d: failed to heartbeat to: peer=27979bdb-2e77-d2c5-e95a-fcc3441736e7 error="command timed out" +--- PASS: TestRaft_TripleNode (0.36s) === RUN TestRaft_LeaderFail -testLoggerAdapter verbose: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.144Z [INFO] server-9b3a9873-81f7-9223-b14f-d303c904b789: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9b3a9873-81f7-9223-b14f-d303c904b789 Address:9b3a9873-81f7-9223-b14f-d303c904b789} {Suffrage:Voter ID:server-054499dc-6587-90bb-231e-873c097815a2 Address:054499dc-6587-90bb-231e-873c097815a2} {Suffrage:Voter ID:server-b25a8bce-8672-f03b-e815-2af799dd3a0e Address:b25a8bce-8672-f03b-e815-2af799dd3a0e}]" - testing.go:149: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.144Z [INFO] server-9b3a9873-81f7-9223-b14f-d303c904b789: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9b3a9873-81f7-9223-b14f-d303c904b789 Address:9b3a9873-81f7-9223-b14f-d303c904b789} {Suffrage:Voter ID:server-054499dc-6587-90bb-231e-873c097815a2 Address:054499dc-6587-90bb-231e-873c097815a2} {Suffrage:Voter ID:server-b25a8bce-8672-f03b-e815-2af799dd3a0e Address:b25a8bce-8672-f03b-e815-2af799dd3a0e}]" -testLoggerAdapter verbose: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.145Z [INFO] server-9b3a9873-81f7-9223-b14f-d303c904b789: entering follower state: follower="Node at 9b3a9873-81f7-9223-b14f-d303c904b789 [Follower]" leader= -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.145Z [INFO] server-054499dc-6587-90bb-231e-873c097815a2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9b3a9873-81f7-9223-b14f-d303c904b789 Address:9b3a9873-81f7-9223-b14f-d303c904b789} {Suffrage:Voter ID:server-054499dc-6587-90bb-231e-873c097815a2 Address:054499dc-6587-90bb-231e-873c097815a2} {Suffrage:Voter ID:server-b25a8bce-8672-f03b-e815-2af799dd3a0e Address:b25a8bce-8672-f03b-e815-2af799dd3a0e}]" - testing.go:149: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.145Z [INFO] server-9b3a9873-81f7-9223-b14f-d303c904b789: entering follower state: follower="Node at 9b3a9873-81f7-9223-b14f-d303c904b789 [Follower]" leader= - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.145Z [INFO] server-054499dc-6587-90bb-231e-873c097815a2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9b3a9873-81f7-9223-b14f-d303c904b789 Address:9b3a9873-81f7-9223-b14f-d303c904b789} {Suffrage:Voter ID:server-054499dc-6587-90bb-231e-873c097815a2 Address:054499dc-6587-90bb-231e-873c097815a2} {Suffrage:Voter ID:server-b25a8bce-8672-f03b-e815-2af799dd3a0e Address:b25a8bce-8672-f03b-e815-2af799dd3a0e}]" -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.145Z [INFO] server-054499dc-6587-90bb-231e-873c097815a2: entering follower state: follower="Node at 054499dc-6587-90bb-231e-873c097815a2 [Follower]" leader= - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.145Z [INFO] server-054499dc-6587-90bb-231e-873c097815a2: entering follower state: follower="Node at 054499dc-6587-90bb-231e-873c097815a2 [Follower]" leader= -testLoggerAdapter verbose: server-b25a8bce-8672-f03b-e815-2af799dd3a0e: 2025-02-09T06:52:24.145Z [INFO] server-b25a8bce-8672-f03b-e815-2af799dd3a0e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9b3a9873-81f7-9223-b14f-d303c904b789 Address:9b3a9873-81f7-9223-b14f-d303c904b789} {Suffrage:Voter ID:server-054499dc-6587-90bb-231e-873c097815a2 Address:054499dc-6587-90bb-231e-873c097815a2} {Suffrage:Voter ID:server-b25a8bce-8672-f03b-e815-2af799dd3a0e Address:b25a8bce-8672-f03b-e815-2af799dd3a0e}]" - testing.go:149: server-b25a8bce-8672-f03b-e815-2af799dd3a0e: 2025-02-09T06:52:24.145Z [INFO] server-b25a8bce-8672-f03b-e815-2af799dd3a0e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9b3a9873-81f7-9223-b14f-d303c904b789 Address:9b3a9873-81f7-9223-b14f-d303c904b789} {Suffrage:Voter ID:server-054499dc-6587-90bb-231e-873c097815a2 Address:054499dc-6587-90bb-231e-873c097815a2} {Suffrage:Voter ID:server-b25a8bce-8672-f03b-e815-2af799dd3a0e Address:b25a8bce-8672-f03b-e815-2af799dd3a0e}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:24.145Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T06:52:24.145Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-b25a8bce-8672-f03b-e815-2af799dd3a0e: 2025-02-09T06:52:24.145Z [INFO] server-b25a8bce-8672-f03b-e815-2af799dd3a0e: entering follower state: follower="Node at b25a8bce-8672-f03b-e815-2af799dd3a0e [Follower]" leader= - testing.go:149: server-b25a8bce-8672-f03b-e815-2af799dd3a0e: 2025-02-09T06:52:24.145Z [INFO] server-b25a8bce-8672-f03b-e815-2af799dd3a0e: entering follower state: follower="Node at b25a8bce-8672-f03b-e815-2af799dd3a0e [Follower]" leader= -testLoggerAdapter verbose: server-b25a8bce-8672-f03b-e815-2af799dd3a0e: 2025-02-09T06:52:24.210Z [WARN] server-b25a8bce-8672-f03b-e815-2af799dd3a0e: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.210Z [WARN] server-054499dc-6587-90bb-231e-873c097815a2: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-b25a8bce-8672-f03b-e815-2af799dd3a0e: 2025-02-09T06:52:24.210Z [WARN] server-b25a8bce-8672-f03b-e815-2af799dd3a0e: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-b25a8bce-8672-f03b-e815-2af799dd3a0e: 2025-02-09T06:52:24.210Z [INFO] server-b25a8bce-8672-f03b-e815-2af799dd3a0e: entering candidate state: node="Node at b25a8bce-8672-f03b-e815-2af799dd3a0e [Candidate]" term=2 - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.210Z [WARN] server-054499dc-6587-90bb-231e-873c097815a2: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.210Z [INFO] server-054499dc-6587-90bb-231e-873c097815a2: entering candidate state: node="Node at 054499dc-6587-90bb-231e-873c097815a2 [Candidate]" term=2 - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.210Z [INFO] server-054499dc-6587-90bb-231e-873c097815a2: entering candidate state: node="Node at 054499dc-6587-90bb-231e-873c097815a2 [Candidate]" term=2 - testing.go:149: server-b25a8bce-8672-f03b-e815-2af799dd3a0e: 2025-02-09T06:52:24.210Z [INFO] server-b25a8bce-8672-f03b-e815-2af799dd3a0e: entering candidate state: node="Node at b25a8bce-8672-f03b-e815-2af799dd3a0e [Candidate]" term=2 -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.210Z [INFO] server-054499dc-6587-90bb-231e-873c097815a2: election won: tally=2 - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.210Z [INFO] server-054499dc-6587-90bb-231e-873c097815a2: election won: tally=2 -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.210Z [INFO] server-054499dc-6587-90bb-231e-873c097815a2: entering leader state: leader="Node at 054499dc-6587-90bb-231e-873c097815a2 [Leader]" - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.210Z [INFO] server-054499dc-6587-90bb-231e-873c097815a2: entering leader state: leader="Node at 054499dc-6587-90bb-231e-873c097815a2 [Leader]" -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.210Z [INFO] server-054499dc-6587-90bb-231e-873c097815a2: added peer, starting replication: peer=server-9b3a9873-81f7-9223-b14f-d303c904b789 - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.210Z [INFO] server-054499dc-6587-90bb-231e-873c097815a2: added peer, starting replication: peer=server-9b3a9873-81f7-9223-b14f-d303c904b789 -testLoggerAdapter verbose: server-b25a8bce-8672-f03b-e815-2af799dd3a0e: 2025-02-09T06:52:24.210Z [INFO] server-b25a8bce-8672-f03b-e815-2af799dd3a0e: duplicate requestVote for same term: term=2 -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.210Z [INFO] server-054499dc-6587-90bb-231e-873c097815a2: added peer, starting replication: peer=server-b25a8bce-8672-f03b-e815-2af799dd3a0e - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.210Z [INFO] server-054499dc-6587-90bb-231e-873c097815a2: added peer, starting replication: peer=server-b25a8bce-8672-f03b-e815-2af799dd3a0e -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.210Z [WARN] server-054499dc-6587-90bb-231e-873c097815a2: rejecting vote request since we have a leader: from=b25a8bce-8672-f03b-e815-2af799dd3a0e leader=054499dc-6587-90bb-231e-873c097815a2 -testLoggerAdapter verbose: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.210Z [INFO] server-9b3a9873-81f7-9223-b14f-d303c904b789: duplicate requestVote for same term: term=2 - testing.go:149: server-b25a8bce-8672-f03b-e815-2af799dd3a0e: 2025-02-09T06:52:24.210Z [INFO] server-b25a8bce-8672-f03b-e815-2af799dd3a0e: duplicate requestVote for same term: term=2 - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.210Z [WARN] server-054499dc-6587-90bb-231e-873c097815a2: rejecting vote request since we have a leader: from=b25a8bce-8672-f03b-e815-2af799dd3a0e leader=054499dc-6587-90bb-231e-873c097815a2 -testLoggerAdapter verbose: server-b25a8bce-8672-f03b-e815-2af799dd3a0e: 2025-02-09T06:52:24.210Z [INFO] server-b25a8bce-8672-f03b-e815-2af799dd3a0e: entering follower state: follower="Node at b25a8bce-8672-f03b-e815-2af799dd3a0e [Follower]" leader=054499dc-6587-90bb-231e-873c097815a2 - testing.go:149: server-b25a8bce-8672-f03b-e815-2af799dd3a0e: 2025-02-09T06:52:24.210Z [INFO] server-b25a8bce-8672-f03b-e815-2af799dd3a0e: entering follower state: follower="Node at b25a8bce-8672-f03b-e815-2af799dd3a0e [Follower]" leader=054499dc-6587-90bb-231e-873c097815a2 -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.210Z [INFO] server-054499dc-6587-90bb-231e-873c097815a2: pipelining replication: peer="{Voter server-b25a8bce-8672-f03b-e815-2af799dd3a0e b25a8bce-8672-f03b-e815-2af799dd3a0e}" - testing.go:149: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.210Z [INFO] server-9b3a9873-81f7-9223-b14f-d303c904b789: duplicate requestVote for same term: term=2 - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.210Z [INFO] server-054499dc-6587-90bb-231e-873c097815a2: pipelining replication: peer="{Voter server-b25a8bce-8672-f03b-e815-2af799dd3a0e b25a8bce-8672-f03b-e815-2af799dd3a0e}" -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.210Z [INFO] server-054499dc-6587-90bb-231e-873c097815a2: pipelining replication: peer="{Voter server-9b3a9873-81f7-9223-b14f-d303c904b789 9b3a9873-81f7-9223-b14f-d303c904b789}" - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.210Z [INFO] server-054499dc-6587-90bb-231e-873c097815a2: pipelining replication: peer="{Voter server-9b3a9873-81f7-9223-b14f-d303c904b789 9b3a9873-81f7-9223-b14f-d303c904b789}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:24.316Z [INFO] stable state for Follower reached at 2025-02-09 06:52:24.210848601 +0000 UTC m=+5.189657392 (2 nodes), 65.556986ms from start of poll, 66.013584ms from cluster start. Timeout at 2025-02-09 06:52:24.316627114 +0000 UTC m=+5.295435881, 105.778489ms after stability - testing.go:149: cluster: 2025-02-09T06:52:24.316Z [INFO] stable state for Follower reached at 2025-02-09 06:52:24.210848601 +0000 UTC m=+5.189657392 (2 nodes), 65.556986ms from start of poll, 66.013584ms from cluster start. Timeout at 2025-02-09 06:52:24.316627114 +0000 UTC m=+5.295435881, 105.778489ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:24.316Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:24.316Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:24.422Z [INFO] stable state for Leader reached at 2025-02-09 06:52:24.316751283 +0000 UTC m=+5.295560047 (1 nodes), 763ns from start of poll, 171.916239ms from cluster start. Timeout at 2025-02-09 06:52:24.422751964 +0000 UTC m=+5.401560722, 106.000675ms after stability - testing.go:149: cluster: 2025-02-09T06:52:24.422Z [INFO] stable state for Leader reached at 2025-02-09 06:52:24.316751283 +0000 UTC m=+5.295560047 (1 nodes), 763ns from start of poll, 171.916239ms from cluster start. Timeout at 2025-02-09 06:52:24.422751964 +0000 UTC m=+5.401560722, 106.000675ms after stability - raft_test.go:336: [INFO] Disconnecting Node at 054499dc-6587-90bb-231e-873c097815a2 [Leader] -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.434Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to heartbeat to: peer=9b3a9873-81f7-9223-b14f-d303c904b789 error="failed to connect to peer: 9b3a9873-81f7-9223-b14f-d303c904b789" - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.434Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to heartbeat to: peer=9b3a9873-81f7-9223-b14f-d303c904b789 error="failed to connect to peer: 9b3a9873-81f7-9223-b14f-d303c904b789" -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.434Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to pipeline appendEntries: peer="{Voter server-9b3a9873-81f7-9223-b14f-d303c904b789 9b3a9873-81f7-9223-b14f-d303c904b789}" error="append pipeline closed" - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.434Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to pipeline appendEntries: peer="{Voter server-9b3a9873-81f7-9223-b14f-d303c904b789 9b3a9873-81f7-9223-b14f-d303c904b789}" error="append pipeline closed" -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.434Z [INFO] server-054499dc-6587-90bb-231e-873c097815a2: aborting pipeline replication: peer="{Voter server-9b3a9873-81f7-9223-b14f-d303c904b789 9b3a9873-81f7-9223-b14f-d303c904b789}" - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.434Z [INFO] server-054499dc-6587-90bb-231e-873c097815a2: aborting pipeline replication: peer="{Voter server-9b3a9873-81f7-9223-b14f-d303c904b789 9b3a9873-81f7-9223-b14f-d303c904b789}" -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.437Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to pipeline appendEntries: peer="{Voter server-b25a8bce-8672-f03b-e815-2af799dd3a0e b25a8bce-8672-f03b-e815-2af799dd3a0e}" error="append pipeline closed" - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.437Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to pipeline appendEntries: peer="{Voter server-b25a8bce-8672-f03b-e815-2af799dd3a0e b25a8bce-8672-f03b-e815-2af799dd3a0e}" error="append pipeline closed" -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.437Z [INFO] server-054499dc-6587-90bb-231e-873c097815a2: aborting pipeline replication: peer="{Voter server-b25a8bce-8672-f03b-e815-2af799dd3a0e b25a8bce-8672-f03b-e815-2af799dd3a0e}" - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.437Z [INFO] server-054499dc-6587-90bb-231e-873c097815a2: aborting pipeline replication: peer="{Voter server-b25a8bce-8672-f03b-e815-2af799dd3a0e b25a8bce-8672-f03b-e815-2af799dd3a0e}" -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.438Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to heartbeat to: peer=b25a8bce-8672-f03b-e815-2af799dd3a0e error="failed to connect to peer: b25a8bce-8672-f03b-e815-2af799dd3a0e" - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.438Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to heartbeat to: peer=b25a8bce-8672-f03b-e815-2af799dd3a0e error="failed to connect to peer: b25a8bce-8672-f03b-e815-2af799dd3a0e" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:24.439Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:24.439Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.443Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to appendEntries to: peer="{Voter server-b25a8bce-8672-f03b-e815-2af799dd3a0e b25a8bce-8672-f03b-e815-2af799dd3a0e}" error="failed to connect to peer: b25a8bce-8672-f03b-e815-2af799dd3a0e" - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.443Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to appendEntries to: peer="{Voter server-b25a8bce-8672-f03b-e815-2af799dd3a0e b25a8bce-8672-f03b-e815-2af799dd3a0e}" error="failed to connect to peer: b25a8bce-8672-f03b-e815-2af799dd3a0e" -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.443Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to appendEntries to: peer="{Voter server-9b3a9873-81f7-9223-b14f-d303c904b789 9b3a9873-81f7-9223-b14f-d303c904b789}" error="failed to connect to peer: 9b3a9873-81f7-9223-b14f-d303c904b789" - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.443Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to appendEntries to: peer="{Voter server-9b3a9873-81f7-9223-b14f-d303c904b789 9b3a9873-81f7-9223-b14f-d303c904b789}" error="failed to connect to peer: 9b3a9873-81f7-9223-b14f-d303c904b789" -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.451Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to heartbeat to: peer=9b3a9873-81f7-9223-b14f-d303c904b789 error="failed to connect to peer: 9b3a9873-81f7-9223-b14f-d303c904b789" - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.451Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to heartbeat to: peer=9b3a9873-81f7-9223-b14f-d303c904b789 error="failed to connect to peer: 9b3a9873-81f7-9223-b14f-d303c904b789" -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.457Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to heartbeat to: peer=b25a8bce-8672-f03b-e815-2af799dd3a0e error="failed to connect to peer: b25a8bce-8672-f03b-e815-2af799dd3a0e" - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.457Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to heartbeat to: peer=b25a8bce-8672-f03b-e815-2af799dd3a0e error="failed to connect to peer: b25a8bce-8672-f03b-e815-2af799dd3a0e" -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.460Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to appendEntries to: peer="{Voter server-b25a8bce-8672-f03b-e815-2af799dd3a0e b25a8bce-8672-f03b-e815-2af799dd3a0e}" error="failed to connect to peer: b25a8bce-8672-f03b-e815-2af799dd3a0e" - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.460Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to appendEntries to: peer="{Voter server-b25a8bce-8672-f03b-e815-2af799dd3a0e b25a8bce-8672-f03b-e815-2af799dd3a0e}" error="failed to connect to peer: b25a8bce-8672-f03b-e815-2af799dd3a0e" -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.464Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to appendEntries to: peer="{Voter server-9b3a9873-81f7-9223-b14f-d303c904b789 9b3a9873-81f7-9223-b14f-d303c904b789}" error="failed to connect to peer: 9b3a9873-81f7-9223-b14f-d303c904b789" - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.464Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to appendEntries to: peer="{Voter server-9b3a9873-81f7-9223-b14f-d303c904b789 9b3a9873-81f7-9223-b14f-d303c904b789}" error="failed to connect to peer: 9b3a9873-81f7-9223-b14f-d303c904b789" -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.468Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to heartbeat to: peer=9b3a9873-81f7-9223-b14f-d303c904b789 error="failed to connect to peer: 9b3a9873-81f7-9223-b14f-d303c904b789" - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.468Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to heartbeat to: peer=9b3a9873-81f7-9223-b14f-d303c904b789 error="failed to connect to peer: 9b3a9873-81f7-9223-b14f-d303c904b789" -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.476Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to heartbeat to: peer=b25a8bce-8672-f03b-e815-2af799dd3a0e error="failed to connect to peer: b25a8bce-8672-f03b-e815-2af799dd3a0e" - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.476Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to heartbeat to: peer=b25a8bce-8672-f03b-e815-2af799dd3a0e error="failed to connect to peer: b25a8bce-8672-f03b-e815-2af799dd3a0e" -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.478Z [WARN] server-054499dc-6587-90bb-231e-873c097815a2: failed to contact: server-id=server-9b3a9873-81f7-9223-b14f-d303c904b789 time=50.63595ms - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.478Z [WARN] server-054499dc-6587-90bb-231e-873c097815a2: failed to contact: server-id=server-9b3a9873-81f7-9223-b14f-d303c904b789 time=50.63595ms -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.480Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to appendEntries to: peer="{Voter server-b25a8bce-8672-f03b-e815-2af799dd3a0e b25a8bce-8672-f03b-e815-2af799dd3a0e}" error="failed to connect to peer: b25a8bce-8672-f03b-e815-2af799dd3a0e" - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.480Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to appendEntries to: peer="{Voter server-b25a8bce-8672-f03b-e815-2af799dd3a0e b25a8bce-8672-f03b-e815-2af799dd3a0e}" error="failed to connect to peer: b25a8bce-8672-f03b-e815-2af799dd3a0e" -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.483Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to appendEntries to: peer="{Voter server-9b3a9873-81f7-9223-b14f-d303c904b789 9b3a9873-81f7-9223-b14f-d303c904b789}" error="failed to connect to peer: 9b3a9873-81f7-9223-b14f-d303c904b789" - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.483Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to appendEntries to: peer="{Voter server-9b3a9873-81f7-9223-b14f-d303c904b789 9b3a9873-81f7-9223-b14f-d303c904b789}" error="failed to connect to peer: 9b3a9873-81f7-9223-b14f-d303c904b789" -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.489Z [WARN] server-054499dc-6587-90bb-231e-873c097815a2: failed to contact: server-id=server-9b3a9873-81f7-9223-b14f-d303c904b789 time=61.400165ms - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.489Z [WARN] server-054499dc-6587-90bb-231e-873c097815a2: failed to contact: server-id=server-9b3a9873-81f7-9223-b14f-d303c904b789 time=61.400165ms -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.489Z [WARN] server-054499dc-6587-90bb-231e-873c097815a2: failed to contact: server-id=server-b25a8bce-8672-f03b-e815-2af799dd3a0e time=59.171303ms - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.489Z [WARN] server-054499dc-6587-90bb-231e-873c097815a2: failed to contact: server-id=server-b25a8bce-8672-f03b-e815-2af799dd3a0e time=59.171303ms -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.489Z [WARN] server-054499dc-6587-90bb-231e-873c097815a2: failed to contact quorum of nodes, stepping down - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.489Z [WARN] server-054499dc-6587-90bb-231e-873c097815a2: failed to contact quorum of nodes, stepping down -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.489Z [INFO] server-054499dc-6587-90bb-231e-873c097815a2: entering follower state: follower="Node at 054499dc-6587-90bb-231e-873c097815a2 [Follower]" leader= - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.489Z [INFO] server-054499dc-6587-90bb-231e-873c097815a2: entering follower state: follower="Node at 054499dc-6587-90bb-231e-873c097815a2 [Follower]" leader= -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.496Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to heartbeat to: peer=9b3a9873-81f7-9223-b14f-d303c904b789 error="failed to connect to peer: 9b3a9873-81f7-9223-b14f-d303c904b789" - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.496Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to heartbeat to: peer=9b3a9873-81f7-9223-b14f-d303c904b789 error="failed to connect to peer: 9b3a9873-81f7-9223-b14f-d303c904b789" -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.502Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to heartbeat to: peer=b25a8bce-8672-f03b-e815-2af799dd3a0e error="failed to connect to peer: b25a8bce-8672-f03b-e815-2af799dd3a0e" - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.502Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to heartbeat to: peer=b25a8bce-8672-f03b-e815-2af799dd3a0e error="failed to connect to peer: b25a8bce-8672-f03b-e815-2af799dd3a0e" -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.510Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to appendEntries to: peer="{Voter server-9b3a9873-81f7-9223-b14f-d303c904b789 9b3a9873-81f7-9223-b14f-d303c904b789}" error="failed to connect to peer: 9b3a9873-81f7-9223-b14f-d303c904b789" - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.510Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to appendEntries to: peer="{Voter server-9b3a9873-81f7-9223-b14f-d303c904b789 9b3a9873-81f7-9223-b14f-d303c904b789}" error="failed to connect to peer: 9b3a9873-81f7-9223-b14f-d303c904b789" -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.510Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to appendEntries to: peer="{Voter server-b25a8bce-8672-f03b-e815-2af799dd3a0e b25a8bce-8672-f03b-e815-2af799dd3a0e}" error="failed to connect to peer: b25a8bce-8672-f03b-e815-2af799dd3a0e" - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.510Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to appendEntries to: peer="{Voter server-b25a8bce-8672-f03b-e815-2af799dd3a0e b25a8bce-8672-f03b-e815-2af799dd3a0e}" error="failed to connect to peer: b25a8bce-8672-f03b-e815-2af799dd3a0e" -testLoggerAdapter verbose: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.514Z [WARN] server-9b3a9873-81f7-9223-b14f-d303c904b789: heartbeat timeout reached, starting election: last-leader=054499dc-6587-90bb-231e-873c097815a2 - testing.go:149: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.514Z [WARN] server-9b3a9873-81f7-9223-b14f-d303c904b789: heartbeat timeout reached, starting election: last-leader=054499dc-6587-90bb-231e-873c097815a2 -testLoggerAdapter verbose: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.514Z [INFO] server-9b3a9873-81f7-9223-b14f-d303c904b789: entering candidate state: node="Node at 9b3a9873-81f7-9223-b14f-d303c904b789 [Candidate]" term=3 - testing.go:149: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.514Z [INFO] server-9b3a9873-81f7-9223-b14f-d303c904b789: entering candidate state: node="Node at 9b3a9873-81f7-9223-b14f-d303c904b789 [Candidate]" term=3 -testLoggerAdapter verbose: server-b25a8bce-8672-f03b-e815-2af799dd3a0e: 2025-02-09T06:52:24.514Z [WARN] server-b25a8bce-8672-f03b-e815-2af799dd3a0e: rejecting vote request since we have a leader: from=9b3a9873-81f7-9223-b14f-d303c904b789 leader=054499dc-6587-90bb-231e-873c097815a2 - testing.go:149: server-b25a8bce-8672-f03b-e815-2af799dd3a0e: 2025-02-09T06:52:24.514Z [WARN] server-b25a8bce-8672-f03b-e815-2af799dd3a0e: rejecting vote request since we have a leader: from=9b3a9873-81f7-9223-b14f-d303c904b789 leader=054499dc-6587-90bb-231e-873c097815a2 -testLoggerAdapter verbose: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.514Z [ERROR] server-9b3a9873-81f7-9223-b14f-d303c904b789: failed to make requestVote RPC: target="{Voter server-054499dc-6587-90bb-231e-873c097815a2 054499dc-6587-90bb-231e-873c097815a2}" error="failed to connect to peer: 054499dc-6587-90bb-231e-873c097815a2" - testing.go:149: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.514Z [ERROR] server-9b3a9873-81f7-9223-b14f-d303c904b789: failed to make requestVote RPC: target="{Voter server-054499dc-6587-90bb-231e-873c097815a2 054499dc-6587-90bb-231e-873c097815a2}" error="failed to connect to peer: 054499dc-6587-90bb-231e-873c097815a2" -testLoggerAdapter verbose: server-b25a8bce-8672-f03b-e815-2af799dd3a0e: 2025-02-09T06:52:24.516Z [WARN] server-b25a8bce-8672-f03b-e815-2af799dd3a0e: heartbeat timeout reached, starting election: last-leader=054499dc-6587-90bb-231e-873c097815a2 - testing.go:149: server-b25a8bce-8672-f03b-e815-2af799dd3a0e: 2025-02-09T06:52:24.516Z [WARN] server-b25a8bce-8672-f03b-e815-2af799dd3a0e: heartbeat timeout reached, starting election: last-leader=054499dc-6587-90bb-231e-873c097815a2 -testLoggerAdapter verbose: server-b25a8bce-8672-f03b-e815-2af799dd3a0e: 2025-02-09T06:52:24.516Z [INFO] server-b25a8bce-8672-f03b-e815-2af799dd3a0e: entering candidate state: node="Node at b25a8bce-8672-f03b-e815-2af799dd3a0e [Candidate]" term=3 - testing.go:149: server-b25a8bce-8672-f03b-e815-2af799dd3a0e: 2025-02-09T06:52:24.516Z [INFO] server-b25a8bce-8672-f03b-e815-2af799dd3a0e: entering candidate state: node="Node at b25a8bce-8672-f03b-e815-2af799dd3a0e [Candidate]" term=3 -testLoggerAdapter verbose: server-b25a8bce-8672-f03b-e815-2af799dd3a0e: 2025-02-09T06:52:24.516Z [ERROR] server-b25a8bce-8672-f03b-e815-2af799dd3a0e: failed to make requestVote RPC: target="{Voter server-054499dc-6587-90bb-231e-873c097815a2 054499dc-6587-90bb-231e-873c097815a2}" error="failed to connect to peer: 054499dc-6587-90bb-231e-873c097815a2" -testLoggerAdapter verbose: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.516Z [INFO] server-9b3a9873-81f7-9223-b14f-d303c904b789: duplicate requestVote for same term: term=3 - testing.go:149: server-b25a8bce-8672-f03b-e815-2af799dd3a0e: 2025-02-09T06:52:24.516Z [ERROR] server-b25a8bce-8672-f03b-e815-2af799dd3a0e: failed to make requestVote RPC: target="{Voter server-054499dc-6587-90bb-231e-873c097815a2 054499dc-6587-90bb-231e-873c097815a2}" error="failed to connect to peer: 054499dc-6587-90bb-231e-873c097815a2" - testing.go:149: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.516Z [INFO] server-9b3a9873-81f7-9223-b14f-d303c904b789: duplicate requestVote for same term: term=3 -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.576Z [WARN] server-054499dc-6587-90bb-231e-873c097815a2: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.576Z [WARN] server-054499dc-6587-90bb-231e-873c097815a2: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.576Z [INFO] server-054499dc-6587-90bb-231e-873c097815a2: entering candidate state: node="Node at 054499dc-6587-90bb-231e-873c097815a2 [Candidate]" term=3 - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.576Z [INFO] server-054499dc-6587-90bb-231e-873c097815a2: entering candidate state: node="Node at 054499dc-6587-90bb-231e-873c097815a2 [Candidate]" term=3 -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.576Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to make requestVote RPC: target="{Voter server-b25a8bce-8672-f03b-e815-2af799dd3a0e b25a8bce-8672-f03b-e815-2af799dd3a0e}" error="failed to connect to peer: b25a8bce-8672-f03b-e815-2af799dd3a0e" - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.576Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to make requestVote RPC: target="{Voter server-b25a8bce-8672-f03b-e815-2af799dd3a0e b25a8bce-8672-f03b-e815-2af799dd3a0e}" error="failed to connect to peer: b25a8bce-8672-f03b-e815-2af799dd3a0e" -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.576Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to make requestVote RPC: target="{Voter server-9b3a9873-81f7-9223-b14f-d303c904b789 9b3a9873-81f7-9223-b14f-d303c904b789}" error="failed to connect to peer: 9b3a9873-81f7-9223-b14f-d303c904b789" - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.576Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to make requestVote RPC: target="{Voter server-9b3a9873-81f7-9223-b14f-d303c904b789 9b3a9873-81f7-9223-b14f-d303c904b789}" error="failed to connect to peer: 9b3a9873-81f7-9223-b14f-d303c904b789" -testLoggerAdapter verbose: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.580Z [WARN] server-9b3a9873-81f7-9223-b14f-d303c904b789: Election timeout reached, restarting election - testing.go:149: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.580Z [WARN] server-9b3a9873-81f7-9223-b14f-d303c904b789: Election timeout reached, restarting election -testLoggerAdapter verbose: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.580Z [INFO] server-9b3a9873-81f7-9223-b14f-d303c904b789: entering candidate state: node="Node at 9b3a9873-81f7-9223-b14f-d303c904b789 [Candidate]" term=4 - testing.go:149: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.580Z [INFO] server-9b3a9873-81f7-9223-b14f-d303c904b789: entering candidate state: node="Node at 9b3a9873-81f7-9223-b14f-d303c904b789 [Candidate]" term=4 -testLoggerAdapter verbose: server-b25a8bce-8672-f03b-e815-2af799dd3a0e: 2025-02-09T06:52:24.580Z [INFO] server-b25a8bce-8672-f03b-e815-2af799dd3a0e: entering follower state: follower="Node at b25a8bce-8672-f03b-e815-2af799dd3a0e [Follower]" leader= -testLoggerAdapter verbose: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.580Z [ERROR] server-9b3a9873-81f7-9223-b14f-d303c904b789: failed to make requestVote RPC: target="{Voter server-054499dc-6587-90bb-231e-873c097815a2 054499dc-6587-90bb-231e-873c097815a2}" error="failed to connect to peer: 054499dc-6587-90bb-231e-873c097815a2" - testing.go:149: server-b25a8bce-8672-f03b-e815-2af799dd3a0e: 2025-02-09T06:52:24.580Z [INFO] server-b25a8bce-8672-f03b-e815-2af799dd3a0e: entering follower state: follower="Node at b25a8bce-8672-f03b-e815-2af799dd3a0e [Follower]" leader= - testing.go:149: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.580Z [ERROR] server-9b3a9873-81f7-9223-b14f-d303c904b789: failed to make requestVote RPC: target="{Voter server-054499dc-6587-90bb-231e-873c097815a2 054499dc-6587-90bb-231e-873c097815a2}" error="failed to connect to peer: 054499dc-6587-90bb-231e-873c097815a2" -testLoggerAdapter verbose: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.580Z [INFO] server-9b3a9873-81f7-9223-b14f-d303c904b789: election won: tally=2 - testing.go:149: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.580Z [INFO] server-9b3a9873-81f7-9223-b14f-d303c904b789: election won: tally=2 -testLoggerAdapter verbose: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.580Z [INFO] server-9b3a9873-81f7-9223-b14f-d303c904b789: entering leader state: leader="Node at 9b3a9873-81f7-9223-b14f-d303c904b789 [Leader]" - testing.go:149: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.580Z [INFO] server-9b3a9873-81f7-9223-b14f-d303c904b789: entering leader state: leader="Node at 9b3a9873-81f7-9223-b14f-d303c904b789 [Leader]" -testLoggerAdapter verbose: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.580Z [INFO] server-9b3a9873-81f7-9223-b14f-d303c904b789: added peer, starting replication: peer=server-054499dc-6587-90bb-231e-873c097815a2 - testing.go:149: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.580Z [INFO] server-9b3a9873-81f7-9223-b14f-d303c904b789: added peer, starting replication: peer=server-054499dc-6587-90bb-231e-873c097815a2 -testLoggerAdapter verbose: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.580Z [INFO] server-9b3a9873-81f7-9223-b14f-d303c904b789: added peer, starting replication: peer=server-b25a8bce-8672-f03b-e815-2af799dd3a0e - testing.go:149: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.580Z [INFO] server-9b3a9873-81f7-9223-b14f-d303c904b789: added peer, starting replication: peer=server-b25a8bce-8672-f03b-e815-2af799dd3a0e -testLoggerAdapter verbose: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.580Z [INFO] server-9b3a9873-81f7-9223-b14f-d303c904b789: pipelining replication: peer="{Voter server-b25a8bce-8672-f03b-e815-2af799dd3a0e b25a8bce-8672-f03b-e815-2af799dd3a0e}" - testing.go:149: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.580Z [INFO] server-9b3a9873-81f7-9223-b14f-d303c904b789: pipelining replication: peer="{Voter server-b25a8bce-8672-f03b-e815-2af799dd3a0e b25a8bce-8672-f03b-e815-2af799dd3a0e}" -testLoggerAdapter verbose: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.581Z [ERROR] server-9b3a9873-81f7-9223-b14f-d303c904b789: failed to appendEntries to: peer="{Voter server-054499dc-6587-90bb-231e-873c097815a2 054499dc-6587-90bb-231e-873c097815a2}" error="failed to connect to peer: 054499dc-6587-90bb-231e-873c097815a2" - testing.go:149: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.581Z [ERROR] server-9b3a9873-81f7-9223-b14f-d303c904b789: failed to appendEntries to: peer="{Voter server-054499dc-6587-90bb-231e-873c097815a2 054499dc-6587-90bb-231e-873c097815a2}" error="failed to connect to peer: 054499dc-6587-90bb-231e-873c097815a2" -testLoggerAdapter verbose: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.590Z [ERROR] server-9b3a9873-81f7-9223-b14f-d303c904b789: failed to heartbeat to: peer=054499dc-6587-90bb-231e-873c097815a2 error="failed to connect to peer: 054499dc-6587-90bb-231e-873c097815a2" - testing.go:149: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.590Z [ERROR] server-9b3a9873-81f7-9223-b14f-d303c904b789: failed to heartbeat to: peer=054499dc-6587-90bb-231e-873c097815a2 error="failed to connect to peer: 054499dc-6587-90bb-231e-873c097815a2" -testLoggerAdapter verbose: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.599Z [ERROR] server-9b3a9873-81f7-9223-b14f-d303c904b789: failed to appendEntries to: peer="{Voter server-054499dc-6587-90bb-231e-873c097815a2 054499dc-6587-90bb-231e-873c097815a2}" error="failed to connect to peer: 054499dc-6587-90bb-231e-873c097815a2" - testing.go:149: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.599Z [ERROR] server-9b3a9873-81f7-9223-b14f-d303c904b789: failed to appendEntries to: peer="{Voter server-054499dc-6587-90bb-231e-873c097815a2 054499dc-6587-90bb-231e-873c097815a2}" error="failed to connect to peer: 054499dc-6587-90bb-231e-873c097815a2" -testLoggerAdapter verbose: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.609Z [ERROR] server-9b3a9873-81f7-9223-b14f-d303c904b789: failed to heartbeat to: peer=054499dc-6587-90bb-231e-873c097815a2 error="failed to connect to peer: 054499dc-6587-90bb-231e-873c097815a2" - testing.go:149: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.609Z [ERROR] server-9b3a9873-81f7-9223-b14f-d303c904b789: failed to heartbeat to: peer=054499dc-6587-90bb-231e-873c097815a2 error="failed to connect to peer: 054499dc-6587-90bb-231e-873c097815a2" -testLoggerAdapter verbose: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.619Z [ERROR] server-9b3a9873-81f7-9223-b14f-d303c904b789: failed to appendEntries to: peer="{Voter server-054499dc-6587-90bb-231e-873c097815a2 054499dc-6587-90bb-231e-873c097815a2}" error="failed to connect to peer: 054499dc-6587-90bb-231e-873c097815a2" - testing.go:149: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.619Z [ERROR] server-9b3a9873-81f7-9223-b14f-d303c904b789: failed to appendEntries to: peer="{Voter server-054499dc-6587-90bb-231e-873c097815a2 054499dc-6587-90bb-231e-873c097815a2}" error="failed to connect to peer: 054499dc-6587-90bb-231e-873c097815a2" -testLoggerAdapter verbose: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.627Z [ERROR] server-9b3a9873-81f7-9223-b14f-d303c904b789: failed to heartbeat to: peer=054499dc-6587-90bb-231e-873c097815a2 error="failed to connect to peer: 054499dc-6587-90bb-231e-873c097815a2" - testing.go:149: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.627Z [ERROR] server-9b3a9873-81f7-9223-b14f-d303c904b789: failed to heartbeat to: peer=054499dc-6587-90bb-231e-873c097815a2 error="failed to connect to peer: 054499dc-6587-90bb-231e-873c097815a2" -testLoggerAdapter verbose: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.631Z [WARN] server-9b3a9873-81f7-9223-b14f-d303c904b789: failed to contact: server-id=server-054499dc-6587-90bb-231e-873c097815a2 time=50.681605ms - testing.go:149: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.631Z [WARN] server-9b3a9873-81f7-9223-b14f-d303c904b789: failed to contact: server-id=server-054499dc-6587-90bb-231e-873c097815a2 time=50.681605ms -testLoggerAdapter verbose: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.646Z [ERROR] server-9b3a9873-81f7-9223-b14f-d303c904b789: failed to appendEntries to: peer="{Voter server-054499dc-6587-90bb-231e-873c097815a2 054499dc-6587-90bb-231e-873c097815a2}" error="failed to connect to peer: 054499dc-6587-90bb-231e-873c097815a2" - testing.go:149: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.646Z [ERROR] server-9b3a9873-81f7-9223-b14f-d303c904b789: failed to appendEntries to: peer="{Voter server-054499dc-6587-90bb-231e-873c097815a2 054499dc-6587-90bb-231e-873c097815a2}" error="failed to connect to peer: 054499dc-6587-90bb-231e-873c097815a2" -testLoggerAdapter verbose: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.653Z [ERROR] server-9b3a9873-81f7-9223-b14f-d303c904b789: failed to heartbeat to: peer=054499dc-6587-90bb-231e-873c097815a2 error="failed to connect to peer: 054499dc-6587-90bb-231e-873c097815a2" - testing.go:149: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.653Z [ERROR] server-9b3a9873-81f7-9223-b14f-d303c904b789: failed to heartbeat to: peer=054499dc-6587-90bb-231e-873c097815a2 error="failed to connect to peer: 054499dc-6587-90bb-231e-873c097815a2" -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.659Z [WARN] server-054499dc-6587-90bb-231e-873c097815a2: Election timeout reached, restarting election - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.659Z [WARN] server-054499dc-6587-90bb-231e-873c097815a2: Election timeout reached, restarting election -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.659Z [INFO] server-054499dc-6587-90bb-231e-873c097815a2: entering candidate state: node="Node at 054499dc-6587-90bb-231e-873c097815a2 [Candidate]" term=4 - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.659Z [INFO] server-054499dc-6587-90bb-231e-873c097815a2: entering candidate state: node="Node at 054499dc-6587-90bb-231e-873c097815a2 [Candidate]" term=4 -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.659Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to make requestVote RPC: target="{Voter server-b25a8bce-8672-f03b-e815-2af799dd3a0e b25a8bce-8672-f03b-e815-2af799dd3a0e}" error="failed to connect to peer: b25a8bce-8672-f03b-e815-2af799dd3a0e" - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.659Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to make requestVote RPC: target="{Voter server-b25a8bce-8672-f03b-e815-2af799dd3a0e b25a8bce-8672-f03b-e815-2af799dd3a0e}" error="failed to connect to peer: b25a8bce-8672-f03b-e815-2af799dd3a0e" -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.659Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to make requestVote RPC: target="{Voter server-9b3a9873-81f7-9223-b14f-d303c904b789 9b3a9873-81f7-9223-b14f-d303c904b789}" error="failed to connect to peer: 9b3a9873-81f7-9223-b14f-d303c904b789" - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.659Z [ERROR] server-054499dc-6587-90bb-231e-873c097815a2: failed to make requestVote RPC: target="{Voter server-9b3a9873-81f7-9223-b14f-d303c904b789 9b3a9873-81f7-9223-b14f-d303c904b789}" error="failed to connect to peer: 9b3a9873-81f7-9223-b14f-d303c904b789" -testLoggerAdapter verbose: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.679Z [WARN] server-9b3a9873-81f7-9223-b14f-d303c904b789: failed to contact: server-id=server-054499dc-6587-90bb-231e-873c097815a2 time=98.40232ms - testing.go:149: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.679Z [WARN] server-9b3a9873-81f7-9223-b14f-d303c904b789: failed to contact: server-id=server-054499dc-6587-90bb-231e-873c097815a2 time=98.40232ms -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:24.686Z [INFO] stable state for Leader reached at 2025-02-09 06:52:24.580973346 +0000 UTC m=+5.559782129 (1 nodes), 141.73754ms from start of poll, 436.138321ms from cluster start. Timeout at 2025-02-09 06:52:24.686537699 +0000 UTC m=+5.665346467, 105.564338ms after stability - testing.go:149: cluster: 2025-02-09T06:52:24.686Z [INFO] stable state for Leader reached at 2025-02-09 06:52:24.580973346 +0000 UTC m=+5.559782129 (1 nodes), 141.73754ms from start of poll, 436.138321ms from cluster start. Timeout at 2025-02-09 06:52:24.686537699 +0000 UTC m=+5.665346467, 105.564338ms after stability - raft_test.go:371: [INFO] Reconnecting Node at 054499dc-6587-90bb-231e-873c097815a2 [Candidate] -testLoggerAdapter verbose: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.695Z [INFO] server-054499dc-6587-90bb-231e-873c097815a2: entering follower state: follower="Node at 054499dc-6587-90bb-231e-873c097815a2 [Follower]" leader=9b3a9873-81f7-9223-b14f-d303c904b789 - testing.go:149: server-054499dc-6587-90bb-231e-873c097815a2: 2025-02-09T06:52:24.695Z [INFO] server-054499dc-6587-90bb-231e-873c097815a2: entering follower state: follower="Node at 054499dc-6587-90bb-231e-873c097815a2 [Follower]" leader=9b3a9873-81f7-9223-b14f-d303c904b789 -testLoggerAdapter verbose: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.695Z [INFO] server-9b3a9873-81f7-9223-b14f-d303c904b789: pipelining replication: peer="{Voter server-054499dc-6587-90bb-231e-873c097815a2 054499dc-6587-90bb-231e-873c097815a2}" - testing.go:149: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.695Z [INFO] server-9b3a9873-81f7-9223-b14f-d303c904b789: pipelining replication: peer="{Voter server-054499dc-6587-90bb-231e-873c097815a2 054499dc-6587-90bb-231e-873c097815a2}" -testLoggerAdapter verbose: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.700Z [INFO] server-9b3a9873-81f7-9223-b14f-d303c904b789: aborting pipeline replication: peer="{Voter server-b25a8bce-8672-f03b-e815-2af799dd3a0e b25a8bce-8672-f03b-e815-2af799dd3a0e}" - testing.go:149: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.700Z [INFO] server-9b3a9873-81f7-9223-b14f-d303c904b789: aborting pipeline replication: peer="{Voter server-b25a8bce-8672-f03b-e815-2af799dd3a0e b25a8bce-8672-f03b-e815-2af799dd3a0e}" -testLoggerAdapter verbose: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.700Z [INFO] server-9b3a9873-81f7-9223-b14f-d303c904b789: aborting pipeline replication: peer="{Voter server-054499dc-6587-90bb-231e-873c097815a2 054499dc-6587-90bb-231e-873c097815a2}" - testing.go:149: server-9b3a9873-81f7-9223-b14f-d303c904b789: 2025-02-09T06:52:24.700Z [INFO] server-9b3a9873-81f7-9223-b14f-d303c904b789: aborting pipeline replication: peer="{Voter server-054499dc-6587-90bb-231e-873c097815a2 054499dc-6587-90bb-231e-873c097815a2}" ---- PASS: TestRaft_LeaderFail (0.56s) +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.130Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9c603e47-dcb5-0e30-0afa-aaca8df23da9 Address:9c603e47-dcb5-0e30-0afa-aaca8df23da9} {Suffrage:Voter ID:server-aef27578-0e68-b160-b9b7-f17b6fd6c17e Address:aef27578-0e68-b160-b9b7-f17b6fd6c17e} {Suffrage:Voter ID:server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 Address:4ff51c45-01a1-521b-0e7f-4a59ba7c74c9}]" + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.130Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9c603e47-dcb5-0e30-0afa-aaca8df23da9 Address:9c603e47-dcb5-0e30-0afa-aaca8df23da9} {Suffrage:Voter ID:server-aef27578-0e68-b160-b9b7-f17b6fd6c17e Address:aef27578-0e68-b160-b9b7-f17b6fd6c17e} {Suffrage:Voter ID:server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 Address:4ff51c45-01a1-521b-0e7f-4a59ba7c74c9}]" +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.131Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: entering follower state: follower="Node at 9c603e47-dcb5-0e30-0afa-aaca8df23da9 [Follower]" leader= + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.131Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: entering follower state: follower="Node at 9c603e47-dcb5-0e30-0afa-aaca8df23da9 [Follower]" leader= +testLoggerAdapter verbose: server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: 2024-01-08T00:30:59.131Z [INFO] server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9c603e47-dcb5-0e30-0afa-aaca8df23da9 Address:9c603e47-dcb5-0e30-0afa-aaca8df23da9} {Suffrage:Voter ID:server-aef27578-0e68-b160-b9b7-f17b6fd6c17e Address:aef27578-0e68-b160-b9b7-f17b6fd6c17e} {Suffrage:Voter ID:server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 Address:4ff51c45-01a1-521b-0e7f-4a59ba7c74c9}]" + testing.go:149: server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: 2024-01-08T00:30:59.131Z [INFO] server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9c603e47-dcb5-0e30-0afa-aaca8df23da9 Address:9c603e47-dcb5-0e30-0afa-aaca8df23da9} {Suffrage:Voter ID:server-aef27578-0e68-b160-b9b7-f17b6fd6c17e Address:aef27578-0e68-b160-b9b7-f17b6fd6c17e} {Suffrage:Voter ID:server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 Address:4ff51c45-01a1-521b-0e7f-4a59ba7c74c9}]" +testLoggerAdapter verbose: server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: 2024-01-08T00:30:59.131Z [INFO] server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: entering follower state: follower="Node at aef27578-0e68-b160-b9b7-f17b6fd6c17e [Follower]" leader= + testing.go:149: server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: 2024-01-08T00:30:59.131Z [INFO] server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: entering follower state: follower="Node at aef27578-0e68-b160-b9b7-f17b6fd6c17e [Follower]" leader= +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.131Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9c603e47-dcb5-0e30-0afa-aaca8df23da9 Address:9c603e47-dcb5-0e30-0afa-aaca8df23da9} {Suffrage:Voter ID:server-aef27578-0e68-b160-b9b7-f17b6fd6c17e Address:aef27578-0e68-b160-b9b7-f17b6fd6c17e} {Suffrage:Voter ID:server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 Address:4ff51c45-01a1-521b-0e7f-4a59ba7c74c9}]" + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.131Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9c603e47-dcb5-0e30-0afa-aaca8df23da9 Address:9c603e47-dcb5-0e30-0afa-aaca8df23da9} {Suffrage:Voter ID:server-aef27578-0e68-b160-b9b7-f17b6fd6c17e Address:aef27578-0e68-b160-b9b7-f17b6fd6c17e} {Suffrage:Voter ID:server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 Address:4ff51c45-01a1-521b-0e7f-4a59ba7c74c9}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:30:59.132Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T00:30:59.132Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.132Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: entering follower state: follower="Node at 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 [Follower]" leader= + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.132Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: entering follower state: follower="Node at 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 [Follower]" leader= +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.197Z [WARN] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.197Z [WARN] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.197Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: entering candidate state: node="Node at 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 [Candidate]" term=2 + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.197Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: entering candidate state: node="Node at 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 [Candidate]" term=2 +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.198Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: election won: tally=2 + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.198Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: election won: tally=2 +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.198Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: entering leader state: leader="Node at 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 [Leader]" + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.198Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: entering leader state: leader="Node at 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 [Leader]" +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.198Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: added peer, starting replication: peer=server-9c603e47-dcb5-0e30-0afa-aaca8df23da9 + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.198Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: added peer, starting replication: peer=server-9c603e47-dcb5-0e30-0afa-aaca8df23da9 +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.198Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: added peer, starting replication: peer=server-aef27578-0e68-b160-b9b7-f17b6fd6c17e + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.198Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: added peer, starting replication: peer=server-aef27578-0e68-b160-b9b7-f17b6fd6c17e +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.198Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: pipelining replication: peer="{Voter server-aef27578-0e68-b160-b9b7-f17b6fd6c17e aef27578-0e68-b160-b9b7-f17b6fd6c17e}" + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.198Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: pipelining replication: peer="{Voter server-aef27578-0e68-b160-b9b7-f17b6fd6c17e aef27578-0e68-b160-b9b7-f17b6fd6c17e}" +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.198Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: pipelining replication: peer="{Voter server-9c603e47-dcb5-0e30-0afa-aaca8df23da9 9c603e47-dcb5-0e30-0afa-aaca8df23da9}" + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.198Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: pipelining replication: peer="{Voter server-9c603e47-dcb5-0e30-0afa-aaca8df23da9 9c603e47-dcb5-0e30-0afa-aaca8df23da9}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:30:59.303Z [INFO] stable state for Follower reached at 2024-01-08 00:30:59.198212299 +0000 UTC m=+5.245101418 (2 nodes), 65.971574ms from start of poll, 67.4763ms from cluster start. Timeout at 2024-01-08 00:30:59.303843107 +0000 UTC m=+5.350732154, 105.630736ms after stability + testing.go:149: cluster: 2024-01-08T00:30:59.303Z [INFO] stable state for Follower reached at 2024-01-08 00:30:59.198212299 +0000 UTC m=+5.245101418 (2 nodes), 65.971574ms from start of poll, 67.4763ms from cluster start. Timeout at 2024-01-08 00:30:59.303843107 +0000 UTC m=+5.350732154, 105.630736ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T00:30:59.304Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:30:59.304Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T00:30:59.409Z [INFO] stable state for Leader reached at 2024-01-08 00:30:59.304038833 +0000 UTC m=+5.350927877 (1 nodes), 2.002µs from start of poll, 173.302759ms from cluster start. Timeout at 2024-01-08 00:30:59.409050697 +0000 UTC m=+5.455939736, 105.011859ms after stability + testing.go:149: cluster: 2024-01-08T00:30:59.409Z [INFO] stable state for Leader reached at 2024-01-08 00:30:59.304038833 +0000 UTC m=+5.350927877 (1 nodes), 2.002µs from start of poll, 173.302759ms from cluster start. Timeout at 2024-01-08 00:30:59.409050697 +0000 UTC m=+5.455939736, 105.011859ms after stability + raft_test.go:336: [INFO] Disconnecting Node at 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 [Leader] +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.422Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to pipeline appendEntries: peer="{Voter server-9c603e47-dcb5-0e30-0afa-aaca8df23da9 9c603e47-dcb5-0e30-0afa-aaca8df23da9}" error="append pipeline closed" + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.422Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to pipeline appendEntries: peer="{Voter server-9c603e47-dcb5-0e30-0afa-aaca8df23da9 9c603e47-dcb5-0e30-0afa-aaca8df23da9}" error="append pipeline closed" +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.422Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: aborting pipeline replication: peer="{Voter server-9c603e47-dcb5-0e30-0afa-aaca8df23da9 9c603e47-dcb5-0e30-0afa-aaca8df23da9}" + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.422Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: aborting pipeline replication: peer="{Voter server-9c603e47-dcb5-0e30-0afa-aaca8df23da9 9c603e47-dcb5-0e30-0afa-aaca8df23da9}" +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.423Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to heartbeat to: peer=9c603e47-dcb5-0e30-0afa-aaca8df23da9 error="failed to connect to peer: 9c603e47-dcb5-0e30-0afa-aaca8df23da9" + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.423Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to heartbeat to: peer=9c603e47-dcb5-0e30-0afa-aaca8df23da9 error="failed to connect to peer: 9c603e47-dcb5-0e30-0afa-aaca8df23da9" +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.426Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to heartbeat to: peer=aef27578-0e68-b160-b9b7-f17b6fd6c17e error="failed to connect to peer: aef27578-0e68-b160-b9b7-f17b6fd6c17e" + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.426Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to heartbeat to: peer=aef27578-0e68-b160-b9b7-f17b6fd6c17e error="failed to connect to peer: aef27578-0e68-b160-b9b7-f17b6fd6c17e" +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.426Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to pipeline appendEntries: peer="{Voter server-aef27578-0e68-b160-b9b7-f17b6fd6c17e aef27578-0e68-b160-b9b7-f17b6fd6c17e}" error="append pipeline closed" + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.426Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to pipeline appendEntries: peer="{Voter server-aef27578-0e68-b160-b9b7-f17b6fd6c17e aef27578-0e68-b160-b9b7-f17b6fd6c17e}" error="append pipeline closed" +testLoggerAdapter verbose: cluster: 2024-01-08T00:30:59.426Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.426Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: aborting pipeline replication: peer="{Voter server-aef27578-0e68-b160-b9b7-f17b6fd6c17e aef27578-0e68-b160-b9b7-f17b6fd6c17e}" + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.426Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: aborting pipeline replication: peer="{Voter server-aef27578-0e68-b160-b9b7-f17b6fd6c17e aef27578-0e68-b160-b9b7-f17b6fd6c17e}" + testing.go:149: cluster: 2024-01-08T00:30:59.426Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.431Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to appendEntries to: peer="{Voter server-9c603e47-dcb5-0e30-0afa-aaca8df23da9 9c603e47-dcb5-0e30-0afa-aaca8df23da9}" error="failed to connect to peer: 9c603e47-dcb5-0e30-0afa-aaca8df23da9" + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.431Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to appendEntries to: peer="{Voter server-9c603e47-dcb5-0e30-0afa-aaca8df23da9 9c603e47-dcb5-0e30-0afa-aaca8df23da9}" error="failed to connect to peer: 9c603e47-dcb5-0e30-0afa-aaca8df23da9" +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.433Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to appendEntries to: peer="{Voter server-aef27578-0e68-b160-b9b7-f17b6fd6c17e aef27578-0e68-b160-b9b7-f17b6fd6c17e}" error="failed to connect to peer: aef27578-0e68-b160-b9b7-f17b6fd6c17e" + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.433Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to appendEntries to: peer="{Voter server-aef27578-0e68-b160-b9b7-f17b6fd6c17e aef27578-0e68-b160-b9b7-f17b6fd6c17e}" error="failed to connect to peer: aef27578-0e68-b160-b9b7-f17b6fd6c17e" +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.442Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to heartbeat to: peer=9c603e47-dcb5-0e30-0afa-aaca8df23da9 error="failed to connect to peer: 9c603e47-dcb5-0e30-0afa-aaca8df23da9" + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.442Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to heartbeat to: peer=9c603e47-dcb5-0e30-0afa-aaca8df23da9 error="failed to connect to peer: 9c603e47-dcb5-0e30-0afa-aaca8df23da9" +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.446Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to heartbeat to: peer=aef27578-0e68-b160-b9b7-f17b6fd6c17e error="failed to connect to peer: aef27578-0e68-b160-b9b7-f17b6fd6c17e" + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.446Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to heartbeat to: peer=aef27578-0e68-b160-b9b7-f17b6fd6c17e error="failed to connect to peer: aef27578-0e68-b160-b9b7-f17b6fd6c17e" +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.449Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to appendEntries to: peer="{Voter server-9c603e47-dcb5-0e30-0afa-aaca8df23da9 9c603e47-dcb5-0e30-0afa-aaca8df23da9}" error="failed to connect to peer: 9c603e47-dcb5-0e30-0afa-aaca8df23da9" + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.449Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to appendEntries to: peer="{Voter server-9c603e47-dcb5-0e30-0afa-aaca8df23da9 9c603e47-dcb5-0e30-0afa-aaca8df23da9}" error="failed to connect to peer: 9c603e47-dcb5-0e30-0afa-aaca8df23da9" +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.454Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to appendEntries to: peer="{Voter server-aef27578-0e68-b160-b9b7-f17b6fd6c17e aef27578-0e68-b160-b9b7-f17b6fd6c17e}" error="failed to connect to peer: aef27578-0e68-b160-b9b7-f17b6fd6c17e" + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.454Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to appendEntries to: peer="{Voter server-aef27578-0e68-b160-b9b7-f17b6fd6c17e aef27578-0e68-b160-b9b7-f17b6fd6c17e}" error="failed to connect to peer: aef27578-0e68-b160-b9b7-f17b6fd6c17e" +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.460Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to heartbeat to: peer=9c603e47-dcb5-0e30-0afa-aaca8df23da9 error="failed to connect to peer: 9c603e47-dcb5-0e30-0afa-aaca8df23da9" + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.460Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to heartbeat to: peer=9c603e47-dcb5-0e30-0afa-aaca8df23da9 error="failed to connect to peer: 9c603e47-dcb5-0e30-0afa-aaca8df23da9" +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.466Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to heartbeat to: peer=aef27578-0e68-b160-b9b7-f17b6fd6c17e error="failed to connect to peer: aef27578-0e68-b160-b9b7-f17b6fd6c17e" + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.466Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to heartbeat to: peer=aef27578-0e68-b160-b9b7-f17b6fd6c17e error="failed to connect to peer: aef27578-0e68-b160-b9b7-f17b6fd6c17e" +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.467Z [WARN] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to contact: server-id=server-9c603e47-dcb5-0e30-0afa-aaca8df23da9 time=50.364282ms + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.467Z [WARN] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to contact: server-id=server-9c603e47-dcb5-0e30-0afa-aaca8df23da9 time=50.364282ms +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.467Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to appendEntries to: peer="{Voter server-9c603e47-dcb5-0e30-0afa-aaca8df23da9 9c603e47-dcb5-0e30-0afa-aaca8df23da9}" error="failed to connect to peer: 9c603e47-dcb5-0e30-0afa-aaca8df23da9" + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.467Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to appendEntries to: peer="{Voter server-9c603e47-dcb5-0e30-0afa-aaca8df23da9 9c603e47-dcb5-0e30-0afa-aaca8df23da9}" error="failed to connect to peer: 9c603e47-dcb5-0e30-0afa-aaca8df23da9" +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.474Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to appendEntries to: peer="{Voter server-aef27578-0e68-b160-b9b7-f17b6fd6c17e aef27578-0e68-b160-b9b7-f17b6fd6c17e}" error="failed to connect to peer: aef27578-0e68-b160-b9b7-f17b6fd6c17e" + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.474Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to appendEntries to: peer="{Voter server-aef27578-0e68-b160-b9b7-f17b6fd6c17e aef27578-0e68-b160-b9b7-f17b6fd6c17e}" error="failed to connect to peer: aef27578-0e68-b160-b9b7-f17b6fd6c17e" +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.478Z [WARN] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to contact: server-id=server-9c603e47-dcb5-0e30-0afa-aaca8df23da9 time=61.161791ms + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.478Z [WARN] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to contact: server-id=server-9c603e47-dcb5-0e30-0afa-aaca8df23da9 time=61.161791ms +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.478Z [WARN] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to contact: server-id=server-aef27578-0e68-b160-b9b7-f17b6fd6c17e time=59.788984ms + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.478Z [WARN] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to contact: server-id=server-aef27578-0e68-b160-b9b7-f17b6fd6c17e time=59.788984ms +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.478Z [WARN] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to contact quorum of nodes, stepping down + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.478Z [WARN] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to contact quorum of nodes, stepping down +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.478Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: entering follower state: follower="Node at 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 [Follower]" leader= + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.478Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: entering follower state: follower="Node at 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 [Follower]" leader= +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.490Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to heartbeat to: peer=9c603e47-dcb5-0e30-0afa-aaca8df23da9 error="failed to connect to peer: 9c603e47-dcb5-0e30-0afa-aaca8df23da9" + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.490Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to heartbeat to: peer=9c603e47-dcb5-0e30-0afa-aaca8df23da9 error="failed to connect to peer: 9c603e47-dcb5-0e30-0afa-aaca8df23da9" +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.494Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to appendEntries to: peer="{Voter server-9c603e47-dcb5-0e30-0afa-aaca8df23da9 9c603e47-dcb5-0e30-0afa-aaca8df23da9}" error="failed to connect to peer: 9c603e47-dcb5-0e30-0afa-aaca8df23da9" + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.494Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to appendEntries to: peer="{Voter server-9c603e47-dcb5-0e30-0afa-aaca8df23da9 9c603e47-dcb5-0e30-0afa-aaca8df23da9}" error="failed to connect to peer: 9c603e47-dcb5-0e30-0afa-aaca8df23da9" +testLoggerAdapter verbose: cluster: 2024-01-08T00:30:59.531Z [INFO] stable state for Leader reached at 2024-01-08 00:30:59.426538546 +0000 UTC m=+5.473427672 (1 nodes), 2.382µs from start of poll, 295.802554ms from cluster start. Timeout at 2024-01-08 00:30:59.531796261 +0000 UTC m=+5.578685296, 105.257624ms after stability + testing.go:149: cluster: 2024-01-08T00:30:59.531Z [INFO] stable state for Leader reached at 2024-01-08 00:30:59.426538546 +0000 UTC m=+5.473427672 (1 nodes), 2.382µs from start of poll, 295.802554ms from cluster start. Timeout at 2024-01-08 00:30:59.531796261 +0000 UTC m=+5.578685296, 105.257624ms after stability +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.537Z [WARN] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: heartbeat timeout reached, starting election: last-leader=4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.537Z [WARN] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: heartbeat timeout reached, starting election: last-leader=4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 +testLoggerAdapter verbose: cluster: 2024-01-08T00:30:59.537Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.537Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: entering candidate state: node="Node at 9c603e47-dcb5-0e30-0afa-aaca8df23da9 [Candidate]" term=3 + testing.go:149: cluster: 2024-01-08T00:30:59.537Z [INFO] starting stability test: raft-state=Leader + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.537Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: entering candidate state: node="Node at 9c603e47-dcb5-0e30-0afa-aaca8df23da9 [Candidate]" term=3 +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.537Z [ERROR] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: failed to make requestVote RPC: target="{Voter server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9}" error="failed to connect to peer: 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9" +testLoggerAdapter verbose: server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: 2024-01-08T00:30:59.537Z [WARN] server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: rejecting vote request since we have a leader: from=9c603e47-dcb5-0e30-0afa-aaca8df23da9 leader=4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.537Z [ERROR] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: failed to make requestVote RPC: target="{Voter server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9}" error="failed to connect to peer: 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9" + testing.go:149: server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: 2024-01-08T00:30:59.537Z [WARN] server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: rejecting vote request since we have a leader: from=9c603e47-dcb5-0e30-0afa-aaca8df23da9 leader=4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 +testLoggerAdapter verbose: server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: 2024-01-08T00:30:59.545Z [WARN] server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: heartbeat timeout reached, starting election: last-leader=4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 + testing.go:149: server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: 2024-01-08T00:30:59.545Z [WARN] server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: heartbeat timeout reached, starting election: last-leader=4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 +testLoggerAdapter verbose: server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: 2024-01-08T00:30:59.545Z [INFO] server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: entering candidate state: node="Node at aef27578-0e68-b160-b9b7-f17b6fd6c17e [Candidate]" term=3 + testing.go:149: server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: 2024-01-08T00:30:59.545Z [INFO] server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: entering candidate state: node="Node at aef27578-0e68-b160-b9b7-f17b6fd6c17e [Candidate]" term=3 +testLoggerAdapter verbose: server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: 2024-01-08T00:30:59.545Z [ERROR] server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: failed to make requestVote RPC: target="{Voter server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9}" error="failed to connect to peer: 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9" + testing.go:149: server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: 2024-01-08T00:30:59.545Z [ERROR] server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: failed to make requestVote RPC: target="{Voter server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9}" error="failed to connect to peer: 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9" +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.545Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: duplicate requestVote for same term: term=3 + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.545Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: duplicate requestVote for same term: term=3 +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.572Z [WARN] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.572Z [WARN] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.572Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: entering candidate state: node="Node at 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 [Candidate]" term=3 + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.572Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: entering candidate state: node="Node at 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 [Candidate]" term=3 +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.572Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to make requestVote RPC: target="{Voter server-aef27578-0e68-b160-b9b7-f17b6fd6c17e aef27578-0e68-b160-b9b7-f17b6fd6c17e}" error="failed to connect to peer: aef27578-0e68-b160-b9b7-f17b6fd6c17e" + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.572Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to make requestVote RPC: target="{Voter server-aef27578-0e68-b160-b9b7-f17b6fd6c17e aef27578-0e68-b160-b9b7-f17b6fd6c17e}" error="failed to connect to peer: aef27578-0e68-b160-b9b7-f17b6fd6c17e" +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.572Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to make requestVote RPC: target="{Voter server-9c603e47-dcb5-0e30-0afa-aaca8df23da9 9c603e47-dcb5-0e30-0afa-aaca8df23da9}" error="failed to connect to peer: 9c603e47-dcb5-0e30-0afa-aaca8df23da9" + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.572Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to make requestVote RPC: target="{Voter server-9c603e47-dcb5-0e30-0afa-aaca8df23da9 9c603e47-dcb5-0e30-0afa-aaca8df23da9}" error="failed to connect to peer: 9c603e47-dcb5-0e30-0afa-aaca8df23da9" +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.612Z [WARN] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: Election timeout reached, restarting election + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.612Z [WARN] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: Election timeout reached, restarting election +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.612Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: entering candidate state: node="Node at 9c603e47-dcb5-0e30-0afa-aaca8df23da9 [Candidate]" term=4 + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.612Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: entering candidate state: node="Node at 9c603e47-dcb5-0e30-0afa-aaca8df23da9 [Candidate]" term=4 +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.612Z [ERROR] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: failed to make requestVote RPC: target="{Voter server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9}" error="failed to connect to peer: 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9" + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.612Z [ERROR] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: failed to make requestVote RPC: target="{Voter server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9}" error="failed to connect to peer: 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9" +testLoggerAdapter verbose: server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: 2024-01-08T00:30:59.612Z [INFO] server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: entering follower state: follower="Node at aef27578-0e68-b160-b9b7-f17b6fd6c17e [Follower]" leader= + testing.go:149: server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: 2024-01-08T00:30:59.612Z [INFO] server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: entering follower state: follower="Node at aef27578-0e68-b160-b9b7-f17b6fd6c17e [Follower]" leader= +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.612Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: election won: tally=2 + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.612Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: election won: tally=2 +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.612Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: entering leader state: leader="Node at 9c603e47-dcb5-0e30-0afa-aaca8df23da9 [Leader]" + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.612Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: entering leader state: leader="Node at 9c603e47-dcb5-0e30-0afa-aaca8df23da9 [Leader]" +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.612Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: added peer, starting replication: peer=server-aef27578-0e68-b160-b9b7-f17b6fd6c17e + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.612Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: added peer, starting replication: peer=server-aef27578-0e68-b160-b9b7-f17b6fd6c17e +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.613Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: added peer, starting replication: peer=server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.613Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: added peer, starting replication: peer=server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.613Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: pipelining replication: peer="{Voter server-aef27578-0e68-b160-b9b7-f17b6fd6c17e aef27578-0e68-b160-b9b7-f17b6fd6c17e}" + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.613Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: pipelining replication: peer="{Voter server-aef27578-0e68-b160-b9b7-f17b6fd6c17e aef27578-0e68-b160-b9b7-f17b6fd6c17e}" +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.613Z [ERROR] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: failed to appendEntries to: peer="{Voter server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9}" error="failed to connect to peer: 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9" + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.613Z [ERROR] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: failed to appendEntries to: peer="{Voter server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9}" error="failed to connect to peer: 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9" +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.621Z [ERROR] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: failed to heartbeat to: peer=4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 error="failed to connect to peer: 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9" + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.621Z [ERROR] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: failed to heartbeat to: peer=4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 error="failed to connect to peer: 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9" +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.633Z [ERROR] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: failed to appendEntries to: peer="{Voter server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9}" error="failed to connect to peer: 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9" + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.633Z [ERROR] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: failed to appendEntries to: peer="{Voter server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9}" error="failed to connect to peer: 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9" +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.639Z [ERROR] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: failed to heartbeat to: peer=4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 error="failed to connect to peer: 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9" + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.639Z [ERROR] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: failed to heartbeat to: peer=4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 error="failed to connect to peer: 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9" +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.653Z [ERROR] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: failed to appendEntries to: peer="{Voter server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9}" error="failed to connect to peer: 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9" + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.653Z [ERROR] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: failed to appendEntries to: peer="{Voter server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9}" error="failed to connect to peer: 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9" +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.660Z [ERROR] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: failed to heartbeat to: peer=4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 error="failed to connect to peer: 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9" + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.660Z [ERROR] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: failed to heartbeat to: peer=4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 error="failed to connect to peer: 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9" +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.663Z [WARN] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: failed to contact: server-id=server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 time=50.094154ms + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.663Z [WARN] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: failed to contact: server-id=server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 time=50.094154ms +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.668Z [WARN] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: Election timeout reached, restarting election + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.668Z [WARN] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: Election timeout reached, restarting election +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.668Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: entering candidate state: node="Node at 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 [Candidate]" term=4 + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.668Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: entering candidate state: node="Node at 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 [Candidate]" term=4 +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.668Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to make requestVote RPC: target="{Voter server-aef27578-0e68-b160-b9b7-f17b6fd6c17e aef27578-0e68-b160-b9b7-f17b6fd6c17e}" error="failed to connect to peer: aef27578-0e68-b160-b9b7-f17b6fd6c17e" + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.668Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to make requestVote RPC: target="{Voter server-aef27578-0e68-b160-b9b7-f17b6fd6c17e aef27578-0e68-b160-b9b7-f17b6fd6c17e}" error="failed to connect to peer: aef27578-0e68-b160-b9b7-f17b6fd6c17e" +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.668Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to make requestVote RPC: target="{Voter server-9c603e47-dcb5-0e30-0afa-aaca8df23da9 9c603e47-dcb5-0e30-0afa-aaca8df23da9}" error="failed to connect to peer: 9c603e47-dcb5-0e30-0afa-aaca8df23da9" + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.668Z [ERROR] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to make requestVote RPC: target="{Voter server-9c603e47-dcb5-0e30-0afa-aaca8df23da9 9c603e47-dcb5-0e30-0afa-aaca8df23da9}" error="failed to connect to peer: 9c603e47-dcb5-0e30-0afa-aaca8df23da9" +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.679Z [ERROR] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: failed to appendEntries to: peer="{Voter server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9}" error="failed to connect to peer: 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9" + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.679Z [ERROR] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: failed to appendEntries to: peer="{Voter server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9}" error="failed to connect to peer: 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9" +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.691Z [ERROR] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: failed to heartbeat to: peer=4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 error="failed to connect to peer: 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9" + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.691Z [ERROR] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: failed to heartbeat to: peer=4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 error="failed to connect to peer: 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9" +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.712Z [WARN] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: failed to contact: server-id=server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 time=99.58262ms + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.712Z [WARN] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: failed to contact: server-id=server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 time=99.58262ms +testLoggerAdapter verbose: cluster: 2024-01-08T00:30:59.718Z [INFO] stable state for Leader reached at 2024-01-08 00:30:59.612988044 +0000 UTC m=+5.659877252 (1 nodes), 75.757545ms from start of poll, 482.252134ms from cluster start. Timeout at 2024-01-08 00:30:59.718179924 +0000 UTC m=+5.765069007, 105.191755ms after stability + testing.go:149: cluster: 2024-01-08T00:30:59.718Z [INFO] stable state for Leader reached at 2024-01-08 00:30:59.612988044 +0000 UTC m=+5.659877252 (1 nodes), 75.757545ms from start of poll, 482.252134ms from cluster start. Timeout at 2024-01-08 00:30:59.718179924 +0000 UTC m=+5.765069007, 105.191755ms after stability + raft_test.go:371: [INFO] Reconnecting Node at 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 [Candidate] +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.724Z [WARN] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: Election timeout reached, restarting election + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.724Z [WARN] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: Election timeout reached, restarting election +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.725Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: entering candidate state: node="Node at 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 [Candidate]" term=5 + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.725Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: entering candidate state: node="Node at 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 [Candidate]" term=5 +testLoggerAdapter verbose: server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: 2024-01-08T00:30:59.725Z [WARN] server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: rejecting vote request since we have a leader: from=4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 leader=9c603e47-dcb5-0e30-0afa-aaca8df23da9 + testing.go:149: server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: 2024-01-08T00:30:59.725Z [WARN] server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: rejecting vote request since we have a leader: from=4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 leader=9c603e47-dcb5-0e30-0afa-aaca8df23da9 +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.725Z [WARN] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: rejecting vote request since we have a leader: from=4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 leader=9c603e47-dcb5-0e30-0afa-aaca8df23da9 + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.725Z [WARN] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: rejecting vote request since we have a leader: from=4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 leader=9c603e47-dcb5-0e30-0afa-aaca8df23da9 +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.729Z [ERROR] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: peer has newer term, stopping replication: peer="{Voter server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9}" + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.729Z [ERROR] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: peer has newer term, stopping replication: peer="{Voter server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9}" +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.729Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: entering follower state: follower="Node at 9c603e47-dcb5-0e30-0afa-aaca8df23da9 [Follower]" leader= + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.729Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: entering follower state: follower="Node at 9c603e47-dcb5-0e30-0afa-aaca8df23da9 [Follower]" leader= +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.729Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: aborting pipeline replication: peer="{Voter server-aef27578-0e68-b160-b9b7-f17b6fd6c17e aef27578-0e68-b160-b9b7-f17b6fd6c17e}" + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.729Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: aborting pipeline replication: peer="{Voter server-aef27578-0e68-b160-b9b7-f17b6fd6c17e aef27578-0e68-b160-b9b7-f17b6fd6c17e}" +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.776Z [WARN] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: Election timeout reached, restarting election + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.776Z [WARN] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: Election timeout reached, restarting election +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.776Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: entering candidate state: node="Node at 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 [Candidate]" term=6 + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.776Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: entering candidate state: node="Node at 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 [Candidate]" term=6 +testLoggerAdapter verbose: server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: 2024-01-08T00:30:59.776Z [WARN] server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: rejecting vote request since we have a leader: from=4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 leader=9c603e47-dcb5-0e30-0afa-aaca8df23da9 +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.776Z [WARN] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: rejecting vote request since our last term is greater: candidate=4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 last-term=4 last-candidate-term=2 + testing.go:149: server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: 2024-01-08T00:30:59.776Z [WARN] server-aef27578-0e68-b160-b9b7-f17b6fd6c17e: rejecting vote request since we have a leader: from=4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 leader=9c603e47-dcb5-0e30-0afa-aaca8df23da9 + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.776Z [WARN] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: rejecting vote request since our last term is greater: candidate=4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 last-term=4 last-candidate-term=2 +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.781Z [WARN] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.781Z [WARN] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.781Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: entering candidate state: node="Node at 9c603e47-dcb5-0e30-0afa-aaca8df23da9 [Candidate]" term=7 + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.781Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: entering candidate state: node="Node at 9c603e47-dcb5-0e30-0afa-aaca8df23da9 [Candidate]" term=7 +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.782Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: entering follower state: follower="Node at 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 [Follower]" leader= + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.782Z [INFO] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: entering follower state: follower="Node at 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 [Follower]" leader= +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.782Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: election won: tally=2 + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.782Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: election won: tally=2 +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.782Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: entering leader state: leader="Node at 9c603e47-dcb5-0e30-0afa-aaca8df23da9 [Leader]" + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.782Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: entering leader state: leader="Node at 9c603e47-dcb5-0e30-0afa-aaca8df23da9 [Leader]" +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.782Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: added peer, starting replication: peer=server-aef27578-0e68-b160-b9b7-f17b6fd6c17e + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.782Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: added peer, starting replication: peer=server-aef27578-0e68-b160-b9b7-f17b6fd6c17e +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.782Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: added peer, starting replication: peer=server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.782Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: added peer, starting replication: peer=server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 +testLoggerAdapter verbose: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.782Z [WARN] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to get previous log: previous-index=5 last-index=3 error="log not found" + testing.go:149: server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: 2024-01-08T00:30:59.782Z [WARN] server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9: failed to get previous log: previous-index=5 last-index=3 error="log not found" +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.782Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: pipelining replication: peer="{Voter server-aef27578-0e68-b160-b9b7-f17b6fd6c17e aef27578-0e68-b160-b9b7-f17b6fd6c17e}" + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.782Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: pipelining replication: peer="{Voter server-aef27578-0e68-b160-b9b7-f17b6fd6c17e aef27578-0e68-b160-b9b7-f17b6fd6c17e}" +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.782Z [WARN] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: appendEntries rejected, sending older logs: peer="{Voter server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9}" next=4 + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.782Z [WARN] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: appendEntries rejected, sending older logs: peer="{Voter server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9}" next=4 +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.782Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: pipelining replication: peer="{Voter server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9}" + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.782Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: pipelining replication: peer="{Voter server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9}" +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.788Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: aborting pipeline replication: peer="{Voter server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9}" + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.788Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: aborting pipeline replication: peer="{Voter server-4ff51c45-01a1-521b-0e7f-4a59ba7c74c9 4ff51c45-01a1-521b-0e7f-4a59ba7c74c9}" +testLoggerAdapter verbose: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.788Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: aborting pipeline replication: peer="{Voter server-aef27578-0e68-b160-b9b7-f17b6fd6c17e aef27578-0e68-b160-b9b7-f17b6fd6c17e}" + testing.go:149: server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: 2024-01-08T00:30:59.788Z [INFO] server-9c603e47-dcb5-0e30-0afa-aaca8df23da9: aborting pipeline replication: peer="{Voter server-aef27578-0e68-b160-b9b7-f17b6fd6c17e aef27578-0e68-b160-b9b7-f17b6fd6c17e}" +--- PASS: TestRaft_LeaderFail (0.66s) === RUN TestRaft_BehindFollower -testLoggerAdapter verbose: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:24.701Z [INFO] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd Address:5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd} {Suffrage:Voter ID:server-0c532726-50ee-1ccc-2cd1-a3fbeb54fc02 Address:0c532726-50ee-1ccc-2cd1-a3fbeb54fc02} {Suffrage:Voter ID:server-4451b7fd-c69b-caca-673d-3bf91c2d7ed0 Address:4451b7fd-c69b-caca-673d-3bf91c2d7ed0}]" - testing.go:149: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:24.701Z [INFO] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd Address:5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd} {Suffrage:Voter ID:server-0c532726-50ee-1ccc-2cd1-a3fbeb54fc02 Address:0c532726-50ee-1ccc-2cd1-a3fbeb54fc02} {Suffrage:Voter ID:server-4451b7fd-c69b-caca-673d-3bf91c2d7ed0 Address:4451b7fd-c69b-caca-673d-3bf91c2d7ed0}]" -testLoggerAdapter verbose: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:24.701Z [INFO] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: entering follower state: follower="Node at 5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd [Follower]" leader= - testing.go:149: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:24.701Z [INFO] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: entering follower state: follower="Node at 5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd [Follower]" leader= -testLoggerAdapter verbose: server-0c532726-50ee-1ccc-2cd1-a3fbeb54fc02: 2025-02-09T06:52:24.701Z [INFO] server-0c532726-50ee-1ccc-2cd1-a3fbeb54fc02: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd Address:5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd} {Suffrage:Voter ID:server-0c532726-50ee-1ccc-2cd1-a3fbeb54fc02 Address:0c532726-50ee-1ccc-2cd1-a3fbeb54fc02} {Suffrage:Voter ID:server-4451b7fd-c69b-caca-673d-3bf91c2d7ed0 Address:4451b7fd-c69b-caca-673d-3bf91c2d7ed0}]" - testing.go:149: server-0c532726-50ee-1ccc-2cd1-a3fbeb54fc02: 2025-02-09T06:52:24.701Z [INFO] server-0c532726-50ee-1ccc-2cd1-a3fbeb54fc02: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd Address:5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd} {Suffrage:Voter ID:server-0c532726-50ee-1ccc-2cd1-a3fbeb54fc02 Address:0c532726-50ee-1ccc-2cd1-a3fbeb54fc02} {Suffrage:Voter ID:server-4451b7fd-c69b-caca-673d-3bf91c2d7ed0 Address:4451b7fd-c69b-caca-673d-3bf91c2d7ed0}]" -testLoggerAdapter verbose: server-0c532726-50ee-1ccc-2cd1-a3fbeb54fc02: 2025-02-09T06:52:24.701Z [INFO] server-0c532726-50ee-1ccc-2cd1-a3fbeb54fc02: entering follower state: follower="Node at 0c532726-50ee-1ccc-2cd1-a3fbeb54fc02 [Follower]" leader= - testing.go:149: server-0c532726-50ee-1ccc-2cd1-a3fbeb54fc02: 2025-02-09T06:52:24.701Z [INFO] server-0c532726-50ee-1ccc-2cd1-a3fbeb54fc02: entering follower state: follower="Node at 0c532726-50ee-1ccc-2cd1-a3fbeb54fc02 [Follower]" leader= -testLoggerAdapter verbose: server-4451b7fd-c69b-caca-673d-3bf91c2d7ed0: 2025-02-09T06:52:24.702Z [INFO] server-4451b7fd-c69b-caca-673d-3bf91c2d7ed0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd Address:5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd} {Suffrage:Voter ID:server-0c532726-50ee-1ccc-2cd1-a3fbeb54fc02 Address:0c532726-50ee-1ccc-2cd1-a3fbeb54fc02} {Suffrage:Voter ID:server-4451b7fd-c69b-caca-673d-3bf91c2d7ed0 Address:4451b7fd-c69b-caca-673d-3bf91c2d7ed0}]" - testing.go:149: server-4451b7fd-c69b-caca-673d-3bf91c2d7ed0: 2025-02-09T06:52:24.702Z [INFO] server-4451b7fd-c69b-caca-673d-3bf91c2d7ed0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd Address:5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd} {Suffrage:Voter ID:server-0c532726-50ee-1ccc-2cd1-a3fbeb54fc02 Address:0c532726-50ee-1ccc-2cd1-a3fbeb54fc02} {Suffrage:Voter ID:server-4451b7fd-c69b-caca-673d-3bf91c2d7ed0 Address:4451b7fd-c69b-caca-673d-3bf91c2d7ed0}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:24.702Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:24.702Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-4451b7fd-c69b-caca-673d-3bf91c2d7ed0: 2025-02-09T06:52:24.702Z [INFO] server-4451b7fd-c69b-caca-673d-3bf91c2d7ed0: entering follower state: follower="Node at 4451b7fd-c69b-caca-673d-3bf91c2d7ed0 [Follower]" leader= - testing.go:149: server-4451b7fd-c69b-caca-673d-3bf91c2d7ed0: 2025-02-09T06:52:24.702Z [INFO] server-4451b7fd-c69b-caca-673d-3bf91c2d7ed0: entering follower state: follower="Node at 4451b7fd-c69b-caca-673d-3bf91c2d7ed0 [Follower]" leader= -testLoggerAdapter verbose: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:24.767Z [WARN] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:24.767Z [WARN] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:24.767Z [INFO] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: entering candidate state: node="Node at 5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd [Candidate]" term=2 - testing.go:149: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:24.767Z [INFO] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: entering candidate state: node="Node at 5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd [Candidate]" term=2 -testLoggerAdapter verbose: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:24.767Z [INFO] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: election won: tally=2 - testing.go:149: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:24.767Z [INFO] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: election won: tally=2 -testLoggerAdapter verbose: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:24.767Z [INFO] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: entering leader state: leader="Node at 5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd [Leader]" - testing.go:149: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:24.767Z [INFO] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: entering leader state: leader="Node at 5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd [Leader]" -testLoggerAdapter verbose: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:24.767Z [INFO] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: added peer, starting replication: peer=server-0c532726-50ee-1ccc-2cd1-a3fbeb54fc02 - testing.go:149: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:24.767Z [INFO] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: added peer, starting replication: peer=server-0c532726-50ee-1ccc-2cd1-a3fbeb54fc02 -testLoggerAdapter verbose: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:24.767Z [INFO] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: added peer, starting replication: peer=server-4451b7fd-c69b-caca-673d-3bf91c2d7ed0 - testing.go:149: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:24.767Z [INFO] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: added peer, starting replication: peer=server-4451b7fd-c69b-caca-673d-3bf91c2d7ed0 -testLoggerAdapter verbose: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:24.767Z [INFO] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: pipelining replication: peer="{Voter server-0c532726-50ee-1ccc-2cd1-a3fbeb54fc02 0c532726-50ee-1ccc-2cd1-a3fbeb54fc02}" - testing.go:149: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:24.767Z [INFO] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: pipelining replication: peer="{Voter server-0c532726-50ee-1ccc-2cd1-a3fbeb54fc02 0c532726-50ee-1ccc-2cd1-a3fbeb54fc02}" -testLoggerAdapter verbose: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:24.768Z [INFO] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: pipelining replication: peer="{Voter server-4451b7fd-c69b-caca-673d-3bf91c2d7ed0 4451b7fd-c69b-caca-673d-3bf91c2d7ed0}" - testing.go:149: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:24.768Z [INFO] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: pipelining replication: peer="{Voter server-4451b7fd-c69b-caca-673d-3bf91c2d7ed0 4451b7fd-c69b-caca-673d-3bf91c2d7ed0}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:24.873Z [INFO] stable state for Leader reached at 2025-02-09 06:52:24.767954859 +0000 UTC m=+5.746763632 (1 nodes), 65.85725ms from start of poll, 66.302732ms from cluster start. Timeout at 2025-02-09 06:52:24.873505288 +0000 UTC m=+5.852314063, 105.550431ms after stability - testing.go:149: cluster: 2025-02-09T06:52:24.873Z [INFO] stable state for Leader reached at 2025-02-09 06:52:24.767954859 +0000 UTC m=+5.746763632 (1 nodes), 65.85725ms from start of poll, 66.302732ms from cluster start. Timeout at 2025-02-09 06:52:24.873505288 +0000 UTC m=+5.852314063, 105.550431ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:24.873Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T06:52:24.873Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:24.979Z [INFO] stable state for Follower reached at 2025-02-09 06:52:24.873597771 +0000 UTC m=+5.852406541 (2 nodes), 684ns from start of poll, 171.945641ms from cluster start. Timeout at 2025-02-09 06:52:24.979425396 +0000 UTC m=+5.958234165, 105.827624ms after stability - testing.go:149: cluster: 2025-02-09T06:52:24.979Z [INFO] stable state for Follower reached at 2025-02-09 06:52:24.873597771 +0000 UTC m=+5.852406541 (2 nodes), 684ns from start of poll, 171.945641ms from cluster start. Timeout at 2025-02-09 06:52:24.979425396 +0000 UTC m=+5.958234165, 105.827624ms after stability -testLoggerAdapter verbose: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:24.979Z [ERROR] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: failed to pipeline appendEntries: peer="{Voter server-0c532726-50ee-1ccc-2cd1-a3fbeb54fc02 0c532726-50ee-1ccc-2cd1-a3fbeb54fc02}" error="append pipeline closed" - testing.go:149: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:24.979Z [ERROR] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: failed to pipeline appendEntries: peer="{Voter server-0c532726-50ee-1ccc-2cd1-a3fbeb54fc02 0c532726-50ee-1ccc-2cd1-a3fbeb54fc02}" error="append pipeline closed" -testLoggerAdapter verbose: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:24.979Z [INFO] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: aborting pipeline replication: peer="{Voter server-0c532726-50ee-1ccc-2cd1-a3fbeb54fc02 0c532726-50ee-1ccc-2cd1-a3fbeb54fc02}" - testing.go:149: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:24.979Z [INFO] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: aborting pipeline replication: peer="{Voter server-0c532726-50ee-1ccc-2cd1-a3fbeb54fc02 0c532726-50ee-1ccc-2cd1-a3fbeb54fc02}" -testLoggerAdapter verbose: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:24.979Z [ERROR] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: failed to appendEntries to: peer="{Voter server-0c532726-50ee-1ccc-2cd1-a3fbeb54fc02 0c532726-50ee-1ccc-2cd1-a3fbeb54fc02}" error="failed to connect to peer: 0c532726-50ee-1ccc-2cd1-a3fbeb54fc02" - testing.go:149: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:24.979Z [ERROR] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: failed to appendEntries to: peer="{Voter server-0c532726-50ee-1ccc-2cd1-a3fbeb54fc02 0c532726-50ee-1ccc-2cd1-a3fbeb54fc02}" error="failed to connect to peer: 0c532726-50ee-1ccc-2cd1-a3fbeb54fc02" +testLoggerAdapter verbose: server-3d691ad1-3554-8502-a80d-ba74a8ccad96: 2024-01-08T00:30:59.789Z [INFO] server-3d691ad1-3554-8502-a80d-ba74a8ccad96: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3d691ad1-3554-8502-a80d-ba74a8ccad96 Address:3d691ad1-3554-8502-a80d-ba74a8ccad96} {Suffrage:Voter ID:server-2332666b-cbed-2be3-6042-5e64ba1cd321 Address:2332666b-cbed-2be3-6042-5e64ba1cd321} {Suffrage:Voter ID:server-06746be3-e087-5413-084f-0a85eafee938 Address:06746be3-e087-5413-084f-0a85eafee938}]" + testing.go:149: server-3d691ad1-3554-8502-a80d-ba74a8ccad96: 2024-01-08T00:30:59.789Z [INFO] server-3d691ad1-3554-8502-a80d-ba74a8ccad96: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3d691ad1-3554-8502-a80d-ba74a8ccad96 Address:3d691ad1-3554-8502-a80d-ba74a8ccad96} {Suffrage:Voter ID:server-2332666b-cbed-2be3-6042-5e64ba1cd321 Address:2332666b-cbed-2be3-6042-5e64ba1cd321} {Suffrage:Voter ID:server-06746be3-e087-5413-084f-0a85eafee938 Address:06746be3-e087-5413-084f-0a85eafee938}]" +testLoggerAdapter verbose: server-3d691ad1-3554-8502-a80d-ba74a8ccad96: 2024-01-08T00:30:59.789Z [INFO] server-3d691ad1-3554-8502-a80d-ba74a8ccad96: entering follower state: follower="Node at 3d691ad1-3554-8502-a80d-ba74a8ccad96 [Follower]" leader= +testLoggerAdapter verbose: server-2332666b-cbed-2be3-6042-5e64ba1cd321: 2024-01-08T00:30:59.789Z [INFO] server-2332666b-cbed-2be3-6042-5e64ba1cd321: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3d691ad1-3554-8502-a80d-ba74a8ccad96 Address:3d691ad1-3554-8502-a80d-ba74a8ccad96} {Suffrage:Voter ID:server-2332666b-cbed-2be3-6042-5e64ba1cd321 Address:2332666b-cbed-2be3-6042-5e64ba1cd321} {Suffrage:Voter ID:server-06746be3-e087-5413-084f-0a85eafee938 Address:06746be3-e087-5413-084f-0a85eafee938}]" + testing.go:149: server-3d691ad1-3554-8502-a80d-ba74a8ccad96: 2024-01-08T00:30:59.789Z [INFO] server-3d691ad1-3554-8502-a80d-ba74a8ccad96: entering follower state: follower="Node at 3d691ad1-3554-8502-a80d-ba74a8ccad96 [Follower]" leader= + testing.go:149: server-2332666b-cbed-2be3-6042-5e64ba1cd321: 2024-01-08T00:30:59.789Z [INFO] server-2332666b-cbed-2be3-6042-5e64ba1cd321: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3d691ad1-3554-8502-a80d-ba74a8ccad96 Address:3d691ad1-3554-8502-a80d-ba74a8ccad96} {Suffrage:Voter ID:server-2332666b-cbed-2be3-6042-5e64ba1cd321 Address:2332666b-cbed-2be3-6042-5e64ba1cd321} {Suffrage:Voter ID:server-06746be3-e087-5413-084f-0a85eafee938 Address:06746be3-e087-5413-084f-0a85eafee938}]" +testLoggerAdapter verbose: server-2332666b-cbed-2be3-6042-5e64ba1cd321: 2024-01-08T00:30:59.789Z [INFO] server-2332666b-cbed-2be3-6042-5e64ba1cd321: entering follower state: follower="Node at 2332666b-cbed-2be3-6042-5e64ba1cd321 [Follower]" leader= +testLoggerAdapter verbose: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:30:59.789Z [INFO] server-06746be3-e087-5413-084f-0a85eafee938: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3d691ad1-3554-8502-a80d-ba74a8ccad96 Address:3d691ad1-3554-8502-a80d-ba74a8ccad96} {Suffrage:Voter ID:server-2332666b-cbed-2be3-6042-5e64ba1cd321 Address:2332666b-cbed-2be3-6042-5e64ba1cd321} {Suffrage:Voter ID:server-06746be3-e087-5413-084f-0a85eafee938 Address:06746be3-e087-5413-084f-0a85eafee938}]" + testing.go:149: server-2332666b-cbed-2be3-6042-5e64ba1cd321: 2024-01-08T00:30:59.789Z [INFO] server-2332666b-cbed-2be3-6042-5e64ba1cd321: entering follower state: follower="Node at 2332666b-cbed-2be3-6042-5e64ba1cd321 [Follower]" leader= + testing.go:149: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:30:59.789Z [INFO] server-06746be3-e087-5413-084f-0a85eafee938: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3d691ad1-3554-8502-a80d-ba74a8ccad96 Address:3d691ad1-3554-8502-a80d-ba74a8ccad96} {Suffrage:Voter ID:server-2332666b-cbed-2be3-6042-5e64ba1cd321 Address:2332666b-cbed-2be3-6042-5e64ba1cd321} {Suffrage:Voter ID:server-06746be3-e087-5413-084f-0a85eafee938 Address:06746be3-e087-5413-084f-0a85eafee938}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:30:59.789Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:30:59.789Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:30:59.789Z [INFO] server-06746be3-e087-5413-084f-0a85eafee938: entering follower state: follower="Node at 06746be3-e087-5413-084f-0a85eafee938 [Follower]" leader= + testing.go:149: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:30:59.789Z [INFO] server-06746be3-e087-5413-084f-0a85eafee938: entering follower state: follower="Node at 06746be3-e087-5413-084f-0a85eafee938 [Follower]" leader= +testLoggerAdapter verbose: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:30:59.867Z [WARN] server-06746be3-e087-5413-084f-0a85eafee938: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:30:59.867Z [WARN] server-06746be3-e087-5413-084f-0a85eafee938: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:30:59.867Z [INFO] server-06746be3-e087-5413-084f-0a85eafee938: entering candidate state: node="Node at 06746be3-e087-5413-084f-0a85eafee938 [Candidate]" term=2 + testing.go:149: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:30:59.867Z [INFO] server-06746be3-e087-5413-084f-0a85eafee938: entering candidate state: node="Node at 06746be3-e087-5413-084f-0a85eafee938 [Candidate]" term=2 +testLoggerAdapter verbose: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:30:59.867Z [INFO] server-06746be3-e087-5413-084f-0a85eafee938: election won: tally=2 + testing.go:149: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:30:59.867Z [INFO] server-06746be3-e087-5413-084f-0a85eafee938: election won: tally=2 +testLoggerAdapter verbose: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:30:59.867Z [INFO] server-06746be3-e087-5413-084f-0a85eafee938: entering leader state: leader="Node at 06746be3-e087-5413-084f-0a85eafee938 [Leader]" + testing.go:149: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:30:59.867Z [INFO] server-06746be3-e087-5413-084f-0a85eafee938: entering leader state: leader="Node at 06746be3-e087-5413-084f-0a85eafee938 [Leader]" +testLoggerAdapter verbose: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:30:59.867Z [INFO] server-06746be3-e087-5413-084f-0a85eafee938: added peer, starting replication: peer=server-3d691ad1-3554-8502-a80d-ba74a8ccad96 + testing.go:149: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:30:59.867Z [INFO] server-06746be3-e087-5413-084f-0a85eafee938: added peer, starting replication: peer=server-3d691ad1-3554-8502-a80d-ba74a8ccad96 +testLoggerAdapter verbose: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:30:59.867Z [INFO] server-06746be3-e087-5413-084f-0a85eafee938: added peer, starting replication: peer=server-2332666b-cbed-2be3-6042-5e64ba1cd321 + testing.go:149: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:30:59.867Z [INFO] server-06746be3-e087-5413-084f-0a85eafee938: added peer, starting replication: peer=server-2332666b-cbed-2be3-6042-5e64ba1cd321 +testLoggerAdapter verbose: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:30:59.867Z [INFO] server-06746be3-e087-5413-084f-0a85eafee938: pipelining replication: peer="{Voter server-3d691ad1-3554-8502-a80d-ba74a8ccad96 3d691ad1-3554-8502-a80d-ba74a8ccad96}" + testing.go:149: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:30:59.867Z [INFO] server-06746be3-e087-5413-084f-0a85eafee938: pipelining replication: peer="{Voter server-3d691ad1-3554-8502-a80d-ba74a8ccad96 3d691ad1-3554-8502-a80d-ba74a8ccad96}" +testLoggerAdapter verbose: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:30:59.867Z [INFO] server-06746be3-e087-5413-084f-0a85eafee938: pipelining replication: peer="{Voter server-2332666b-cbed-2be3-6042-5e64ba1cd321 2332666b-cbed-2be3-6042-5e64ba1cd321}" + testing.go:149: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:30:59.867Z [INFO] server-06746be3-e087-5413-084f-0a85eafee938: pipelining replication: peer="{Voter server-2332666b-cbed-2be3-6042-5e64ba1cd321 2332666b-cbed-2be3-6042-5e64ba1cd321}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:30:59.972Z [INFO] stable state for Leader reached at 2024-01-08 00:30:59.867712268 +0000 UTC m=+5.914601325 (1 nodes), 77.726084ms from start of poll, 78.310338ms from cluster start. Timeout at 2024-01-08 00:30:59.97287267 +0000 UTC m=+6.019761714, 105.160389ms after stability + testing.go:149: cluster: 2024-01-08T00:30:59.972Z [INFO] stable state for Leader reached at 2024-01-08 00:30:59.867712268 +0000 UTC m=+5.914601325 (1 nodes), 77.726084ms from start of poll, 78.310338ms from cluster start. Timeout at 2024-01-08 00:30:59.97287267 +0000 UTC m=+6.019761714, 105.160389ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T00:30:59.972Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T00:30:59.972Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:00.078Z [INFO] stable state for Follower reached at 2024-01-08 00:30:59.973000959 +0000 UTC m=+6.019890014 (2 nodes), 2.153µs from start of poll, 183.599027ms from cluster start. Timeout at 2024-01-08 00:31:00.078007259 +0000 UTC m=+6.124896322, 105.006308ms after stability + testing.go:149: cluster: 2024-01-08T00:31:00.078Z [INFO] stable state for Follower reached at 2024-01-08 00:30:59.973000959 +0000 UTC m=+6.019890014 (2 nodes), 2.153µs from start of poll, 183.599027ms from cluster start. Timeout at 2024-01-08 00:31:00.078007259 +0000 UTC m=+6.124896322, 105.006308ms after stability +testLoggerAdapter verbose: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:31:00.078Z [ERROR] server-06746be3-e087-5413-084f-0a85eafee938: failed to pipeline appendEntries: peer="{Voter server-3d691ad1-3554-8502-a80d-ba74a8ccad96 3d691ad1-3554-8502-a80d-ba74a8ccad96}" error="append pipeline closed" + testing.go:149: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:31:00.078Z [ERROR] server-06746be3-e087-5413-084f-0a85eafee938: failed to pipeline appendEntries: peer="{Voter server-3d691ad1-3554-8502-a80d-ba74a8ccad96 3d691ad1-3554-8502-a80d-ba74a8ccad96}" error="append pipeline closed" +testLoggerAdapter verbose: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:31:00.078Z [INFO] server-06746be3-e087-5413-084f-0a85eafee938: aborting pipeline replication: peer="{Voter server-3d691ad1-3554-8502-a80d-ba74a8ccad96 3d691ad1-3554-8502-a80d-ba74a8ccad96}" + testing.go:149: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:31:00.078Z [INFO] server-06746be3-e087-5413-084f-0a85eafee938: aborting pipeline replication: peer="{Voter server-3d691ad1-3554-8502-a80d-ba74a8ccad96 3d691ad1-3554-8502-a80d-ba74a8ccad96}" +testLoggerAdapter verbose: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:31:00.078Z [ERROR] server-06746be3-e087-5413-084f-0a85eafee938: failed to appendEntries to: peer="{Voter server-3d691ad1-3554-8502-a80d-ba74a8ccad96 3d691ad1-3554-8502-a80d-ba74a8ccad96}" error="failed to connect to peer: 3d691ad1-3554-8502-a80d-ba74a8ccad96" + testing.go:149: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:31:00.078Z [ERROR] server-06746be3-e087-5413-084f-0a85eafee938: failed to appendEntries to: peer="{Voter server-3d691ad1-3554-8502-a80d-ba74a8ccad96 3d691ad1-3554-8502-a80d-ba74a8ccad96}" error="failed to connect to peer: 3d691ad1-3554-8502-a80d-ba74a8ccad96" raft_test.go:420: [INFO] Finished apply without behind follower -testLoggerAdapter verbose: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:24.989Z [INFO] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: pipelining replication: peer="{Voter server-0c532726-50ee-1ccc-2cd1-a3fbeb54fc02 0c532726-50ee-1ccc-2cd1-a3fbeb54fc02}" - testing.go:149: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:24.989Z [INFO] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: pipelining replication: peer="{Voter server-0c532726-50ee-1ccc-2cd1-a3fbeb54fc02 0c532726-50ee-1ccc-2cd1-a3fbeb54fc02}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:25.002Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:25.002Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:25.108Z [INFO] stable state for Leader reached at 2025-02-09 06:52:25.002758626 +0000 UTC m=+5.981567394 (1 nodes), 478ns from start of poll, 301.106494ms from cluster start. Timeout at 2025-02-09 06:52:25.108448446 +0000 UTC m=+6.087257214, 105.68982ms after stability - testing.go:149: cluster: 2025-02-09T06:52:25.108Z [INFO] stable state for Leader reached at 2025-02-09 06:52:25.002758626 +0000 UTC m=+5.981567394 (1 nodes), 478ns from start of poll, 301.106494ms from cluster start. Timeout at 2025-02-09 06:52:25.108448446 +0000 UTC m=+6.087257214, 105.68982ms after stability -testLoggerAdapter verbose: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:25.108Z [INFO] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: aborting pipeline replication: peer="{Voter server-4451b7fd-c69b-caca-673d-3bf91c2d7ed0 4451b7fd-c69b-caca-673d-3bf91c2d7ed0}" - testing.go:149: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:25.108Z [INFO] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: aborting pipeline replication: peer="{Voter server-4451b7fd-c69b-caca-673d-3bf91c2d7ed0 4451b7fd-c69b-caca-673d-3bf91c2d7ed0}" -testLoggerAdapter verbose: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:25.108Z [INFO] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: aborting pipeline replication: peer="{Voter server-0c532726-50ee-1ccc-2cd1-a3fbeb54fc02 0c532726-50ee-1ccc-2cd1-a3fbeb54fc02}" - testing.go:149: server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: 2025-02-09T06:52:25.108Z [INFO] server-5284d03a-60d3-c4bb-f2ec-ef2156bbc8dd: aborting pipeline replication: peer="{Voter server-0c532726-50ee-1ccc-2cd1-a3fbeb54fc02 0c532726-50ee-1ccc-2cd1-a3fbeb54fc02}" ---- PASS: TestRaft_BehindFollower (0.41s) +testLoggerAdapter verbose: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:31:00.088Z [INFO] server-06746be3-e087-5413-084f-0a85eafee938: pipelining replication: peer="{Voter server-3d691ad1-3554-8502-a80d-ba74a8ccad96 3d691ad1-3554-8502-a80d-ba74a8ccad96}" + testing.go:149: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:31:00.088Z [INFO] server-06746be3-e087-5413-084f-0a85eafee938: pipelining replication: peer="{Voter server-3d691ad1-3554-8502-a80d-ba74a8ccad96 3d691ad1-3554-8502-a80d-ba74a8ccad96}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:00.097Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:00.097Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:00.203Z [INFO] stable state for Leader reached at 2024-01-08 00:31:00.097714237 +0000 UTC m=+6.144603273 (1 nodes), 1.815µs from start of poll, 308.312286ms from cluster start. Timeout at 2024-01-08 00:31:00.203381996 +0000 UTC m=+6.250271061, 105.667788ms after stability + testing.go:149: cluster: 2024-01-08T00:31:00.203Z [INFO] stable state for Leader reached at 2024-01-08 00:31:00.097714237 +0000 UTC m=+6.144603273 (1 nodes), 1.815µs from start of poll, 308.312286ms from cluster start. Timeout at 2024-01-08 00:31:00.203381996 +0000 UTC m=+6.250271061, 105.667788ms after stability +testLoggerAdapter verbose: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:31:00.203Z [INFO] server-06746be3-e087-5413-084f-0a85eafee938: aborting pipeline replication: peer="{Voter server-2332666b-cbed-2be3-6042-5e64ba1cd321 2332666b-cbed-2be3-6042-5e64ba1cd321}" + testing.go:149: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:31:00.203Z [INFO] server-06746be3-e087-5413-084f-0a85eafee938: aborting pipeline replication: peer="{Voter server-2332666b-cbed-2be3-6042-5e64ba1cd321 2332666b-cbed-2be3-6042-5e64ba1cd321}" +testLoggerAdapter verbose: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:31:00.203Z [INFO] server-06746be3-e087-5413-084f-0a85eafee938: aborting pipeline replication: peer="{Voter server-3d691ad1-3554-8502-a80d-ba74a8ccad96 3d691ad1-3554-8502-a80d-ba74a8ccad96}" + testing.go:149: server-06746be3-e087-5413-084f-0a85eafee938: 2024-01-08T00:31:00.203Z [INFO] server-06746be3-e087-5413-084f-0a85eafee938: aborting pipeline replication: peer="{Voter server-3d691ad1-3554-8502-a80d-ba74a8ccad96 3d691ad1-3554-8502-a80d-ba74a8ccad96}" +--- PASS: TestRaft_BehindFollower (0.42s) === RUN TestRaft_ApplyNonLeader -testLoggerAdapter verbose: server-24fc268c-7d21-51c8-02e1-3cd3597bc926: 2025-02-09T06:52:25.109Z [INFO] server-24fc268c-7d21-51c8-02e1-3cd3597bc926: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-24fc268c-7d21-51c8-02e1-3cd3597bc926 Address:24fc268c-7d21-51c8-02e1-3cd3597bc926} {Suffrage:Voter ID:server-b547d4e9-ca08-04ce-7a61-da70135403c1 Address:b547d4e9-ca08-04ce-7a61-da70135403c1} {Suffrage:Voter ID:server-bd572076-5f37-fe83-1979-73c4797ce710 Address:bd572076-5f37-fe83-1979-73c4797ce710}]" - testing.go:149: server-24fc268c-7d21-51c8-02e1-3cd3597bc926: 2025-02-09T06:52:25.109Z [INFO] server-24fc268c-7d21-51c8-02e1-3cd3597bc926: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-24fc268c-7d21-51c8-02e1-3cd3597bc926 Address:24fc268c-7d21-51c8-02e1-3cd3597bc926} {Suffrage:Voter ID:server-b547d4e9-ca08-04ce-7a61-da70135403c1 Address:b547d4e9-ca08-04ce-7a61-da70135403c1} {Suffrage:Voter ID:server-bd572076-5f37-fe83-1979-73c4797ce710 Address:bd572076-5f37-fe83-1979-73c4797ce710}]" -testLoggerAdapter verbose: server-b547d4e9-ca08-04ce-7a61-da70135403c1: 2025-02-09T06:52:25.109Z [INFO] server-b547d4e9-ca08-04ce-7a61-da70135403c1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-24fc268c-7d21-51c8-02e1-3cd3597bc926 Address:24fc268c-7d21-51c8-02e1-3cd3597bc926} {Suffrage:Voter ID:server-b547d4e9-ca08-04ce-7a61-da70135403c1 Address:b547d4e9-ca08-04ce-7a61-da70135403c1} {Suffrage:Voter ID:server-bd572076-5f37-fe83-1979-73c4797ce710 Address:bd572076-5f37-fe83-1979-73c4797ce710}]" - testing.go:149: server-b547d4e9-ca08-04ce-7a61-da70135403c1: 2025-02-09T06:52:25.109Z [INFO] server-b547d4e9-ca08-04ce-7a61-da70135403c1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-24fc268c-7d21-51c8-02e1-3cd3597bc926 Address:24fc268c-7d21-51c8-02e1-3cd3597bc926} {Suffrage:Voter ID:server-b547d4e9-ca08-04ce-7a61-da70135403c1 Address:b547d4e9-ca08-04ce-7a61-da70135403c1} {Suffrage:Voter ID:server-bd572076-5f37-fe83-1979-73c4797ce710 Address:bd572076-5f37-fe83-1979-73c4797ce710}]" -testLoggerAdapter verbose: server-bd572076-5f37-fe83-1979-73c4797ce710: 2025-02-09T06:52:25.109Z [INFO] server-bd572076-5f37-fe83-1979-73c4797ce710: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-24fc268c-7d21-51c8-02e1-3cd3597bc926 Address:24fc268c-7d21-51c8-02e1-3cd3597bc926} {Suffrage:Voter ID:server-b547d4e9-ca08-04ce-7a61-da70135403c1 Address:b547d4e9-ca08-04ce-7a61-da70135403c1} {Suffrage:Voter ID:server-bd572076-5f37-fe83-1979-73c4797ce710 Address:bd572076-5f37-fe83-1979-73c4797ce710}]" - testing.go:149: server-bd572076-5f37-fe83-1979-73c4797ce710: 2025-02-09T06:52:25.109Z [INFO] server-bd572076-5f37-fe83-1979-73c4797ce710: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-24fc268c-7d21-51c8-02e1-3cd3597bc926 Address:24fc268c-7d21-51c8-02e1-3cd3597bc926} {Suffrage:Voter ID:server-b547d4e9-ca08-04ce-7a61-da70135403c1 Address:b547d4e9-ca08-04ce-7a61-da70135403c1} {Suffrage:Voter ID:server-bd572076-5f37-fe83-1979-73c4797ce710 Address:bd572076-5f37-fe83-1979-73c4797ce710}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:25.109Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:25.109Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-24fc268c-7d21-51c8-02e1-3cd3597bc926: 2025-02-09T06:52:25.109Z [INFO] server-24fc268c-7d21-51c8-02e1-3cd3597bc926: entering follower state: follower="Node at 24fc268c-7d21-51c8-02e1-3cd3597bc926 [Follower]" leader= - testing.go:149: server-24fc268c-7d21-51c8-02e1-3cd3597bc926: 2025-02-09T06:52:25.109Z [INFO] server-24fc268c-7d21-51c8-02e1-3cd3597bc926: entering follower state: follower="Node at 24fc268c-7d21-51c8-02e1-3cd3597bc926 [Follower]" leader= -testLoggerAdapter verbose: server-b547d4e9-ca08-04ce-7a61-da70135403c1: 2025-02-09T06:52:25.109Z [INFO] server-b547d4e9-ca08-04ce-7a61-da70135403c1: entering follower state: follower="Node at b547d4e9-ca08-04ce-7a61-da70135403c1 [Follower]" leader= -testLoggerAdapter verbose: server-bd572076-5f37-fe83-1979-73c4797ce710: 2025-02-09T06:52:25.109Z [INFO] server-bd572076-5f37-fe83-1979-73c4797ce710: entering follower state: follower="Node at bd572076-5f37-fe83-1979-73c4797ce710 [Follower]" leader= - testing.go:149: server-b547d4e9-ca08-04ce-7a61-da70135403c1: 2025-02-09T06:52:25.109Z [INFO] server-b547d4e9-ca08-04ce-7a61-da70135403c1: entering follower state: follower="Node at b547d4e9-ca08-04ce-7a61-da70135403c1 [Follower]" leader= - testing.go:149: server-bd572076-5f37-fe83-1979-73c4797ce710: 2025-02-09T06:52:25.109Z [INFO] server-bd572076-5f37-fe83-1979-73c4797ce710: entering follower state: follower="Node at bd572076-5f37-fe83-1979-73c4797ce710 [Follower]" leader= -testLoggerAdapter verbose: server-bd572076-5f37-fe83-1979-73c4797ce710: 2025-02-09T06:52:25.178Z [WARN] server-bd572076-5f37-fe83-1979-73c4797ce710: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-bd572076-5f37-fe83-1979-73c4797ce710: 2025-02-09T06:52:25.178Z [WARN] server-bd572076-5f37-fe83-1979-73c4797ce710: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-bd572076-5f37-fe83-1979-73c4797ce710: 2025-02-09T06:52:25.178Z [INFO] server-bd572076-5f37-fe83-1979-73c4797ce710: entering candidate state: node="Node at bd572076-5f37-fe83-1979-73c4797ce710 [Candidate]" term=2 - testing.go:149: server-bd572076-5f37-fe83-1979-73c4797ce710: 2025-02-09T06:52:25.178Z [INFO] server-bd572076-5f37-fe83-1979-73c4797ce710: entering candidate state: node="Node at bd572076-5f37-fe83-1979-73c4797ce710 [Candidate]" term=2 -testLoggerAdapter verbose: server-bd572076-5f37-fe83-1979-73c4797ce710: 2025-02-09T06:52:25.178Z [INFO] server-bd572076-5f37-fe83-1979-73c4797ce710: election won: tally=2 - testing.go:149: server-bd572076-5f37-fe83-1979-73c4797ce710: 2025-02-09T06:52:25.178Z [INFO] server-bd572076-5f37-fe83-1979-73c4797ce710: election won: tally=2 -testLoggerAdapter verbose: server-bd572076-5f37-fe83-1979-73c4797ce710: 2025-02-09T06:52:25.178Z [INFO] server-bd572076-5f37-fe83-1979-73c4797ce710: entering leader state: leader="Node at bd572076-5f37-fe83-1979-73c4797ce710 [Leader]" - testing.go:149: server-bd572076-5f37-fe83-1979-73c4797ce710: 2025-02-09T06:52:25.178Z [INFO] server-bd572076-5f37-fe83-1979-73c4797ce710: entering leader state: leader="Node at bd572076-5f37-fe83-1979-73c4797ce710 [Leader]" -testLoggerAdapter verbose: server-bd572076-5f37-fe83-1979-73c4797ce710: 2025-02-09T06:52:25.178Z [INFO] server-bd572076-5f37-fe83-1979-73c4797ce710: added peer, starting replication: peer=server-24fc268c-7d21-51c8-02e1-3cd3597bc926 - testing.go:149: server-bd572076-5f37-fe83-1979-73c4797ce710: 2025-02-09T06:52:25.178Z [INFO] server-bd572076-5f37-fe83-1979-73c4797ce710: added peer, starting replication: peer=server-24fc268c-7d21-51c8-02e1-3cd3597bc926 -testLoggerAdapter verbose: server-bd572076-5f37-fe83-1979-73c4797ce710: 2025-02-09T06:52:25.178Z [INFO] server-bd572076-5f37-fe83-1979-73c4797ce710: added peer, starting replication: peer=server-b547d4e9-ca08-04ce-7a61-da70135403c1 - testing.go:149: server-bd572076-5f37-fe83-1979-73c4797ce710: 2025-02-09T06:52:25.178Z [INFO] server-bd572076-5f37-fe83-1979-73c4797ce710: added peer, starting replication: peer=server-b547d4e9-ca08-04ce-7a61-da70135403c1 -testLoggerAdapter verbose: server-bd572076-5f37-fe83-1979-73c4797ce710: 2025-02-09T06:52:25.178Z [INFO] server-bd572076-5f37-fe83-1979-73c4797ce710: pipelining replication: peer="{Voter server-24fc268c-7d21-51c8-02e1-3cd3597bc926 24fc268c-7d21-51c8-02e1-3cd3597bc926}" - testing.go:149: server-bd572076-5f37-fe83-1979-73c4797ce710: 2025-02-09T06:52:25.178Z [INFO] server-bd572076-5f37-fe83-1979-73c4797ce710: pipelining replication: peer="{Voter server-24fc268c-7d21-51c8-02e1-3cd3597bc926 24fc268c-7d21-51c8-02e1-3cd3597bc926}" -testLoggerAdapter verbose: server-bd572076-5f37-fe83-1979-73c4797ce710: 2025-02-09T06:52:25.178Z [INFO] server-bd572076-5f37-fe83-1979-73c4797ce710: pipelining replication: peer="{Voter server-b547d4e9-ca08-04ce-7a61-da70135403c1 b547d4e9-ca08-04ce-7a61-da70135403c1}" - testing.go:149: server-bd572076-5f37-fe83-1979-73c4797ce710: 2025-02-09T06:52:25.178Z [INFO] server-bd572076-5f37-fe83-1979-73c4797ce710: pipelining replication: peer="{Voter server-b547d4e9-ca08-04ce-7a61-da70135403c1 b547d4e9-ca08-04ce-7a61-da70135403c1}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:25.283Z [INFO] stable state for Leader reached at 2025-02-09 06:52:25.178332458 +0000 UTC m=+6.157141260 (1 nodes), 68.418307ms from start of poll, 68.867813ms from cluster start. Timeout at 2025-02-09 06:52:25.283367483 +0000 UTC m=+6.262176250, 105.03499ms after stability - testing.go:149: cluster: 2025-02-09T06:52:25.283Z [INFO] stable state for Leader reached at 2025-02-09 06:52:25.178332458 +0000 UTC m=+6.157141260 (1 nodes), 68.418307ms from start of poll, 68.867813ms from cluster start. Timeout at 2025-02-09 06:52:25.283367483 +0000 UTC m=+6.262176250, 105.03499ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:25.283Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T06:52:25.283Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:25.389Z [INFO] stable state for Follower reached at 2025-02-09 06:52:25.28343057 +0000 UTC m=+6.262239335 (2 nodes), 703ns from start of poll, 173.965888ms from cluster start. Timeout at 2025-02-09 06:52:25.38918756 +0000 UTC m=+6.367996328, 105.756993ms after stability - testing.go:149: cluster: 2025-02-09T06:52:25.389Z [INFO] stable state for Follower reached at 2025-02-09 06:52:25.28343057 +0000 UTC m=+6.262239335 (2 nodes), 703ns from start of poll, 173.965888ms from cluster start. Timeout at 2025-02-09 06:52:25.38918756 +0000 UTC m=+6.367996328, 105.756993ms after stability -testLoggerAdapter verbose: server-bd572076-5f37-fe83-1979-73c4797ce710: 2025-02-09T06:52:25.389Z [INFO] server-bd572076-5f37-fe83-1979-73c4797ce710: aborting pipeline replication: peer="{Voter server-b547d4e9-ca08-04ce-7a61-da70135403c1 b547d4e9-ca08-04ce-7a61-da70135403c1}" - testing.go:149: server-bd572076-5f37-fe83-1979-73c4797ce710: 2025-02-09T06:52:25.389Z [INFO] server-bd572076-5f37-fe83-1979-73c4797ce710: aborting pipeline replication: peer="{Voter server-b547d4e9-ca08-04ce-7a61-da70135403c1 b547d4e9-ca08-04ce-7a61-da70135403c1}" -testLoggerAdapter verbose: server-bd572076-5f37-fe83-1979-73c4797ce710: 2025-02-09T06:52:25.389Z [INFO] server-bd572076-5f37-fe83-1979-73c4797ce710: aborting pipeline replication: peer="{Voter server-24fc268c-7d21-51c8-02e1-3cd3597bc926 24fc268c-7d21-51c8-02e1-3cd3597bc926}" - testing.go:149: server-bd572076-5f37-fe83-1979-73c4797ce710: 2025-02-09T06:52:25.389Z [INFO] server-bd572076-5f37-fe83-1979-73c4797ce710: aborting pipeline replication: peer="{Voter server-24fc268c-7d21-51c8-02e1-3cd3597bc926 24fc268c-7d21-51c8-02e1-3cd3597bc926}" ---- PASS: TestRaft_ApplyNonLeader (0.28s) +testLoggerAdapter verbose: server-c94ae48a-40c5-cead-43b2-426a0d62a506: 2024-01-08T00:31:00.204Z [INFO] server-c94ae48a-40c5-cead-43b2-426a0d62a506: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c94ae48a-40c5-cead-43b2-426a0d62a506 Address:c94ae48a-40c5-cead-43b2-426a0d62a506} {Suffrage:Voter ID:server-ee235919-0126-d33f-6a9a-fbd6e9bd2216 Address:ee235919-0126-d33f-6a9a-fbd6e9bd2216} {Suffrage:Voter ID:server-567da5c9-fcd6-028c-64c5-eac7045ef0b7 Address:567da5c9-fcd6-028c-64c5-eac7045ef0b7}]" + testing.go:149: server-c94ae48a-40c5-cead-43b2-426a0d62a506: 2024-01-08T00:31:00.204Z [INFO] server-c94ae48a-40c5-cead-43b2-426a0d62a506: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c94ae48a-40c5-cead-43b2-426a0d62a506 Address:c94ae48a-40c5-cead-43b2-426a0d62a506} {Suffrage:Voter ID:server-ee235919-0126-d33f-6a9a-fbd6e9bd2216 Address:ee235919-0126-d33f-6a9a-fbd6e9bd2216} {Suffrage:Voter ID:server-567da5c9-fcd6-028c-64c5-eac7045ef0b7 Address:567da5c9-fcd6-028c-64c5-eac7045ef0b7}]" +testLoggerAdapter verbose: server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: 2024-01-08T00:31:00.204Z [INFO] server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c94ae48a-40c5-cead-43b2-426a0d62a506 Address:c94ae48a-40c5-cead-43b2-426a0d62a506} {Suffrage:Voter ID:server-ee235919-0126-d33f-6a9a-fbd6e9bd2216 Address:ee235919-0126-d33f-6a9a-fbd6e9bd2216} {Suffrage:Voter ID:server-567da5c9-fcd6-028c-64c5-eac7045ef0b7 Address:567da5c9-fcd6-028c-64c5-eac7045ef0b7}]" + testing.go:149: server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: 2024-01-08T00:31:00.204Z [INFO] server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c94ae48a-40c5-cead-43b2-426a0d62a506 Address:c94ae48a-40c5-cead-43b2-426a0d62a506} {Suffrage:Voter ID:server-ee235919-0126-d33f-6a9a-fbd6e9bd2216 Address:ee235919-0126-d33f-6a9a-fbd6e9bd2216} {Suffrage:Voter ID:server-567da5c9-fcd6-028c-64c5-eac7045ef0b7 Address:567da5c9-fcd6-028c-64c5-eac7045ef0b7}]" +testLoggerAdapter verbose: server-567da5c9-fcd6-028c-64c5-eac7045ef0b7: 2024-01-08T00:31:00.205Z [INFO] server-567da5c9-fcd6-028c-64c5-eac7045ef0b7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c94ae48a-40c5-cead-43b2-426a0d62a506 Address:c94ae48a-40c5-cead-43b2-426a0d62a506} {Suffrage:Voter ID:server-ee235919-0126-d33f-6a9a-fbd6e9bd2216 Address:ee235919-0126-d33f-6a9a-fbd6e9bd2216} {Suffrage:Voter ID:server-567da5c9-fcd6-028c-64c5-eac7045ef0b7 Address:567da5c9-fcd6-028c-64c5-eac7045ef0b7}]" + testing.go:149: server-567da5c9-fcd6-028c-64c5-eac7045ef0b7: 2024-01-08T00:31:00.205Z [INFO] server-567da5c9-fcd6-028c-64c5-eac7045ef0b7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c94ae48a-40c5-cead-43b2-426a0d62a506 Address:c94ae48a-40c5-cead-43b2-426a0d62a506} {Suffrage:Voter ID:server-ee235919-0126-d33f-6a9a-fbd6e9bd2216 Address:ee235919-0126-d33f-6a9a-fbd6e9bd2216} {Suffrage:Voter ID:server-567da5c9-fcd6-028c-64c5-eac7045ef0b7 Address:567da5c9-fcd6-028c-64c5-eac7045ef0b7}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:00.205Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:00.205Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-c94ae48a-40c5-cead-43b2-426a0d62a506: 2024-01-08T00:31:00.205Z [INFO] server-c94ae48a-40c5-cead-43b2-426a0d62a506: entering follower state: follower="Node at c94ae48a-40c5-cead-43b2-426a0d62a506 [Follower]" leader= + testing.go:149: server-c94ae48a-40c5-cead-43b2-426a0d62a506: 2024-01-08T00:31:00.205Z [INFO] server-c94ae48a-40c5-cead-43b2-426a0d62a506: entering follower state: follower="Node at c94ae48a-40c5-cead-43b2-426a0d62a506 [Follower]" leader= +testLoggerAdapter verbose: server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: 2024-01-08T00:31:00.205Z [INFO] server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: entering follower state: follower="Node at ee235919-0126-d33f-6a9a-fbd6e9bd2216 [Follower]" leader= + testing.go:149: server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: 2024-01-08T00:31:00.205Z [INFO] server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: entering follower state: follower="Node at ee235919-0126-d33f-6a9a-fbd6e9bd2216 [Follower]" leader= +testLoggerAdapter verbose: server-567da5c9-fcd6-028c-64c5-eac7045ef0b7: 2024-01-08T00:31:00.205Z [INFO] server-567da5c9-fcd6-028c-64c5-eac7045ef0b7: entering follower state: follower="Node at 567da5c9-fcd6-028c-64c5-eac7045ef0b7 [Follower]" leader= + testing.go:149: server-567da5c9-fcd6-028c-64c5-eac7045ef0b7: 2024-01-08T00:31:00.205Z [INFO] server-567da5c9-fcd6-028c-64c5-eac7045ef0b7: entering follower state: follower="Node at 567da5c9-fcd6-028c-64c5-eac7045ef0b7 [Follower]" leader= +testLoggerAdapter verbose: server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: 2024-01-08T00:31:00.298Z [WARN] server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: 2024-01-08T00:31:00.298Z [WARN] server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: 2024-01-08T00:31:00.298Z [INFO] server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: entering candidate state: node="Node at ee235919-0126-d33f-6a9a-fbd6e9bd2216 [Candidate]" term=2 + testing.go:149: server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: 2024-01-08T00:31:00.298Z [INFO] server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: entering candidate state: node="Node at ee235919-0126-d33f-6a9a-fbd6e9bd2216 [Candidate]" term=2 +testLoggerAdapter verbose: server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: 2024-01-08T00:31:00.298Z [INFO] server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: election won: tally=2 + testing.go:149: server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: 2024-01-08T00:31:00.298Z [INFO] server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: election won: tally=2 +testLoggerAdapter verbose: server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: 2024-01-08T00:31:00.298Z [INFO] server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: entering leader state: leader="Node at ee235919-0126-d33f-6a9a-fbd6e9bd2216 [Leader]" + testing.go:149: server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: 2024-01-08T00:31:00.298Z [INFO] server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: entering leader state: leader="Node at ee235919-0126-d33f-6a9a-fbd6e9bd2216 [Leader]" +testLoggerAdapter verbose: server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: 2024-01-08T00:31:00.298Z [INFO] server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: added peer, starting replication: peer=server-c94ae48a-40c5-cead-43b2-426a0d62a506 + testing.go:149: server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: 2024-01-08T00:31:00.298Z [INFO] server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: added peer, starting replication: peer=server-c94ae48a-40c5-cead-43b2-426a0d62a506 +testLoggerAdapter verbose: server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: 2024-01-08T00:31:00.298Z [INFO] server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: added peer, starting replication: peer=server-567da5c9-fcd6-028c-64c5-eac7045ef0b7 + testing.go:149: server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: 2024-01-08T00:31:00.298Z [INFO] server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: added peer, starting replication: peer=server-567da5c9-fcd6-028c-64c5-eac7045ef0b7 +testLoggerAdapter verbose: server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: 2024-01-08T00:31:00.298Z [INFO] server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: pipelining replication: peer="{Voter server-567da5c9-fcd6-028c-64c5-eac7045ef0b7 567da5c9-fcd6-028c-64c5-eac7045ef0b7}" + testing.go:149: server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: 2024-01-08T00:31:00.298Z [INFO] server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: pipelining replication: peer="{Voter server-567da5c9-fcd6-028c-64c5-eac7045ef0b7 567da5c9-fcd6-028c-64c5-eac7045ef0b7}" +testLoggerAdapter verbose: server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: 2024-01-08T00:31:00.299Z [INFO] server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: pipelining replication: peer="{Voter server-c94ae48a-40c5-cead-43b2-426a0d62a506 c94ae48a-40c5-cead-43b2-426a0d62a506}" + testing.go:149: server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: 2024-01-08T00:31:00.299Z [INFO] server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: pipelining replication: peer="{Voter server-c94ae48a-40c5-cead-43b2-426a0d62a506 c94ae48a-40c5-cead-43b2-426a0d62a506}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:00.405Z [INFO] stable state for Leader reached at 2024-01-08 00:31:00.299191428 +0000 UTC m=+6.346080483 (1 nodes), 94.064449ms from start of poll, 94.579875ms from cluster start. Timeout at 2024-01-08 00:31:00.405083111 +0000 UTC m=+6.451972167, 105.891684ms after stability + testing.go:149: cluster: 2024-01-08T00:31:00.405Z [INFO] stable state for Leader reached at 2024-01-08 00:31:00.299191428 +0000 UTC m=+6.346080483 (1 nodes), 94.064449ms from start of poll, 94.579875ms from cluster start. Timeout at 2024-01-08 00:31:00.405083111 +0000 UTC m=+6.451972167, 105.891684ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:00.405Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T00:31:00.405Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:00.510Z [INFO] stable state for Follower reached at 2024-01-08 00:31:00.4052561 +0000 UTC m=+6.452145314 (2 nodes), 2.707µs from start of poll, 200.644706ms from cluster start. Timeout at 2024-01-08 00:31:00.510524944 +0000 UTC m=+6.557414102, 105.268788ms after stability + testing.go:149: cluster: 2024-01-08T00:31:00.510Z [INFO] stable state for Follower reached at 2024-01-08 00:31:00.4052561 +0000 UTC m=+6.452145314 (2 nodes), 2.707µs from start of poll, 200.644706ms from cluster start. Timeout at 2024-01-08 00:31:00.510524944 +0000 UTC m=+6.557414102, 105.268788ms after stability +testLoggerAdapter verbose: server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: 2024-01-08T00:31:00.510Z [INFO] server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: aborting pipeline replication: peer="{Voter server-567da5c9-fcd6-028c-64c5-eac7045ef0b7 567da5c9-fcd6-028c-64c5-eac7045ef0b7}" + testing.go:149: server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: 2024-01-08T00:31:00.510Z [INFO] server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: aborting pipeline replication: peer="{Voter server-567da5c9-fcd6-028c-64c5-eac7045ef0b7 567da5c9-fcd6-028c-64c5-eac7045ef0b7}" +testLoggerAdapter verbose: server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: 2024-01-08T00:31:00.510Z [INFO] server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: aborting pipeline replication: peer="{Voter server-c94ae48a-40c5-cead-43b2-426a0d62a506 c94ae48a-40c5-cead-43b2-426a0d62a506}" + testing.go:149: server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: 2024-01-08T00:31:00.510Z [INFO] server-ee235919-0126-d33f-6a9a-fbd6e9bd2216: aborting pipeline replication: peer="{Voter server-c94ae48a-40c5-cead-43b2-426a0d62a506 c94ae48a-40c5-cead-43b2-426a0d62a506}" +--- PASS: TestRaft_ApplyNonLeader (0.31s) === RUN TestRaft_ApplyConcurrent -testLoggerAdapter verbose: server-e761ae66-e3fa-4d4d-b517-728cac79cd66: 2025-02-09T06:52:25.390Z [INFO] server-e761ae66-e3fa-4d4d-b517-728cac79cd66: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e761ae66-e3fa-4d4d-b517-728cac79cd66 Address:e761ae66-e3fa-4d4d-b517-728cac79cd66} {Suffrage:Voter ID:server-8552be4b-133a-3012-18f3-b660365357dc Address:8552be4b-133a-3012-18f3-b660365357dc} {Suffrage:Voter ID:server-50e7fee0-037c-4c7b-a19c-5c3b887655ef Address:50e7fee0-037c-4c7b-a19c-5c3b887655ef}]" - testing.go:149: server-e761ae66-e3fa-4d4d-b517-728cac79cd66: 2025-02-09T06:52:25.390Z [INFO] server-e761ae66-e3fa-4d4d-b517-728cac79cd66: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e761ae66-e3fa-4d4d-b517-728cac79cd66 Address:e761ae66-e3fa-4d4d-b517-728cac79cd66} {Suffrage:Voter ID:server-8552be4b-133a-3012-18f3-b660365357dc Address:8552be4b-133a-3012-18f3-b660365357dc} {Suffrage:Voter ID:server-50e7fee0-037c-4c7b-a19c-5c3b887655ef Address:50e7fee0-037c-4c7b-a19c-5c3b887655ef}]" -testLoggerAdapter verbose: server-8552be4b-133a-3012-18f3-b660365357dc: 2025-02-09T06:52:25.390Z [INFO] server-8552be4b-133a-3012-18f3-b660365357dc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e761ae66-e3fa-4d4d-b517-728cac79cd66 Address:e761ae66-e3fa-4d4d-b517-728cac79cd66} {Suffrage:Voter ID:server-8552be4b-133a-3012-18f3-b660365357dc Address:8552be4b-133a-3012-18f3-b660365357dc} {Suffrage:Voter ID:server-50e7fee0-037c-4c7b-a19c-5c3b887655ef Address:50e7fee0-037c-4c7b-a19c-5c3b887655ef}]" - testing.go:149: server-8552be4b-133a-3012-18f3-b660365357dc: 2025-02-09T06:52:25.390Z [INFO] server-8552be4b-133a-3012-18f3-b660365357dc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e761ae66-e3fa-4d4d-b517-728cac79cd66 Address:e761ae66-e3fa-4d4d-b517-728cac79cd66} {Suffrage:Voter ID:server-8552be4b-133a-3012-18f3-b660365357dc Address:8552be4b-133a-3012-18f3-b660365357dc} {Suffrage:Voter ID:server-50e7fee0-037c-4c7b-a19c-5c3b887655ef Address:50e7fee0-037c-4c7b-a19c-5c3b887655ef}]" -testLoggerAdapter verbose: server-50e7fee0-037c-4c7b-a19c-5c3b887655ef: 2025-02-09T06:52:25.390Z [INFO] server-50e7fee0-037c-4c7b-a19c-5c3b887655ef: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e761ae66-e3fa-4d4d-b517-728cac79cd66 Address:e761ae66-e3fa-4d4d-b517-728cac79cd66} {Suffrage:Voter ID:server-8552be4b-133a-3012-18f3-b660365357dc Address:8552be4b-133a-3012-18f3-b660365357dc} {Suffrage:Voter ID:server-50e7fee0-037c-4c7b-a19c-5c3b887655ef Address:50e7fee0-037c-4c7b-a19c-5c3b887655ef}]" -testLoggerAdapter verbose: server-8552be4b-133a-3012-18f3-b660365357dc: 2025-02-09T06:52:25.390Z [INFO] server-8552be4b-133a-3012-18f3-b660365357dc: entering follower state: follower="Node at 8552be4b-133a-3012-18f3-b660365357dc [Follower]" leader= -testLoggerAdapter verbose: server-e761ae66-e3fa-4d4d-b517-728cac79cd66: 2025-02-09T06:52:25.390Z [INFO] server-e761ae66-e3fa-4d4d-b517-728cac79cd66: entering follower state: follower="Node at e761ae66-e3fa-4d4d-b517-728cac79cd66 [Follower]" leader= - testing.go:149: server-50e7fee0-037c-4c7b-a19c-5c3b887655ef: 2025-02-09T06:52:25.390Z [INFO] server-50e7fee0-037c-4c7b-a19c-5c3b887655ef: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e761ae66-e3fa-4d4d-b517-728cac79cd66 Address:e761ae66-e3fa-4d4d-b517-728cac79cd66} {Suffrage:Voter ID:server-8552be4b-133a-3012-18f3-b660365357dc Address:8552be4b-133a-3012-18f3-b660365357dc} {Suffrage:Voter ID:server-50e7fee0-037c-4c7b-a19c-5c3b887655ef Address:50e7fee0-037c-4c7b-a19c-5c3b887655ef}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:25.390Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:25.390Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-50e7fee0-037c-4c7b-a19c-5c3b887655ef: 2025-02-09T06:52:25.390Z [INFO] server-50e7fee0-037c-4c7b-a19c-5c3b887655ef: entering follower state: follower="Node at 50e7fee0-037c-4c7b-a19c-5c3b887655ef [Follower]" leader= - testing.go:149: server-8552be4b-133a-3012-18f3-b660365357dc: 2025-02-09T06:52:25.390Z [INFO] server-8552be4b-133a-3012-18f3-b660365357dc: entering follower state: follower="Node at 8552be4b-133a-3012-18f3-b660365357dc [Follower]" leader= - testing.go:149: server-e761ae66-e3fa-4d4d-b517-728cac79cd66: 2025-02-09T06:52:25.390Z [INFO] server-e761ae66-e3fa-4d4d-b517-728cac79cd66: entering follower state: follower="Node at e761ae66-e3fa-4d4d-b517-728cac79cd66 [Follower]" leader= - testing.go:149: server-50e7fee0-037c-4c7b-a19c-5c3b887655ef: 2025-02-09T06:52:25.390Z [INFO] server-50e7fee0-037c-4c7b-a19c-5c3b887655ef: entering follower state: follower="Node at 50e7fee0-037c-4c7b-a19c-5c3b887655ef [Follower]" leader= -testLoggerAdapter verbose: server-8552be4b-133a-3012-18f3-b660365357dc: 2025-02-09T06:52:25.507Z [WARN] server-8552be4b-133a-3012-18f3-b660365357dc: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-8552be4b-133a-3012-18f3-b660365357dc: 2025-02-09T06:52:25.507Z [WARN] server-8552be4b-133a-3012-18f3-b660365357dc: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-8552be4b-133a-3012-18f3-b660365357dc: 2025-02-09T06:52:25.507Z [INFO] server-8552be4b-133a-3012-18f3-b660365357dc: entering candidate state: node="Node at 8552be4b-133a-3012-18f3-b660365357dc [Candidate]" term=2 - testing.go:149: server-8552be4b-133a-3012-18f3-b660365357dc: 2025-02-09T06:52:25.507Z [INFO] server-8552be4b-133a-3012-18f3-b660365357dc: entering candidate state: node="Node at 8552be4b-133a-3012-18f3-b660365357dc [Candidate]" term=2 -testLoggerAdapter verbose: server-8552be4b-133a-3012-18f3-b660365357dc: 2025-02-09T06:52:25.507Z [INFO] server-8552be4b-133a-3012-18f3-b660365357dc: election won: tally=2 - testing.go:149: server-8552be4b-133a-3012-18f3-b660365357dc: 2025-02-09T06:52:25.507Z [INFO] server-8552be4b-133a-3012-18f3-b660365357dc: election won: tally=2 -testLoggerAdapter verbose: server-8552be4b-133a-3012-18f3-b660365357dc: 2025-02-09T06:52:25.507Z [INFO] server-8552be4b-133a-3012-18f3-b660365357dc: entering leader state: leader="Node at 8552be4b-133a-3012-18f3-b660365357dc [Leader]" - testing.go:149: server-8552be4b-133a-3012-18f3-b660365357dc: 2025-02-09T06:52:25.507Z [INFO] server-8552be4b-133a-3012-18f3-b660365357dc: entering leader state: leader="Node at 8552be4b-133a-3012-18f3-b660365357dc [Leader]" -testLoggerAdapter verbose: server-8552be4b-133a-3012-18f3-b660365357dc: 2025-02-09T06:52:25.507Z [INFO] server-8552be4b-133a-3012-18f3-b660365357dc: added peer, starting replication: peer=server-e761ae66-e3fa-4d4d-b517-728cac79cd66 - testing.go:149: server-8552be4b-133a-3012-18f3-b660365357dc: 2025-02-09T06:52:25.507Z [INFO] server-8552be4b-133a-3012-18f3-b660365357dc: added peer, starting replication: peer=server-e761ae66-e3fa-4d4d-b517-728cac79cd66 -testLoggerAdapter verbose: server-8552be4b-133a-3012-18f3-b660365357dc: 2025-02-09T06:52:25.507Z [INFO] server-8552be4b-133a-3012-18f3-b660365357dc: added peer, starting replication: peer=server-50e7fee0-037c-4c7b-a19c-5c3b887655ef - testing.go:149: server-8552be4b-133a-3012-18f3-b660365357dc: 2025-02-09T06:52:25.507Z [INFO] server-8552be4b-133a-3012-18f3-b660365357dc: added peer, starting replication: peer=server-50e7fee0-037c-4c7b-a19c-5c3b887655ef -testLoggerAdapter verbose: server-8552be4b-133a-3012-18f3-b660365357dc: 2025-02-09T06:52:25.507Z [INFO] server-8552be4b-133a-3012-18f3-b660365357dc: pipelining replication: peer="{Voter server-e761ae66-e3fa-4d4d-b517-728cac79cd66 e761ae66-e3fa-4d4d-b517-728cac79cd66}" - testing.go:149: server-8552be4b-133a-3012-18f3-b660365357dc: 2025-02-09T06:52:25.507Z [INFO] server-8552be4b-133a-3012-18f3-b660365357dc: pipelining replication: peer="{Voter server-e761ae66-e3fa-4d4d-b517-728cac79cd66 e761ae66-e3fa-4d4d-b517-728cac79cd66}" -testLoggerAdapter verbose: server-8552be4b-133a-3012-18f3-b660365357dc: 2025-02-09T06:52:25.507Z [INFO] server-8552be4b-133a-3012-18f3-b660365357dc: pipelining replication: peer="{Voter server-50e7fee0-037c-4c7b-a19c-5c3b887655ef 50e7fee0-037c-4c7b-a19c-5c3b887655ef}" - testing.go:149: server-8552be4b-133a-3012-18f3-b660365357dc: 2025-02-09T06:52:25.507Z [INFO] server-8552be4b-133a-3012-18f3-b660365357dc: pipelining replication: peer="{Voter server-50e7fee0-037c-4c7b-a19c-5c3b887655ef 50e7fee0-037c-4c7b-a19c-5c3b887655ef}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:25.712Z [INFO] stable state for Leader reached at 2025-02-09 06:52:25.507618887 +0000 UTC m=+6.486427657 (1 nodes), 117.016117ms from start of poll, 117.457266ms from cluster start. Timeout at 2025-02-09 06:52:25.712767928 +0000 UTC m=+6.691576696, 205.149039ms after stability - testing.go:149: cluster: 2025-02-09T06:52:25.712Z [INFO] stable state for Leader reached at 2025-02-09 06:52:25.507618887 +0000 UTC m=+6.486427657 (1 nodes), 117.016117ms from start of poll, 117.457266ms from cluster start. Timeout at 2025-02-09 06:52:25.712767928 +0000 UTC m=+6.691576696, 205.149039ms after stability -testLoggerAdapter verbose: server-8552be4b-133a-3012-18f3-b660365357dc: 2025-02-09T06:52:25.724Z [INFO] server-8552be4b-133a-3012-18f3-b660365357dc: aborting pipeline replication: peer="{Voter server-50e7fee0-037c-4c7b-a19c-5c3b887655ef 50e7fee0-037c-4c7b-a19c-5c3b887655ef}" - testing.go:149: server-8552be4b-133a-3012-18f3-b660365357dc: 2025-02-09T06:52:25.724Z [INFO] server-8552be4b-133a-3012-18f3-b660365357dc: aborting pipeline replication: peer="{Voter server-50e7fee0-037c-4c7b-a19c-5c3b887655ef 50e7fee0-037c-4c7b-a19c-5c3b887655ef}" -testLoggerAdapter verbose: server-8552be4b-133a-3012-18f3-b660365357dc: 2025-02-09T06:52:25.724Z [INFO] server-8552be4b-133a-3012-18f3-b660365357dc: aborting pipeline replication: peer="{Voter server-e761ae66-e3fa-4d4d-b517-728cac79cd66 e761ae66-e3fa-4d4d-b517-728cac79cd66}" - testing.go:149: server-8552be4b-133a-3012-18f3-b660365357dc: 2025-02-09T06:52:25.724Z [INFO] server-8552be4b-133a-3012-18f3-b660365357dc: aborting pipeline replication: peer="{Voter server-e761ae66-e3fa-4d4d-b517-728cac79cd66 e761ae66-e3fa-4d4d-b517-728cac79cd66}" -testLoggerAdapter verbose: server-8552be4b-133a-3012-18f3-b660365357dc: 2025-02-09T06:52:25.774Z [ERROR] server-8552be4b-133a-3012-18f3-b660365357dc: failed to heartbeat to: peer=e761ae66-e3fa-4d4d-b517-728cac79cd66 error="command timed out" - testing.go:149: server-8552be4b-133a-3012-18f3-b660365357dc: 2025-02-09T06:52:25.774Z [ERROR] server-8552be4b-133a-3012-18f3-b660365357dc: failed to heartbeat to: peer=e761ae66-e3fa-4d4d-b517-728cac79cd66 error="command timed out" ---- PASS: TestRaft_ApplyConcurrent (0.39s) +testLoggerAdapter verbose: server-e34a038b-075e-3cbe-6d6c-013f2913909e: 2024-01-08T00:31:00.512Z [INFO] server-e34a038b-075e-3cbe-6d6c-013f2913909e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e34a038b-075e-3cbe-6d6c-013f2913909e Address:e34a038b-075e-3cbe-6d6c-013f2913909e} {Suffrage:Voter ID:server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507 Address:4c99d2b2-3331-681e-5fb0-42ad6ea7e507} {Suffrage:Voter ID:server-c7c7daf6-a912-c817-eea7-cd8c3644e532 Address:c7c7daf6-a912-c817-eea7-cd8c3644e532}]" + testing.go:149: server-e34a038b-075e-3cbe-6d6c-013f2913909e: 2024-01-08T00:31:00.512Z [INFO] server-e34a038b-075e-3cbe-6d6c-013f2913909e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e34a038b-075e-3cbe-6d6c-013f2913909e Address:e34a038b-075e-3cbe-6d6c-013f2913909e} {Suffrage:Voter ID:server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507 Address:4c99d2b2-3331-681e-5fb0-42ad6ea7e507} {Suffrage:Voter ID:server-c7c7daf6-a912-c817-eea7-cd8c3644e532 Address:c7c7daf6-a912-c817-eea7-cd8c3644e532}]" +testLoggerAdapter verbose: server-e34a038b-075e-3cbe-6d6c-013f2913909e: 2024-01-08T00:31:00.512Z [INFO] server-e34a038b-075e-3cbe-6d6c-013f2913909e: entering follower state: follower="Node at e34a038b-075e-3cbe-6d6c-013f2913909e [Follower]" leader= +testLoggerAdapter verbose: server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: 2024-01-08T00:31:00.512Z [INFO] server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e34a038b-075e-3cbe-6d6c-013f2913909e Address:e34a038b-075e-3cbe-6d6c-013f2913909e} {Suffrage:Voter ID:server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507 Address:4c99d2b2-3331-681e-5fb0-42ad6ea7e507} {Suffrage:Voter ID:server-c7c7daf6-a912-c817-eea7-cd8c3644e532 Address:c7c7daf6-a912-c817-eea7-cd8c3644e532}]" + testing.go:149: server-e34a038b-075e-3cbe-6d6c-013f2913909e: 2024-01-08T00:31:00.512Z [INFO] server-e34a038b-075e-3cbe-6d6c-013f2913909e: entering follower state: follower="Node at e34a038b-075e-3cbe-6d6c-013f2913909e [Follower]" leader= + testing.go:149: server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: 2024-01-08T00:31:00.512Z [INFO] server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e34a038b-075e-3cbe-6d6c-013f2913909e Address:e34a038b-075e-3cbe-6d6c-013f2913909e} {Suffrage:Voter ID:server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507 Address:4c99d2b2-3331-681e-5fb0-42ad6ea7e507} {Suffrage:Voter ID:server-c7c7daf6-a912-c817-eea7-cd8c3644e532 Address:c7c7daf6-a912-c817-eea7-cd8c3644e532}]" +testLoggerAdapter verbose: server-c7c7daf6-a912-c817-eea7-cd8c3644e532: 2024-01-08T00:31:00.512Z [INFO] server-c7c7daf6-a912-c817-eea7-cd8c3644e532: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e34a038b-075e-3cbe-6d6c-013f2913909e Address:e34a038b-075e-3cbe-6d6c-013f2913909e} {Suffrage:Voter ID:server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507 Address:4c99d2b2-3331-681e-5fb0-42ad6ea7e507} {Suffrage:Voter ID:server-c7c7daf6-a912-c817-eea7-cd8c3644e532 Address:c7c7daf6-a912-c817-eea7-cd8c3644e532}]" + testing.go:149: server-c7c7daf6-a912-c817-eea7-cd8c3644e532: 2024-01-08T00:31:00.512Z [INFO] server-c7c7daf6-a912-c817-eea7-cd8c3644e532: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e34a038b-075e-3cbe-6d6c-013f2913909e Address:e34a038b-075e-3cbe-6d6c-013f2913909e} {Suffrage:Voter ID:server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507 Address:4c99d2b2-3331-681e-5fb0-42ad6ea7e507} {Suffrage:Voter ID:server-c7c7daf6-a912-c817-eea7-cd8c3644e532 Address:c7c7daf6-a912-c817-eea7-cd8c3644e532}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:00.512Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:00.512Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: 2024-01-08T00:31:00.512Z [INFO] server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: entering follower state: follower="Node at 4c99d2b2-3331-681e-5fb0-42ad6ea7e507 [Follower]" leader= + testing.go:149: server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: 2024-01-08T00:31:00.512Z [INFO] server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: entering follower state: follower="Node at 4c99d2b2-3331-681e-5fb0-42ad6ea7e507 [Follower]" leader= +testLoggerAdapter verbose: server-c7c7daf6-a912-c817-eea7-cd8c3644e532: 2024-01-08T00:31:00.512Z [INFO] server-c7c7daf6-a912-c817-eea7-cd8c3644e532: entering follower state: follower="Node at c7c7daf6-a912-c817-eea7-cd8c3644e532 [Follower]" leader= + testing.go:149: server-c7c7daf6-a912-c817-eea7-cd8c3644e532: 2024-01-08T00:31:00.512Z [INFO] server-c7c7daf6-a912-c817-eea7-cd8c3644e532: entering follower state: follower="Node at c7c7daf6-a912-c817-eea7-cd8c3644e532 [Follower]" leader= +testLoggerAdapter verbose: server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: 2024-01-08T00:31:00.662Z [WARN] server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: 2024-01-08T00:31:00.662Z [WARN] server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: 2024-01-08T00:31:00.662Z [INFO] server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: entering candidate state: node="Node at 4c99d2b2-3331-681e-5fb0-42ad6ea7e507 [Candidate]" term=2 + testing.go:149: server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: 2024-01-08T00:31:00.662Z [INFO] server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: entering candidate state: node="Node at 4c99d2b2-3331-681e-5fb0-42ad6ea7e507 [Candidate]" term=2 +testLoggerAdapter verbose: server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: 2024-01-08T00:31:00.662Z [INFO] server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: election won: tally=2 + testing.go:149: server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: 2024-01-08T00:31:00.662Z [INFO] server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: election won: tally=2 +testLoggerAdapter verbose: server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: 2024-01-08T00:31:00.662Z [INFO] server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: entering leader state: leader="Node at 4c99d2b2-3331-681e-5fb0-42ad6ea7e507 [Leader]" + testing.go:149: server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: 2024-01-08T00:31:00.662Z [INFO] server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: entering leader state: leader="Node at 4c99d2b2-3331-681e-5fb0-42ad6ea7e507 [Leader]" +testLoggerAdapter verbose: server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: 2024-01-08T00:31:00.662Z [INFO] server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: added peer, starting replication: peer=server-e34a038b-075e-3cbe-6d6c-013f2913909e + testing.go:149: server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: 2024-01-08T00:31:00.662Z [INFO] server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: added peer, starting replication: peer=server-e34a038b-075e-3cbe-6d6c-013f2913909e +testLoggerAdapter verbose: server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: 2024-01-08T00:31:00.662Z [INFO] server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: added peer, starting replication: peer=server-c7c7daf6-a912-c817-eea7-cd8c3644e532 + testing.go:149: server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: 2024-01-08T00:31:00.662Z [INFO] server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: added peer, starting replication: peer=server-c7c7daf6-a912-c817-eea7-cd8c3644e532 +testLoggerAdapter verbose: server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: 2024-01-08T00:31:00.662Z [INFO] server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: pipelining replication: peer="{Voter server-c7c7daf6-a912-c817-eea7-cd8c3644e532 c7c7daf6-a912-c817-eea7-cd8c3644e532}" + testing.go:149: server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: 2024-01-08T00:31:00.662Z [INFO] server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: pipelining replication: peer="{Voter server-c7c7daf6-a912-c817-eea7-cd8c3644e532 c7c7daf6-a912-c817-eea7-cd8c3644e532}" +testLoggerAdapter verbose: server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: 2024-01-08T00:31:00.662Z [INFO] server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: pipelining replication: peer="{Voter server-e34a038b-075e-3cbe-6d6c-013f2913909e e34a038b-075e-3cbe-6d6c-013f2913909e}" + testing.go:149: server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: 2024-01-08T00:31:00.662Z [INFO] server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: pipelining replication: peer="{Voter server-e34a038b-075e-3cbe-6d6c-013f2913909e e34a038b-075e-3cbe-6d6c-013f2913909e}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:00.869Z [INFO] stable state for Leader reached at 2024-01-08 00:31:00.662925761 +0000 UTC m=+6.709814815 (1 nodes), 150.20566ms from start of poll, 150.859475ms from cluster start. Timeout at 2024-01-08 00:31:00.868704599 +0000 UTC m=+6.915593685, 205.77887ms after stability + testing.go:149: cluster: 2024-01-08T00:31:00.869Z [INFO] stable state for Leader reached at 2024-01-08 00:31:00.662925761 +0000 UTC m=+6.709814815 (1 nodes), 150.20566ms from start of poll, 150.859475ms from cluster start. Timeout at 2024-01-08 00:31:00.868704599 +0000 UTC m=+6.915593685, 205.77887ms after stability +testLoggerAdapter verbose: server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: 2024-01-08T00:31:00.882Z [INFO] server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: aborting pipeline replication: peer="{Voter server-c7c7daf6-a912-c817-eea7-cd8c3644e532 c7c7daf6-a912-c817-eea7-cd8c3644e532}" + testing.go:149: server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: 2024-01-08T00:31:00.882Z [INFO] server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: aborting pipeline replication: peer="{Voter server-c7c7daf6-a912-c817-eea7-cd8c3644e532 c7c7daf6-a912-c817-eea7-cd8c3644e532}" +testLoggerAdapter verbose: server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: 2024-01-08T00:31:00.882Z [INFO] server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: aborting pipeline replication: peer="{Voter server-e34a038b-075e-3cbe-6d6c-013f2913909e e34a038b-075e-3cbe-6d6c-013f2913909e}" + testing.go:149: server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: 2024-01-08T00:31:00.882Z [INFO] server-4c99d2b2-3331-681e-5fb0-42ad6ea7e507: aborting pipeline replication: peer="{Voter server-e34a038b-075e-3cbe-6d6c-013f2913909e e34a038b-075e-3cbe-6d6c-013f2913909e}" +--- PASS: TestRaft_ApplyConcurrent (0.37s) === RUN TestRaft_ApplyConcurrent_Timeout -testLoggerAdapter verbose: server-dc607b2a-5e8a-119e-ee9a-7a350a77e40d: 2025-02-09T06:52:25.775Z [INFO] server-dc607b2a-5e8a-119e-ee9a-7a350a77e40d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dc607b2a-5e8a-119e-ee9a-7a350a77e40d Address:dc607b2a-5e8a-119e-ee9a-7a350a77e40d}]" - testing.go:149: server-dc607b2a-5e8a-119e-ee9a-7a350a77e40d: 2025-02-09T06:52:25.775Z [INFO] server-dc607b2a-5e8a-119e-ee9a-7a350a77e40d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-dc607b2a-5e8a-119e-ee9a-7a350a77e40d Address:dc607b2a-5e8a-119e-ee9a-7a350a77e40d}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:25.775Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:25.775Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-dc607b2a-5e8a-119e-ee9a-7a350a77e40d: 2025-02-09T06:52:25.775Z [INFO] server-dc607b2a-5e8a-119e-ee9a-7a350a77e40d: entering follower state: follower="Node at dc607b2a-5e8a-119e-ee9a-7a350a77e40d [Follower]" leader= - testing.go:149: server-dc607b2a-5e8a-119e-ee9a-7a350a77e40d: 2025-02-09T06:52:25.775Z [INFO] server-dc607b2a-5e8a-119e-ee9a-7a350a77e40d: entering follower state: follower="Node at dc607b2a-5e8a-119e-ee9a-7a350a77e40d [Follower]" leader= -testLoggerAdapter verbose: server-dc607b2a-5e8a-119e-ee9a-7a350a77e40d: 2025-02-09T06:52:25.921Z [WARN] server-dc607b2a-5e8a-119e-ee9a-7a350a77e40d: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-dc607b2a-5e8a-119e-ee9a-7a350a77e40d: 2025-02-09T06:52:25.921Z [WARN] server-dc607b2a-5e8a-119e-ee9a-7a350a77e40d: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-dc607b2a-5e8a-119e-ee9a-7a350a77e40d: 2025-02-09T06:52:25.921Z [INFO] server-dc607b2a-5e8a-119e-ee9a-7a350a77e40d: entering candidate state: node="Node at dc607b2a-5e8a-119e-ee9a-7a350a77e40d [Candidate]" term=2 - testing.go:149: server-dc607b2a-5e8a-119e-ee9a-7a350a77e40d: 2025-02-09T06:52:25.921Z [INFO] server-dc607b2a-5e8a-119e-ee9a-7a350a77e40d: entering candidate state: node="Node at dc607b2a-5e8a-119e-ee9a-7a350a77e40d [Candidate]" term=2 -testLoggerAdapter verbose: server-dc607b2a-5e8a-119e-ee9a-7a350a77e40d: 2025-02-09T06:52:25.921Z [INFO] server-dc607b2a-5e8a-119e-ee9a-7a350a77e40d: election won: tally=1 - testing.go:149: server-dc607b2a-5e8a-119e-ee9a-7a350a77e40d: 2025-02-09T06:52:25.921Z [INFO] server-dc607b2a-5e8a-119e-ee9a-7a350a77e40d: election won: tally=1 -testLoggerAdapter verbose: server-dc607b2a-5e8a-119e-ee9a-7a350a77e40d: 2025-02-09T06:52:25.921Z [INFO] server-dc607b2a-5e8a-119e-ee9a-7a350a77e40d: entering leader state: leader="Node at dc607b2a-5e8a-119e-ee9a-7a350a77e40d [Leader]" - testing.go:149: server-dc607b2a-5e8a-119e-ee9a-7a350a77e40d: 2025-02-09T06:52:25.921Z [INFO] server-dc607b2a-5e8a-119e-ee9a-7a350a77e40d: entering leader state: leader="Node at dc607b2a-5e8a-119e-ee9a-7a350a77e40d [Leader]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:26.123Z [INFO] stable state for Leader reached at 2025-02-09 06:52:25.921807154 +0000 UTC m=+6.900615937 (1 nodes), 146.464935ms from start of poll, 146.65945ms from cluster start. Timeout at 2025-02-09 06:52:26.123505157 +0000 UTC m=+7.102313941, 201.698004ms after stability - testing.go:149: cluster: 2025-02-09T06:52:26.123Z [INFO] stable state for Leader reached at 2025-02-09 06:52:25.921807154 +0000 UTC m=+6.900615937 (1 nodes), 146.464935ms from start of poll, 146.65945ms from cluster start. Timeout at 2025-02-09 06:52:26.123505157 +0000 UTC m=+7.102313941, 201.698004ms after stability ---- PASS: TestRaft_ApplyConcurrent_Timeout (0.35s) +testLoggerAdapter verbose: server-42711a1b-8ca5-3d4a-dbff-88b96e634082: 2024-01-08T00:31:00.883Z [INFO] server-42711a1b-8ca5-3d4a-dbff-88b96e634082: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-42711a1b-8ca5-3d4a-dbff-88b96e634082 Address:42711a1b-8ca5-3d4a-dbff-88b96e634082}]" + testing.go:149: server-42711a1b-8ca5-3d4a-dbff-88b96e634082: 2024-01-08T00:31:00.883Z [INFO] server-42711a1b-8ca5-3d4a-dbff-88b96e634082: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-42711a1b-8ca5-3d4a-dbff-88b96e634082 Address:42711a1b-8ca5-3d4a-dbff-88b96e634082}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:00.883Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:00.883Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-42711a1b-8ca5-3d4a-dbff-88b96e634082: 2024-01-08T00:31:00.883Z [INFO] server-42711a1b-8ca5-3d4a-dbff-88b96e634082: entering follower state: follower="Node at 42711a1b-8ca5-3d4a-dbff-88b96e634082 [Follower]" leader= + testing.go:149: server-42711a1b-8ca5-3d4a-dbff-88b96e634082: 2024-01-08T00:31:00.883Z [INFO] server-42711a1b-8ca5-3d4a-dbff-88b96e634082: entering follower state: follower="Node at 42711a1b-8ca5-3d4a-dbff-88b96e634082 [Follower]" leader= +testLoggerAdapter verbose: server-42711a1b-8ca5-3d4a-dbff-88b96e634082: 2024-01-08T00:31:01.054Z [WARN] server-42711a1b-8ca5-3d4a-dbff-88b96e634082: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-42711a1b-8ca5-3d4a-dbff-88b96e634082: 2024-01-08T00:31:01.054Z [WARN] server-42711a1b-8ca5-3d4a-dbff-88b96e634082: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-42711a1b-8ca5-3d4a-dbff-88b96e634082: 2024-01-08T00:31:01.054Z [INFO] server-42711a1b-8ca5-3d4a-dbff-88b96e634082: entering candidate state: node="Node at 42711a1b-8ca5-3d4a-dbff-88b96e634082 [Candidate]" term=2 + testing.go:149: server-42711a1b-8ca5-3d4a-dbff-88b96e634082: 2024-01-08T00:31:01.054Z [INFO] server-42711a1b-8ca5-3d4a-dbff-88b96e634082: entering candidate state: node="Node at 42711a1b-8ca5-3d4a-dbff-88b96e634082 [Candidate]" term=2 +testLoggerAdapter verbose: server-42711a1b-8ca5-3d4a-dbff-88b96e634082: 2024-01-08T00:31:01.054Z [INFO] server-42711a1b-8ca5-3d4a-dbff-88b96e634082: election won: tally=1 + testing.go:149: server-42711a1b-8ca5-3d4a-dbff-88b96e634082: 2024-01-08T00:31:01.054Z [INFO] server-42711a1b-8ca5-3d4a-dbff-88b96e634082: election won: tally=1 +testLoggerAdapter verbose: server-42711a1b-8ca5-3d4a-dbff-88b96e634082: 2024-01-08T00:31:01.054Z [INFO] server-42711a1b-8ca5-3d4a-dbff-88b96e634082: entering leader state: leader="Node at 42711a1b-8ca5-3d4a-dbff-88b96e634082 [Leader]" + testing.go:149: server-42711a1b-8ca5-3d4a-dbff-88b96e634082: 2024-01-08T00:31:01.054Z [INFO] server-42711a1b-8ca5-3d4a-dbff-88b96e634082: entering leader state: leader="Node at 42711a1b-8ca5-3d4a-dbff-88b96e634082 [Leader]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:01.256Z [INFO] stable state for Leader reached at 2024-01-08 00:31:01.054795003 +0000 UTC m=+7.101684045 (1 nodes), 171.672614ms from start of poll, 171.855826ms from cluster start. Timeout at 2024-01-08 00:31:01.25694606 +0000 UTC m=+7.303835185, 202.15114ms after stability + testing.go:149: cluster: 2024-01-08T00:31:01.256Z [INFO] stable state for Leader reached at 2024-01-08 00:31:01.054795003 +0000 UTC m=+7.101684045 (1 nodes), 171.672614ms from start of poll, 171.855826ms from cluster start. Timeout at 2024-01-08 00:31:01.25694606 +0000 UTC m=+7.303835185, 202.15114ms after stability +--- PASS: TestRaft_ApplyConcurrent_Timeout (0.38s) === RUN TestRaft_JoinNode -testLoggerAdapter verbose: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.124Z [INFO] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a49a5533-1dc6-6d7a-9697-cec15b098f11 Address:a49a5533-1dc6-6d7a-9697-cec15b098f11} {Suffrage:Voter ID:server-93eb6166-f567-334d-42d6-870eb7277315 Address:93eb6166-f567-334d-42d6-870eb7277315}]" - testing.go:149: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.124Z [INFO] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a49a5533-1dc6-6d7a-9697-cec15b098f11 Address:a49a5533-1dc6-6d7a-9697-cec15b098f11} {Suffrage:Voter ID:server-93eb6166-f567-334d-42d6-870eb7277315 Address:93eb6166-f567-334d-42d6-870eb7277315}]" -testLoggerAdapter verbose: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.124Z [INFO] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: entering follower state: follower="Node at a49a5533-1dc6-6d7a-9697-cec15b098f11 [Follower]" leader= - testing.go:149: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.124Z [INFO] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: entering follower state: follower="Node at a49a5533-1dc6-6d7a-9697-cec15b098f11 [Follower]" leader= -testLoggerAdapter verbose: server-93eb6166-f567-334d-42d6-870eb7277315: 2025-02-09T06:52:26.124Z [INFO] server-93eb6166-f567-334d-42d6-870eb7277315: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a49a5533-1dc6-6d7a-9697-cec15b098f11 Address:a49a5533-1dc6-6d7a-9697-cec15b098f11} {Suffrage:Voter ID:server-93eb6166-f567-334d-42d6-870eb7277315 Address:93eb6166-f567-334d-42d6-870eb7277315}]" - testing.go:149: server-93eb6166-f567-334d-42d6-870eb7277315: 2025-02-09T06:52:26.124Z [INFO] server-93eb6166-f567-334d-42d6-870eb7277315: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a49a5533-1dc6-6d7a-9697-cec15b098f11 Address:a49a5533-1dc6-6d7a-9697-cec15b098f11} {Suffrage:Voter ID:server-93eb6166-f567-334d-42d6-870eb7277315 Address:93eb6166-f567-334d-42d6-870eb7277315}]" -testLoggerAdapter verbose: server-93eb6166-f567-334d-42d6-870eb7277315: 2025-02-09T06:52:26.124Z [INFO] server-93eb6166-f567-334d-42d6-870eb7277315: entering follower state: follower="Node at 93eb6166-f567-334d-42d6-870eb7277315 [Follower]" leader= - testing.go:149: server-93eb6166-f567-334d-42d6-870eb7277315: 2025-02-09T06:52:26.124Z [INFO] server-93eb6166-f567-334d-42d6-870eb7277315: entering follower state: follower="Node at 93eb6166-f567-334d-42d6-870eb7277315 [Follower]" leader= -testLoggerAdapter verbose: server-ca87de4a-7740-adf4-b3d7-64b46f167789: 2025-02-09T06:52:26.124Z [INFO] server-ca87de4a-7740-adf4-b3d7-64b46f167789: initial configuration: index=0 servers=[] - testing.go:149: server-ca87de4a-7740-adf4-b3d7-64b46f167789: 2025-02-09T06:52:26.124Z [INFO] server-ca87de4a-7740-adf4-b3d7-64b46f167789: initial configuration: index=0 servers=[] -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:26.124Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:26.124Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-ca87de4a-7740-adf4-b3d7-64b46f167789: 2025-02-09T06:52:26.124Z [INFO] server-ca87de4a-7740-adf4-b3d7-64b46f167789: entering follower state: follower="Node at ca87de4a-7740-adf4-b3d7-64b46f167789 [Follower]" leader= - testing.go:149: server-ca87de4a-7740-adf4-b3d7-64b46f167789: 2025-02-09T06:52:26.124Z [INFO] server-ca87de4a-7740-adf4-b3d7-64b46f167789: entering follower state: follower="Node at ca87de4a-7740-adf4-b3d7-64b46f167789 [Follower]" leader= -testLoggerAdapter verbose: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.208Z [WARN] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.208Z [WARN] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.208Z [INFO] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: entering candidate state: node="Node at a49a5533-1dc6-6d7a-9697-cec15b098f11 [Candidate]" term=2 - testing.go:149: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.208Z [INFO] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: entering candidate state: node="Node at a49a5533-1dc6-6d7a-9697-cec15b098f11 [Candidate]" term=2 -testLoggerAdapter verbose: server-93eb6166-f567-334d-42d6-870eb7277315: 2025-02-09T06:52:26.208Z [WARN] server-93eb6166-f567-334d-42d6-870eb7277315: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-93eb6166-f567-334d-42d6-870eb7277315: 2025-02-09T06:52:26.208Z [WARN] server-93eb6166-f567-334d-42d6-870eb7277315: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-93eb6166-f567-334d-42d6-870eb7277315: 2025-02-09T06:52:26.208Z [INFO] server-93eb6166-f567-334d-42d6-870eb7277315: entering candidate state: node="Node at 93eb6166-f567-334d-42d6-870eb7277315 [Candidate]" term=2 - testing.go:149: server-93eb6166-f567-334d-42d6-870eb7277315: 2025-02-09T06:52:26.208Z [INFO] server-93eb6166-f567-334d-42d6-870eb7277315: entering candidate state: node="Node at 93eb6166-f567-334d-42d6-870eb7277315 [Candidate]" term=2 -testLoggerAdapter verbose: server-93eb6166-f567-334d-42d6-870eb7277315: 2025-02-09T06:52:26.208Z [INFO] server-93eb6166-f567-334d-42d6-870eb7277315: duplicate requestVote for same term: term=2 - testing.go:149: server-93eb6166-f567-334d-42d6-870eb7277315: 2025-02-09T06:52:26.208Z [INFO] server-93eb6166-f567-334d-42d6-870eb7277315: duplicate requestVote for same term: term=2 -testLoggerAdapter verbose: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.208Z [INFO] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: duplicate requestVote for same term: term=2 - testing.go:149: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.208Z [INFO] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: duplicate requestVote for same term: term=2 -testLoggerAdapter verbose: server-ca87de4a-7740-adf4-b3d7-64b46f167789: 2025-02-09T06:52:26.211Z [WARN] server-ca87de4a-7740-adf4-b3d7-64b46f167789: no known peers, aborting election - testing.go:149: server-ca87de4a-7740-adf4-b3d7-64b46f167789: 2025-02-09T06:52:26.211Z [WARN] server-ca87de4a-7740-adf4-b3d7-64b46f167789: no known peers, aborting election -testLoggerAdapter verbose: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.266Z [WARN] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: Election timeout reached, restarting election - testing.go:149: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.266Z [WARN] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: Election timeout reached, restarting election -testLoggerAdapter verbose: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.266Z [INFO] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: entering candidate state: node="Node at a49a5533-1dc6-6d7a-9697-cec15b098f11 [Candidate]" term=3 - testing.go:149: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.266Z [INFO] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: entering candidate state: node="Node at a49a5533-1dc6-6d7a-9697-cec15b098f11 [Candidate]" term=3 -testLoggerAdapter verbose: server-93eb6166-f567-334d-42d6-870eb7277315: 2025-02-09T06:52:26.266Z [INFO] server-93eb6166-f567-334d-42d6-870eb7277315: entering follower state: follower="Node at 93eb6166-f567-334d-42d6-870eb7277315 [Follower]" leader= - testing.go:149: server-93eb6166-f567-334d-42d6-870eb7277315: 2025-02-09T06:52:26.266Z [INFO] server-93eb6166-f567-334d-42d6-870eb7277315: entering follower state: follower="Node at 93eb6166-f567-334d-42d6-870eb7277315 [Follower]" leader= -testLoggerAdapter verbose: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.266Z [INFO] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: election won: tally=2 - testing.go:149: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.266Z [INFO] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: election won: tally=2 -testLoggerAdapter verbose: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.266Z [INFO] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: entering leader state: leader="Node at a49a5533-1dc6-6d7a-9697-cec15b098f11 [Leader]" - testing.go:149: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.266Z [INFO] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: entering leader state: leader="Node at a49a5533-1dc6-6d7a-9697-cec15b098f11 [Leader]" -testLoggerAdapter verbose: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.266Z [INFO] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: added peer, starting replication: peer=server-93eb6166-f567-334d-42d6-870eb7277315 - testing.go:149: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.266Z [INFO] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: added peer, starting replication: peer=server-93eb6166-f567-334d-42d6-870eb7277315 -testLoggerAdapter verbose: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.266Z [INFO] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: pipelining replication: peer="{Voter server-93eb6166-f567-334d-42d6-870eb7277315 93eb6166-f567-334d-42d6-870eb7277315}" - testing.go:149: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.266Z [INFO] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: pipelining replication: peer="{Voter server-93eb6166-f567-334d-42d6-870eb7277315 93eb6166-f567-334d-42d6-870eb7277315}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:26.371Z [INFO] stable state for Leader reached at 2025-02-09 06:52:26.266277205 +0000 UTC m=+7.245085971 (1 nodes), 141.563202ms from start of poll, 142.06085ms from cluster start. Timeout at 2025-02-09 06:52:26.371863987 +0000 UTC m=+7.350672746, 105.586775ms after stability - testing.go:149: cluster: 2025-02-09T06:52:26.371Z [INFO] stable state for Leader reached at 2025-02-09 06:52:26.266277205 +0000 UTC m=+7.245085971 (1 nodes), 141.563202ms from start of poll, 142.06085ms from cluster start. Timeout at 2025-02-09 06:52:26.371863987 +0000 UTC m=+7.350672746, 105.586775ms after stability -testLoggerAdapter verbose: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.371Z [INFO] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: updating configuration: command=AddStaging server-id=server-ca87de4a-7740-adf4-b3d7-64b46f167789 server-addr=ca87de4a-7740-adf4-b3d7-64b46f167789 servers="[{Suffrage:Voter ID:server-a49a5533-1dc6-6d7a-9697-cec15b098f11 Address:a49a5533-1dc6-6d7a-9697-cec15b098f11} {Suffrage:Voter ID:server-93eb6166-f567-334d-42d6-870eb7277315 Address:93eb6166-f567-334d-42d6-870eb7277315} {Suffrage:Voter ID:server-ca87de4a-7740-adf4-b3d7-64b46f167789 Address:ca87de4a-7740-adf4-b3d7-64b46f167789}]" - testing.go:149: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.371Z [INFO] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: updating configuration: command=AddStaging server-id=server-ca87de4a-7740-adf4-b3d7-64b46f167789 server-addr=ca87de4a-7740-adf4-b3d7-64b46f167789 servers="[{Suffrage:Voter ID:server-a49a5533-1dc6-6d7a-9697-cec15b098f11 Address:a49a5533-1dc6-6d7a-9697-cec15b098f11} {Suffrage:Voter ID:server-93eb6166-f567-334d-42d6-870eb7277315 Address:93eb6166-f567-334d-42d6-870eb7277315} {Suffrage:Voter ID:server-ca87de4a-7740-adf4-b3d7-64b46f167789 Address:ca87de4a-7740-adf4-b3d7-64b46f167789}]" -testLoggerAdapter verbose: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.371Z [INFO] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: added peer, starting replication: peer=server-ca87de4a-7740-adf4-b3d7-64b46f167789 - testing.go:149: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.371Z [INFO] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: added peer, starting replication: peer=server-ca87de4a-7740-adf4-b3d7-64b46f167789 -testLoggerAdapter verbose: server-ca87de4a-7740-adf4-b3d7-64b46f167789: 2025-02-09T06:52:26.372Z [WARN] server-ca87de4a-7740-adf4-b3d7-64b46f167789: failed to get previous log: previous-index=3 last-index=0 error="log not found" - testing.go:149: server-ca87de4a-7740-adf4-b3d7-64b46f167789: 2025-02-09T06:52:26.372Z [WARN] server-ca87de4a-7740-adf4-b3d7-64b46f167789: failed to get previous log: previous-index=3 last-index=0 error="log not found" -testLoggerAdapter verbose: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.372Z [WARN] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: appendEntries rejected, sending older logs: peer="{Voter server-ca87de4a-7740-adf4-b3d7-64b46f167789 ca87de4a-7740-adf4-b3d7-64b46f167789}" next=1 - testing.go:149: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.372Z [WARN] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: appendEntries rejected, sending older logs: peer="{Voter server-ca87de4a-7740-adf4-b3d7-64b46f167789 ca87de4a-7740-adf4-b3d7-64b46f167789}" next=1 -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:26.372Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:26.372Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.372Z [INFO] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: pipelining replication: peer="{Voter server-ca87de4a-7740-adf4-b3d7-64b46f167789 ca87de4a-7740-adf4-b3d7-64b46f167789}" - testing.go:149: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.372Z [INFO] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: pipelining replication: peer="{Voter server-ca87de4a-7740-adf4-b3d7-64b46f167789 ca87de4a-7740-adf4-b3d7-64b46f167789}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:26.477Z [INFO] stable state for Leader reached at 2025-02-09 06:52:26.37221757 +0000 UTC m=+7.351026332 (1 nodes), 524ns from start of poll, 248.001211ms from cluster start. Timeout at 2025-02-09 06:52:26.47792676 +0000 UTC m=+7.456735529, 105.709197ms after stability - testing.go:149: cluster: 2025-02-09T06:52:26.477Z [INFO] stable state for Leader reached at 2025-02-09 06:52:26.37221757 +0000 UTC m=+7.351026332 (1 nodes), 524ns from start of poll, 248.001211ms from cluster start. Timeout at 2025-02-09 06:52:26.47792676 +0000 UTC m=+7.456735529, 105.709197ms after stability -testLoggerAdapter verbose: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.478Z [INFO] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: aborting pipeline replication: peer="{Voter server-ca87de4a-7740-adf4-b3d7-64b46f167789 ca87de4a-7740-adf4-b3d7-64b46f167789}" - testing.go:149: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.478Z [INFO] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: aborting pipeline replication: peer="{Voter server-ca87de4a-7740-adf4-b3d7-64b46f167789 ca87de4a-7740-adf4-b3d7-64b46f167789}" -testLoggerAdapter verbose: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.478Z [INFO] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: aborting pipeline replication: peer="{Voter server-93eb6166-f567-334d-42d6-870eb7277315 93eb6166-f567-334d-42d6-870eb7277315}" - testing.go:149: server-a49a5533-1dc6-6d7a-9697-cec15b098f11: 2025-02-09T06:52:26.478Z [INFO] server-a49a5533-1dc6-6d7a-9697-cec15b098f11: aborting pipeline replication: peer="{Voter server-93eb6166-f567-334d-42d6-870eb7277315 93eb6166-f567-334d-42d6-870eb7277315}" ---- PASS: TestRaft_JoinNode (0.35s) +testLoggerAdapter verbose: server-7198e2e6-b1c3-0003-aae7-a998e679b2be: 2024-01-08T00:31:01.259Z [INFO] server-7198e2e6-b1c3-0003-aae7-a998e679b2be: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7198e2e6-b1c3-0003-aae7-a998e679b2be Address:7198e2e6-b1c3-0003-aae7-a998e679b2be} {Suffrage:Voter ID:server-04f12023-517d-7f2b-f1cf-06545fbcde04 Address:04f12023-517d-7f2b-f1cf-06545fbcde04}]" + testing.go:149: server-7198e2e6-b1c3-0003-aae7-a998e679b2be: 2024-01-08T00:31:01.259Z [INFO] server-7198e2e6-b1c3-0003-aae7-a998e679b2be: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7198e2e6-b1c3-0003-aae7-a998e679b2be Address:7198e2e6-b1c3-0003-aae7-a998e679b2be} {Suffrage:Voter ID:server-04f12023-517d-7f2b-f1cf-06545fbcde04 Address:04f12023-517d-7f2b-f1cf-06545fbcde04}]" +testLoggerAdapter verbose: server-04f12023-517d-7f2b-f1cf-06545fbcde04: 2024-01-08T00:31:01.260Z [INFO] server-04f12023-517d-7f2b-f1cf-06545fbcde04: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7198e2e6-b1c3-0003-aae7-a998e679b2be Address:7198e2e6-b1c3-0003-aae7-a998e679b2be} {Suffrage:Voter ID:server-04f12023-517d-7f2b-f1cf-06545fbcde04 Address:04f12023-517d-7f2b-f1cf-06545fbcde04}]" + testing.go:149: server-04f12023-517d-7f2b-f1cf-06545fbcde04: 2024-01-08T00:31:01.260Z [INFO] server-04f12023-517d-7f2b-f1cf-06545fbcde04: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7198e2e6-b1c3-0003-aae7-a998e679b2be Address:7198e2e6-b1c3-0003-aae7-a998e679b2be} {Suffrage:Voter ID:server-04f12023-517d-7f2b-f1cf-06545fbcde04 Address:04f12023-517d-7f2b-f1cf-06545fbcde04}]" +testLoggerAdapter verbose: server-6a4b3372-729a-f122-9465-d18f6df32d65: 2024-01-08T00:31:01.260Z [INFO] server-6a4b3372-729a-f122-9465-d18f6df32d65: initial configuration: index=0 servers=[] + testing.go:149: server-6a4b3372-729a-f122-9465-d18f6df32d65: 2024-01-08T00:31:01.260Z [INFO] server-6a4b3372-729a-f122-9465-d18f6df32d65: initial configuration: index=0 servers=[] +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:01.260Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:01.260Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-7198e2e6-b1c3-0003-aae7-a998e679b2be: 2024-01-08T00:31:01.260Z [INFO] server-7198e2e6-b1c3-0003-aae7-a998e679b2be: entering follower state: follower="Node at 7198e2e6-b1c3-0003-aae7-a998e679b2be [Follower]" leader= + testing.go:149: server-7198e2e6-b1c3-0003-aae7-a998e679b2be: 2024-01-08T00:31:01.260Z [INFO] server-7198e2e6-b1c3-0003-aae7-a998e679b2be: entering follower state: follower="Node at 7198e2e6-b1c3-0003-aae7-a998e679b2be [Follower]" leader= +testLoggerAdapter verbose: server-04f12023-517d-7f2b-f1cf-06545fbcde04: 2024-01-08T00:31:01.260Z [INFO] server-04f12023-517d-7f2b-f1cf-06545fbcde04: entering follower state: follower="Node at 04f12023-517d-7f2b-f1cf-06545fbcde04 [Follower]" leader= + testing.go:149: server-04f12023-517d-7f2b-f1cf-06545fbcde04: 2024-01-08T00:31:01.260Z [INFO] server-04f12023-517d-7f2b-f1cf-06545fbcde04: entering follower state: follower="Node at 04f12023-517d-7f2b-f1cf-06545fbcde04 [Follower]" leader= +testLoggerAdapter verbose: server-6a4b3372-729a-f122-9465-d18f6df32d65: 2024-01-08T00:31:01.260Z [INFO] server-6a4b3372-729a-f122-9465-d18f6df32d65: entering follower state: follower="Node at 6a4b3372-729a-f122-9465-d18f6df32d65 [Follower]" leader= + testing.go:149: server-6a4b3372-729a-f122-9465-d18f6df32d65: 2024-01-08T00:31:01.260Z [INFO] server-6a4b3372-729a-f122-9465-d18f6df32d65: entering follower state: follower="Node at 6a4b3372-729a-f122-9465-d18f6df32d65 [Follower]" leader= +testLoggerAdapter verbose: server-7198e2e6-b1c3-0003-aae7-a998e679b2be: 2024-01-08T00:31:01.312Z [WARN] server-7198e2e6-b1c3-0003-aae7-a998e679b2be: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-7198e2e6-b1c3-0003-aae7-a998e679b2be: 2024-01-08T00:31:01.312Z [WARN] server-7198e2e6-b1c3-0003-aae7-a998e679b2be: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-7198e2e6-b1c3-0003-aae7-a998e679b2be: 2024-01-08T00:31:01.312Z [INFO] server-7198e2e6-b1c3-0003-aae7-a998e679b2be: entering candidate state: node="Node at 7198e2e6-b1c3-0003-aae7-a998e679b2be [Candidate]" term=2 + testing.go:149: server-7198e2e6-b1c3-0003-aae7-a998e679b2be: 2024-01-08T00:31:01.312Z [INFO] server-7198e2e6-b1c3-0003-aae7-a998e679b2be: entering candidate state: node="Node at 7198e2e6-b1c3-0003-aae7-a998e679b2be [Candidate]" term=2 +testLoggerAdapter verbose: server-7198e2e6-b1c3-0003-aae7-a998e679b2be: 2024-01-08T00:31:01.313Z [INFO] server-7198e2e6-b1c3-0003-aae7-a998e679b2be: election won: tally=2 + testing.go:149: server-7198e2e6-b1c3-0003-aae7-a998e679b2be: 2024-01-08T00:31:01.313Z [INFO] server-7198e2e6-b1c3-0003-aae7-a998e679b2be: election won: tally=2 +testLoggerAdapter verbose: server-7198e2e6-b1c3-0003-aae7-a998e679b2be: 2024-01-08T00:31:01.313Z [INFO] server-7198e2e6-b1c3-0003-aae7-a998e679b2be: entering leader state: leader="Node at 7198e2e6-b1c3-0003-aae7-a998e679b2be [Leader]" + testing.go:149: server-7198e2e6-b1c3-0003-aae7-a998e679b2be: 2024-01-08T00:31:01.313Z [INFO] server-7198e2e6-b1c3-0003-aae7-a998e679b2be: entering leader state: leader="Node at 7198e2e6-b1c3-0003-aae7-a998e679b2be [Leader]" +testLoggerAdapter verbose: server-7198e2e6-b1c3-0003-aae7-a998e679b2be: 2024-01-08T00:31:01.313Z [INFO] server-7198e2e6-b1c3-0003-aae7-a998e679b2be: added peer, starting replication: peer=server-04f12023-517d-7f2b-f1cf-06545fbcde04 + testing.go:149: server-7198e2e6-b1c3-0003-aae7-a998e679b2be: 2024-01-08T00:31:01.313Z [INFO] server-7198e2e6-b1c3-0003-aae7-a998e679b2be: added peer, starting replication: peer=server-04f12023-517d-7f2b-f1cf-06545fbcde04 +testLoggerAdapter verbose: server-7198e2e6-b1c3-0003-aae7-a998e679b2be: 2024-01-08T00:31:01.313Z [INFO] server-7198e2e6-b1c3-0003-aae7-a998e679b2be: pipelining replication: peer="{Voter server-04f12023-517d-7f2b-f1cf-06545fbcde04 04f12023-517d-7f2b-f1cf-06545fbcde04}" + testing.go:149: server-7198e2e6-b1c3-0003-aae7-a998e679b2be: 2024-01-08T00:31:01.313Z [INFO] server-7198e2e6-b1c3-0003-aae7-a998e679b2be: pipelining replication: peer="{Voter server-04f12023-517d-7f2b-f1cf-06545fbcde04 04f12023-517d-7f2b-f1cf-06545fbcde04}" +testLoggerAdapter verbose: server-6a4b3372-729a-f122-9465-d18f6df32d65: 2024-01-08T00:31:01.316Z [WARN] server-6a4b3372-729a-f122-9465-d18f6df32d65: no known peers, aborting election + testing.go:149: server-6a4b3372-729a-f122-9465-d18f6df32d65: 2024-01-08T00:31:01.316Z [WARN] server-6a4b3372-729a-f122-9465-d18f6df32d65: no known peers, aborting election +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:01.419Z [INFO] stable state for Leader reached at 2024-01-08 00:31:01.313450468 +0000 UTC m=+7.360339712 (1 nodes), 53.073445ms from start of poll, 53.575307ms from cluster start. Timeout at 2024-01-08 00:31:01.419476132 +0000 UTC m=+7.466365416, 106.025704ms after stability + testing.go:149: cluster: 2024-01-08T00:31:01.419Z [INFO] stable state for Leader reached at 2024-01-08 00:31:01.313450468 +0000 UTC m=+7.360339712 (1 nodes), 53.073445ms from start of poll, 53.575307ms from cluster start. Timeout at 2024-01-08 00:31:01.419476132 +0000 UTC m=+7.466365416, 106.025704ms after stability +testLoggerAdapter verbose: server-7198e2e6-b1c3-0003-aae7-a998e679b2be: 2024-01-08T00:31:01.419Z [INFO] server-7198e2e6-b1c3-0003-aae7-a998e679b2be: updating configuration: command=AddStaging server-id=server-6a4b3372-729a-f122-9465-d18f6df32d65 server-addr=6a4b3372-729a-f122-9465-d18f6df32d65 servers="[{Suffrage:Voter ID:server-7198e2e6-b1c3-0003-aae7-a998e679b2be Address:7198e2e6-b1c3-0003-aae7-a998e679b2be} {Suffrage:Voter ID:server-04f12023-517d-7f2b-f1cf-06545fbcde04 Address:04f12023-517d-7f2b-f1cf-06545fbcde04} {Suffrage:Voter ID:server-6a4b3372-729a-f122-9465-d18f6df32d65 Address:6a4b3372-729a-f122-9465-d18f6df32d65}]" + testing.go:149: server-7198e2e6-b1c3-0003-aae7-a998e679b2be: 2024-01-08T00:31:01.419Z [INFO] server-7198e2e6-b1c3-0003-aae7-a998e679b2be: updating configuration: command=AddStaging server-id=server-6a4b3372-729a-f122-9465-d18f6df32d65 server-addr=6a4b3372-729a-f122-9465-d18f6df32d65 servers="[{Suffrage:Voter ID:server-7198e2e6-b1c3-0003-aae7-a998e679b2be Address:7198e2e6-b1c3-0003-aae7-a998e679b2be} {Suffrage:Voter ID:server-04f12023-517d-7f2b-f1cf-06545fbcde04 Address:04f12023-517d-7f2b-f1cf-06545fbcde04} {Suffrage:Voter ID:server-6a4b3372-729a-f122-9465-d18f6df32d65 Address:6a4b3372-729a-f122-9465-d18f6df32d65}]" +testLoggerAdapter verbose: server-7198e2e6-b1c3-0003-aae7-a998e679b2be: 2024-01-08T00:31:01.419Z [INFO] server-7198e2e6-b1c3-0003-aae7-a998e679b2be: added peer, starting replication: peer=server-6a4b3372-729a-f122-9465-d18f6df32d65 + testing.go:149: server-7198e2e6-b1c3-0003-aae7-a998e679b2be: 2024-01-08T00:31:01.419Z [INFO] server-7198e2e6-b1c3-0003-aae7-a998e679b2be: added peer, starting replication: peer=server-6a4b3372-729a-f122-9465-d18f6df32d65 +testLoggerAdapter verbose: server-6a4b3372-729a-f122-9465-d18f6df32d65: 2024-01-08T00:31:01.419Z [WARN] server-6a4b3372-729a-f122-9465-d18f6df32d65: failed to get previous log: previous-index=3 last-index=0 error="log not found" + testing.go:149: server-6a4b3372-729a-f122-9465-d18f6df32d65: 2024-01-08T00:31:01.419Z [WARN] server-6a4b3372-729a-f122-9465-d18f6df32d65: failed to get previous log: previous-index=3 last-index=0 error="log not found" +testLoggerAdapter verbose: server-7198e2e6-b1c3-0003-aae7-a998e679b2be: 2024-01-08T00:31:01.419Z [WARN] server-7198e2e6-b1c3-0003-aae7-a998e679b2be: appendEntries rejected, sending older logs: peer="{Voter server-6a4b3372-729a-f122-9465-d18f6df32d65 6a4b3372-729a-f122-9465-d18f6df32d65}" next=1 + testing.go:149: server-7198e2e6-b1c3-0003-aae7-a998e679b2be: 2024-01-08T00:31:01.419Z [WARN] server-7198e2e6-b1c3-0003-aae7-a998e679b2be: appendEntries rejected, sending older logs: peer="{Voter server-6a4b3372-729a-f122-9465-d18f6df32d65 6a4b3372-729a-f122-9465-d18f6df32d65}" next=1 +testLoggerAdapter verbose: server-7198e2e6-b1c3-0003-aae7-a998e679b2be: 2024-01-08T00:31:01.420Z [INFO] server-7198e2e6-b1c3-0003-aae7-a998e679b2be: pipelining replication: peer="{Voter server-6a4b3372-729a-f122-9465-d18f6df32d65 6a4b3372-729a-f122-9465-d18f6df32d65}" + testing.go:149: server-7198e2e6-b1c3-0003-aae7-a998e679b2be: 2024-01-08T00:31:01.420Z [INFO] server-7198e2e6-b1c3-0003-aae7-a998e679b2be: pipelining replication: peer="{Voter server-6a4b3372-729a-f122-9465-d18f6df32d65 6a4b3372-729a-f122-9465-d18f6df32d65}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:01.420Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:01.420Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:01.525Z [INFO] stable state for Leader reached at 2024-01-08 00:31:01.420274228 +0000 UTC m=+7.467163510 (1 nodes), 2.18µs from start of poll, 160.399105ms from cluster start. Timeout at 2024-01-08 00:31:01.525741828 +0000 UTC m=+7.572631066, 105.467556ms after stability + testing.go:149: cluster: 2024-01-08T00:31:01.525Z [INFO] stable state for Leader reached at 2024-01-08 00:31:01.420274228 +0000 UTC m=+7.467163510 (1 nodes), 2.18µs from start of poll, 160.399105ms from cluster start. Timeout at 2024-01-08 00:31:01.525741828 +0000 UTC m=+7.572631066, 105.467556ms after stability +testLoggerAdapter verbose: server-7198e2e6-b1c3-0003-aae7-a998e679b2be: 2024-01-08T00:31:01.525Z [INFO] server-7198e2e6-b1c3-0003-aae7-a998e679b2be: aborting pipeline replication: peer="{Voter server-6a4b3372-729a-f122-9465-d18f6df32d65 6a4b3372-729a-f122-9465-d18f6df32d65}" + testing.go:149: server-7198e2e6-b1c3-0003-aae7-a998e679b2be: 2024-01-08T00:31:01.525Z [INFO] server-7198e2e6-b1c3-0003-aae7-a998e679b2be: aborting pipeline replication: peer="{Voter server-6a4b3372-729a-f122-9465-d18f6df32d65 6a4b3372-729a-f122-9465-d18f6df32d65}" +testLoggerAdapter verbose: server-7198e2e6-b1c3-0003-aae7-a998e679b2be: 2024-01-08T00:31:01.526Z [INFO] server-7198e2e6-b1c3-0003-aae7-a998e679b2be: aborting pipeline replication: peer="{Voter server-04f12023-517d-7f2b-f1cf-06545fbcde04 04f12023-517d-7f2b-f1cf-06545fbcde04}" + testing.go:149: server-7198e2e6-b1c3-0003-aae7-a998e679b2be: 2024-01-08T00:31:01.526Z [INFO] server-7198e2e6-b1c3-0003-aae7-a998e679b2be: aborting pipeline replication: peer="{Voter server-04f12023-517d-7f2b-f1cf-06545fbcde04 04f12023-517d-7f2b-f1cf-06545fbcde04}" +--- PASS: TestRaft_JoinNode (0.27s) === RUN TestRaft_JoinNode_ConfigStore -testLoggerAdapter verbose: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.478Z [INFO] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-06a37545-98a9-5e84-74a8-fc56d794d5b0 Address:06a37545-98a9-5e84-74a8-fc56d794d5b0}]" - testing.go:149: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.478Z [INFO] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-06a37545-98a9-5e84-74a8-fc56d794d5b0 Address:06a37545-98a9-5e84-74a8-fc56d794d5b0}]" -testLoggerAdapter verbose: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.478Z [INFO] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: entering follower state: follower="Node at 06a37545-98a9-5e84-74a8-fc56d794d5b0 [Follower]" leader= - testing.go:149: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.478Z [INFO] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: entering follower state: follower="Node at 06a37545-98a9-5e84-74a8-fc56d794d5b0 [Follower]" leader= -testLoggerAdapter verbose: server-fa50b62f-5914-cf72-40da-c508a2882fa2: 2025-02-09T06:52:26.478Z [INFO] server-fa50b62f-5914-cf72-40da-c508a2882fa2: initial configuration: index=0 servers=[] - testing.go:149: server-fa50b62f-5914-cf72-40da-c508a2882fa2: 2025-02-09T06:52:26.478Z [INFO] server-fa50b62f-5914-cf72-40da-c508a2882fa2: initial configuration: index=0 servers=[] -testLoggerAdapter verbose: server-fa50b62f-5914-cf72-40da-c508a2882fa2: 2025-02-09T06:52:26.479Z [INFO] server-fa50b62f-5914-cf72-40da-c508a2882fa2: entering follower state: follower="Node at fa50b62f-5914-cf72-40da-c508a2882fa2 [Follower]" leader= - testing.go:149: server-fa50b62f-5914-cf72-40da-c508a2882fa2: 2025-02-09T06:52:26.479Z [INFO] server-fa50b62f-5914-cf72-40da-c508a2882fa2: entering follower state: follower="Node at fa50b62f-5914-cf72-40da-c508a2882fa2 [Follower]" leader= -testLoggerAdapter verbose: server-f9d6f83f-2682-e217-a8d0-30bac4ef600d: 2025-02-09T06:52:26.479Z [INFO] server-f9d6f83f-2682-e217-a8d0-30bac4ef600d: initial configuration: index=0 servers=[] - testing.go:149: server-f9d6f83f-2682-e217-a8d0-30bac4ef600d: 2025-02-09T06:52:26.479Z [INFO] server-f9d6f83f-2682-e217-a8d0-30bac4ef600d: initial configuration: index=0 servers=[] -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:26.479Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:26.479Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-f9d6f83f-2682-e217-a8d0-30bac4ef600d: 2025-02-09T06:52:26.479Z [INFO] server-f9d6f83f-2682-e217-a8d0-30bac4ef600d: entering follower state: follower="Node at f9d6f83f-2682-e217-a8d0-30bac4ef600d [Follower]" leader= - testing.go:149: server-f9d6f83f-2682-e217-a8d0-30bac4ef600d: 2025-02-09T06:52:26.479Z [INFO] server-f9d6f83f-2682-e217-a8d0-30bac4ef600d: entering follower state: follower="Node at f9d6f83f-2682-e217-a8d0-30bac4ef600d [Follower]" leader= -testLoggerAdapter verbose: server-fa50b62f-5914-cf72-40da-c508a2882fa2: 2025-02-09T06:52:26.554Z [WARN] server-fa50b62f-5914-cf72-40da-c508a2882fa2: no known peers, aborting election - testing.go:149: server-fa50b62f-5914-cf72-40da-c508a2882fa2: 2025-02-09T06:52:26.554Z [WARN] server-fa50b62f-5914-cf72-40da-c508a2882fa2: no known peers, aborting election -testLoggerAdapter verbose: server-f9d6f83f-2682-e217-a8d0-30bac4ef600d: 2025-02-09T06:52:26.563Z [WARN] server-f9d6f83f-2682-e217-a8d0-30bac4ef600d: no known peers, aborting election - testing.go:149: server-f9d6f83f-2682-e217-a8d0-30bac4ef600d: 2025-02-09T06:52:26.563Z [WARN] server-f9d6f83f-2682-e217-a8d0-30bac4ef600d: no known peers, aborting election -testLoggerAdapter verbose: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.572Z [WARN] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.572Z [WARN] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.572Z [INFO] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: entering candidate state: node="Node at 06a37545-98a9-5e84-74a8-fc56d794d5b0 [Candidate]" term=2 - testing.go:149: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.572Z [INFO] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: entering candidate state: node="Node at 06a37545-98a9-5e84-74a8-fc56d794d5b0 [Candidate]" term=2 -testLoggerAdapter verbose: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.572Z [INFO] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: election won: tally=1 - testing.go:149: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.572Z [INFO] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: election won: tally=1 -testLoggerAdapter verbose: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.572Z [INFO] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: entering leader state: leader="Node at 06a37545-98a9-5e84-74a8-fc56d794d5b0 [Leader]" - testing.go:149: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.572Z [INFO] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: entering leader state: leader="Node at 06a37545-98a9-5e84-74a8-fc56d794d5b0 [Leader]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:26.677Z [INFO] stable state for Leader reached at 2025-02-09 06:52:26.572268901 +0000 UTC m=+7.551077664 (1 nodes), 93.043349ms from start of poll, 93.607956ms from cluster start. Timeout at 2025-02-09 06:52:26.677418669 +0000 UTC m=+7.656227438, 105.149774ms after stability - testing.go:149: cluster: 2025-02-09T06:52:26.677Z [INFO] stable state for Leader reached at 2025-02-09 06:52:26.572268901 +0000 UTC m=+7.551077664 (1 nodes), 93.043349ms from start of poll, 93.607956ms from cluster start. Timeout at 2025-02-09 06:52:26.677418669 +0000 UTC m=+7.656227438, 105.149774ms after stability -testLoggerAdapter verbose: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.677Z [INFO] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: updating configuration: command=AddStaging server-id=server-fa50b62f-5914-cf72-40da-c508a2882fa2 server-addr=fa50b62f-5914-cf72-40da-c508a2882fa2 servers="[{Suffrage:Voter ID:server-06a37545-98a9-5e84-74a8-fc56d794d5b0 Address:06a37545-98a9-5e84-74a8-fc56d794d5b0} {Suffrage:Voter ID:server-fa50b62f-5914-cf72-40da-c508a2882fa2 Address:fa50b62f-5914-cf72-40da-c508a2882fa2}]" - testing.go:149: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.677Z [INFO] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: updating configuration: command=AddStaging server-id=server-fa50b62f-5914-cf72-40da-c508a2882fa2 server-addr=fa50b62f-5914-cf72-40da-c508a2882fa2 servers="[{Suffrage:Voter ID:server-06a37545-98a9-5e84-74a8-fc56d794d5b0 Address:06a37545-98a9-5e84-74a8-fc56d794d5b0} {Suffrage:Voter ID:server-fa50b62f-5914-cf72-40da-c508a2882fa2 Address:fa50b62f-5914-cf72-40da-c508a2882fa2}]" -testLoggerAdapter verbose: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.677Z [INFO] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: added peer, starting replication: peer=server-fa50b62f-5914-cf72-40da-c508a2882fa2 - testing.go:149: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.677Z [INFO] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: added peer, starting replication: peer=server-fa50b62f-5914-cf72-40da-c508a2882fa2 -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:26.677Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-fa50b62f-5914-cf72-40da-c508a2882fa2: 2025-02-09T06:52:26.677Z [WARN] server-fa50b62f-5914-cf72-40da-c508a2882fa2: failed to get previous log: previous-index=3 last-index=0 error="log not found" - testing.go:149: cluster: 2025-02-09T06:52:26.677Z [INFO] starting stability test: raft-state=Leader - testing.go:149: server-fa50b62f-5914-cf72-40da-c508a2882fa2: 2025-02-09T06:52:26.677Z [WARN] server-fa50b62f-5914-cf72-40da-c508a2882fa2: failed to get previous log: previous-index=3 last-index=0 error="log not found" -testLoggerAdapter verbose: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.677Z [WARN] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: appendEntries rejected, sending older logs: peer="{Voter server-fa50b62f-5914-cf72-40da-c508a2882fa2 fa50b62f-5914-cf72-40da-c508a2882fa2}" next=1 - testing.go:149: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.677Z [WARN] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: appendEntries rejected, sending older logs: peer="{Voter server-fa50b62f-5914-cf72-40da-c508a2882fa2 fa50b62f-5914-cf72-40da-c508a2882fa2}" next=1 -testLoggerAdapter verbose: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.677Z [INFO] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: pipelining replication: peer="{Voter server-fa50b62f-5914-cf72-40da-c508a2882fa2 fa50b62f-5914-cf72-40da-c508a2882fa2}" - testing.go:149: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.677Z [INFO] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: pipelining replication: peer="{Voter server-fa50b62f-5914-cf72-40da-c508a2882fa2 fa50b62f-5914-cf72-40da-c508a2882fa2}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:26.783Z [INFO] stable state for Leader reached at 2025-02-09 06:52:26.677627384 +0000 UTC m=+7.656436148 (1 nodes), 405ns from start of poll, 198.96644ms from cluster start. Timeout at 2025-02-09 06:52:26.783313656 +0000 UTC m=+7.762122425, 105.686277ms after stability - testing.go:149: cluster: 2025-02-09T06:52:26.783Z [INFO] stable state for Leader reached at 2025-02-09 06:52:26.677627384 +0000 UTC m=+7.656436148 (1 nodes), 405ns from start of poll, 198.96644ms from cluster start. Timeout at 2025-02-09 06:52:26.783313656 +0000 UTC m=+7.762122425, 105.686277ms after stability -testLoggerAdapter verbose: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.783Z [INFO] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: updating configuration: command=AddStaging server-id=server-f9d6f83f-2682-e217-a8d0-30bac4ef600d server-addr=f9d6f83f-2682-e217-a8d0-30bac4ef600d servers="[{Suffrage:Voter ID:server-06a37545-98a9-5e84-74a8-fc56d794d5b0 Address:06a37545-98a9-5e84-74a8-fc56d794d5b0} {Suffrage:Voter ID:server-fa50b62f-5914-cf72-40da-c508a2882fa2 Address:fa50b62f-5914-cf72-40da-c508a2882fa2} {Suffrage:Voter ID:server-f9d6f83f-2682-e217-a8d0-30bac4ef600d Address:f9d6f83f-2682-e217-a8d0-30bac4ef600d}]" - testing.go:149: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.783Z [INFO] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: updating configuration: command=AddStaging server-id=server-f9d6f83f-2682-e217-a8d0-30bac4ef600d server-addr=f9d6f83f-2682-e217-a8d0-30bac4ef600d servers="[{Suffrage:Voter ID:server-06a37545-98a9-5e84-74a8-fc56d794d5b0 Address:06a37545-98a9-5e84-74a8-fc56d794d5b0} {Suffrage:Voter ID:server-fa50b62f-5914-cf72-40da-c508a2882fa2 Address:fa50b62f-5914-cf72-40da-c508a2882fa2} {Suffrage:Voter ID:server-f9d6f83f-2682-e217-a8d0-30bac4ef600d Address:f9d6f83f-2682-e217-a8d0-30bac4ef600d}]" -testLoggerAdapter verbose: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.783Z [INFO] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: added peer, starting replication: peer=server-f9d6f83f-2682-e217-a8d0-30bac4ef600d - testing.go:149: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.783Z [INFO] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: added peer, starting replication: peer=server-f9d6f83f-2682-e217-a8d0-30bac4ef600d -testLoggerAdapter verbose: server-f9d6f83f-2682-e217-a8d0-30bac4ef600d: 2025-02-09T06:52:26.783Z [WARN] server-f9d6f83f-2682-e217-a8d0-30bac4ef600d: failed to get previous log: previous-index=4 last-index=0 error="log not found" - testing.go:149: server-f9d6f83f-2682-e217-a8d0-30bac4ef600d: 2025-02-09T06:52:26.783Z [WARN] server-f9d6f83f-2682-e217-a8d0-30bac4ef600d: failed to get previous log: previous-index=4 last-index=0 error="log not found" -testLoggerAdapter verbose: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.783Z [WARN] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: appendEntries rejected, sending older logs: peer="{Voter server-f9d6f83f-2682-e217-a8d0-30bac4ef600d f9d6f83f-2682-e217-a8d0-30bac4ef600d}" next=1 - testing.go:149: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.783Z [WARN] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: appendEntries rejected, sending older logs: peer="{Voter server-f9d6f83f-2682-e217-a8d0-30bac4ef600d f9d6f83f-2682-e217-a8d0-30bac4ef600d}" next=1 -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:26.783Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:26.783Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.783Z [INFO] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: pipelining replication: peer="{Voter server-f9d6f83f-2682-e217-a8d0-30bac4ef600d f9d6f83f-2682-e217-a8d0-30bac4ef600d}" - testing.go:149: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.783Z [INFO] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: pipelining replication: peer="{Voter server-f9d6f83f-2682-e217-a8d0-30bac4ef600d f9d6f83f-2682-e217-a8d0-30bac4ef600d}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:26.888Z [INFO] stable state for Leader reached at 2025-02-09 06:52:26.783637964 +0000 UTC m=+7.762446727 (1 nodes), 463ns from start of poll, 304.977019ms from cluster start. Timeout at 2025-02-09 06:52:26.88872786 +0000 UTC m=+7.867536619, 105.089892ms after stability - testing.go:149: cluster: 2025-02-09T06:52:26.888Z [INFO] stable state for Leader reached at 2025-02-09 06:52:26.783637964 +0000 UTC m=+7.762446727 (1 nodes), 463ns from start of poll, 304.977019ms from cluster start. Timeout at 2025-02-09 06:52:26.88872786 +0000 UTC m=+7.867536619, 105.089892ms after stability -testLoggerAdapter verbose: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.888Z [INFO] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: aborting pipeline replication: peer="{Voter server-f9d6f83f-2682-e217-a8d0-30bac4ef600d f9d6f83f-2682-e217-a8d0-30bac4ef600d}" - testing.go:149: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.888Z [INFO] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: aborting pipeline replication: peer="{Voter server-f9d6f83f-2682-e217-a8d0-30bac4ef600d f9d6f83f-2682-e217-a8d0-30bac4ef600d}" -testLoggerAdapter verbose: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.888Z [INFO] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: aborting pipeline replication: peer="{Voter server-fa50b62f-5914-cf72-40da-c508a2882fa2 fa50b62f-5914-cf72-40da-c508a2882fa2}" - testing.go:149: server-06a37545-98a9-5e84-74a8-fc56d794d5b0: 2025-02-09T06:52:26.888Z [INFO] server-06a37545-98a9-5e84-74a8-fc56d794d5b0: aborting pipeline replication: peer="{Voter server-fa50b62f-5914-cf72-40da-c508a2882fa2 fa50b62f-5914-cf72-40da-c508a2882fa2}" ---- PASS: TestRaft_JoinNode_ConfigStore (0.41s) +testLoggerAdapter verbose: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.527Z [INFO] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-db8a847a-b3d9-9eb0-022b-79c71f04809d Address:db8a847a-b3d9-9eb0-022b-79c71f04809d}]" + testing.go:149: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.527Z [INFO] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-db8a847a-b3d9-9eb0-022b-79c71f04809d Address:db8a847a-b3d9-9eb0-022b-79c71f04809d}]" +testLoggerAdapter verbose: server-f04316e6-db58-f69e-9ee1-2882377e6fff: 2024-01-08T00:31:01.527Z [INFO] server-f04316e6-db58-f69e-9ee1-2882377e6fff: initial configuration: index=0 servers=[] + testing.go:149: server-f04316e6-db58-f69e-9ee1-2882377e6fff: 2024-01-08T00:31:01.527Z [INFO] server-f04316e6-db58-f69e-9ee1-2882377e6fff: initial configuration: index=0 servers=[] +testLoggerAdapter verbose: server-3994d176-a39a-d04f-3682-d70777352990: 2024-01-08T00:31:01.527Z [INFO] server-3994d176-a39a-d04f-3682-d70777352990: initial configuration: index=0 servers=[] + testing.go:149: server-3994d176-a39a-d04f-3682-d70777352990: 2024-01-08T00:31:01.527Z [INFO] server-3994d176-a39a-d04f-3682-d70777352990: initial configuration: index=0 servers=[] +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:01.527Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:01.527Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.527Z [INFO] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: entering follower state: follower="Node at db8a847a-b3d9-9eb0-022b-79c71f04809d [Follower]" leader= + testing.go:149: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.527Z [INFO] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: entering follower state: follower="Node at db8a847a-b3d9-9eb0-022b-79c71f04809d [Follower]" leader= +testLoggerAdapter verbose: server-f04316e6-db58-f69e-9ee1-2882377e6fff: 2024-01-08T00:31:01.527Z [INFO] server-f04316e6-db58-f69e-9ee1-2882377e6fff: entering follower state: follower="Node at f04316e6-db58-f69e-9ee1-2882377e6fff [Follower]" leader= + testing.go:149: server-f04316e6-db58-f69e-9ee1-2882377e6fff: 2024-01-08T00:31:01.527Z [INFO] server-f04316e6-db58-f69e-9ee1-2882377e6fff: entering follower state: follower="Node at f04316e6-db58-f69e-9ee1-2882377e6fff [Follower]" leader= +testLoggerAdapter verbose: server-3994d176-a39a-d04f-3682-d70777352990: 2024-01-08T00:31:01.527Z [INFO] server-3994d176-a39a-d04f-3682-d70777352990: entering follower state: follower="Node at 3994d176-a39a-d04f-3682-d70777352990 [Follower]" leader= + testing.go:149: server-3994d176-a39a-d04f-3682-d70777352990: 2024-01-08T00:31:01.527Z [INFO] server-3994d176-a39a-d04f-3682-d70777352990: entering follower state: follower="Node at 3994d176-a39a-d04f-3682-d70777352990 [Follower]" leader= +testLoggerAdapter verbose: server-f04316e6-db58-f69e-9ee1-2882377e6fff: 2024-01-08T00:31:01.588Z [WARN] server-f04316e6-db58-f69e-9ee1-2882377e6fff: no known peers, aborting election + testing.go:149: server-f04316e6-db58-f69e-9ee1-2882377e6fff: 2024-01-08T00:31:01.588Z [WARN] server-f04316e6-db58-f69e-9ee1-2882377e6fff: no known peers, aborting election +testLoggerAdapter verbose: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.590Z [WARN] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.590Z [WARN] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.590Z [INFO] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: entering candidate state: node="Node at db8a847a-b3d9-9eb0-022b-79c71f04809d [Candidate]" term=2 + testing.go:149: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.590Z [INFO] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: entering candidate state: node="Node at db8a847a-b3d9-9eb0-022b-79c71f04809d [Candidate]" term=2 +testLoggerAdapter verbose: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.590Z [INFO] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: election won: tally=1 + testing.go:149: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.590Z [INFO] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: election won: tally=1 +testLoggerAdapter verbose: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.590Z [INFO] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: entering leader state: leader="Node at db8a847a-b3d9-9eb0-022b-79c71f04809d [Leader]" + testing.go:149: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.590Z [INFO] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: entering leader state: leader="Node at db8a847a-b3d9-9eb0-022b-79c71f04809d [Leader]" +testLoggerAdapter verbose: server-3994d176-a39a-d04f-3682-d70777352990: 2024-01-08T00:31:01.597Z [WARN] server-3994d176-a39a-d04f-3682-d70777352990: no known peers, aborting election + testing.go:149: server-3994d176-a39a-d04f-3682-d70777352990: 2024-01-08T00:31:01.597Z [WARN] server-3994d176-a39a-d04f-3682-d70777352990: no known peers, aborting election +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:01.695Z [INFO] stable state for Leader reached at 2024-01-08 00:31:01.59051652 +0000 UTC m=+7.637405624 (1 nodes), 62.814091ms from start of poll, 63.47673ms from cluster start. Timeout at 2024-01-08 00:31:01.695537593 +0000 UTC m=+7.742426818, 105.021194ms after stability + testing.go:149: cluster: 2024-01-08T00:31:01.695Z [INFO] stable state for Leader reached at 2024-01-08 00:31:01.59051652 +0000 UTC m=+7.637405624 (1 nodes), 62.814091ms from start of poll, 63.47673ms from cluster start. Timeout at 2024-01-08 00:31:01.695537593 +0000 UTC m=+7.742426818, 105.021194ms after stability +testLoggerAdapter verbose: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.695Z [INFO] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: updating configuration: command=AddStaging server-id=server-f04316e6-db58-f69e-9ee1-2882377e6fff server-addr=f04316e6-db58-f69e-9ee1-2882377e6fff servers="[{Suffrage:Voter ID:server-db8a847a-b3d9-9eb0-022b-79c71f04809d Address:db8a847a-b3d9-9eb0-022b-79c71f04809d} {Suffrage:Voter ID:server-f04316e6-db58-f69e-9ee1-2882377e6fff Address:f04316e6-db58-f69e-9ee1-2882377e6fff}]" + testing.go:149: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.695Z [INFO] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: updating configuration: command=AddStaging server-id=server-f04316e6-db58-f69e-9ee1-2882377e6fff server-addr=f04316e6-db58-f69e-9ee1-2882377e6fff servers="[{Suffrage:Voter ID:server-db8a847a-b3d9-9eb0-022b-79c71f04809d Address:db8a847a-b3d9-9eb0-022b-79c71f04809d} {Suffrage:Voter ID:server-f04316e6-db58-f69e-9ee1-2882377e6fff Address:f04316e6-db58-f69e-9ee1-2882377e6fff}]" +testLoggerAdapter verbose: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.695Z [INFO] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: added peer, starting replication: peer=server-f04316e6-db58-f69e-9ee1-2882377e6fff + testing.go:149: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.695Z [INFO] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: added peer, starting replication: peer=server-f04316e6-db58-f69e-9ee1-2882377e6fff +testLoggerAdapter verbose: server-f04316e6-db58-f69e-9ee1-2882377e6fff: 2024-01-08T00:31:01.695Z [WARN] server-f04316e6-db58-f69e-9ee1-2882377e6fff: failed to get previous log: previous-index=3 last-index=0 error="log not found" + testing.go:149: server-f04316e6-db58-f69e-9ee1-2882377e6fff: 2024-01-08T00:31:01.695Z [WARN] server-f04316e6-db58-f69e-9ee1-2882377e6fff: failed to get previous log: previous-index=3 last-index=0 error="log not found" +testLoggerAdapter verbose: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.695Z [WARN] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: appendEntries rejected, sending older logs: peer="{Voter server-f04316e6-db58-f69e-9ee1-2882377e6fff f04316e6-db58-f69e-9ee1-2882377e6fff}" next=1 + testing.go:149: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.695Z [WARN] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: appendEntries rejected, sending older logs: peer="{Voter server-f04316e6-db58-f69e-9ee1-2882377e6fff f04316e6-db58-f69e-9ee1-2882377e6fff}" next=1 +testLoggerAdapter verbose: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.696Z [INFO] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: pipelining replication: peer="{Voter server-f04316e6-db58-f69e-9ee1-2882377e6fff f04316e6-db58-f69e-9ee1-2882377e6fff}" + testing.go:149: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.696Z [INFO] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: pipelining replication: peer="{Voter server-f04316e6-db58-f69e-9ee1-2882377e6fff f04316e6-db58-f69e-9ee1-2882377e6fff}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:01.696Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:01.696Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:01.801Z [INFO] stable state for Leader reached at 2024-01-08 00:31:01.696189828 +0000 UTC m=+7.743079006 (1 nodes), 2.274µs from start of poll, 169.150112ms from cluster start. Timeout at 2024-01-08 00:31:01.801544492 +0000 UTC m=+7.848433727, 105.354721ms after stability + testing.go:149: cluster: 2024-01-08T00:31:01.801Z [INFO] stable state for Leader reached at 2024-01-08 00:31:01.696189828 +0000 UTC m=+7.743079006 (1 nodes), 2.274µs from start of poll, 169.150112ms from cluster start. Timeout at 2024-01-08 00:31:01.801544492 +0000 UTC m=+7.848433727, 105.354721ms after stability +testLoggerAdapter verbose: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.801Z [INFO] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: updating configuration: command=AddStaging server-id=server-3994d176-a39a-d04f-3682-d70777352990 server-addr=3994d176-a39a-d04f-3682-d70777352990 servers="[{Suffrage:Voter ID:server-db8a847a-b3d9-9eb0-022b-79c71f04809d Address:db8a847a-b3d9-9eb0-022b-79c71f04809d} {Suffrage:Voter ID:server-f04316e6-db58-f69e-9ee1-2882377e6fff Address:f04316e6-db58-f69e-9ee1-2882377e6fff} {Suffrage:Voter ID:server-3994d176-a39a-d04f-3682-d70777352990 Address:3994d176-a39a-d04f-3682-d70777352990}]" + testing.go:149: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.801Z [INFO] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: updating configuration: command=AddStaging server-id=server-3994d176-a39a-d04f-3682-d70777352990 server-addr=3994d176-a39a-d04f-3682-d70777352990 servers="[{Suffrage:Voter ID:server-db8a847a-b3d9-9eb0-022b-79c71f04809d Address:db8a847a-b3d9-9eb0-022b-79c71f04809d} {Suffrage:Voter ID:server-f04316e6-db58-f69e-9ee1-2882377e6fff Address:f04316e6-db58-f69e-9ee1-2882377e6fff} {Suffrage:Voter ID:server-3994d176-a39a-d04f-3682-d70777352990 Address:3994d176-a39a-d04f-3682-d70777352990}]" +testLoggerAdapter verbose: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.801Z [INFO] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: added peer, starting replication: peer=server-3994d176-a39a-d04f-3682-d70777352990 + testing.go:149: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.801Z [INFO] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: added peer, starting replication: peer=server-3994d176-a39a-d04f-3682-d70777352990 +testLoggerAdapter verbose: server-3994d176-a39a-d04f-3682-d70777352990: 2024-01-08T00:31:01.801Z [WARN] server-3994d176-a39a-d04f-3682-d70777352990: failed to get previous log: previous-index=4 last-index=0 error="log not found" + testing.go:149: server-3994d176-a39a-d04f-3682-d70777352990: 2024-01-08T00:31:01.801Z [WARN] server-3994d176-a39a-d04f-3682-d70777352990: failed to get previous log: previous-index=4 last-index=0 error="log not found" +testLoggerAdapter verbose: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.801Z [WARN] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: appendEntries rejected, sending older logs: peer="{Voter server-3994d176-a39a-d04f-3682-d70777352990 3994d176-a39a-d04f-3682-d70777352990}" next=1 + testing.go:149: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.801Z [WARN] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: appendEntries rejected, sending older logs: peer="{Voter server-3994d176-a39a-d04f-3682-d70777352990 3994d176-a39a-d04f-3682-d70777352990}" next=1 +testLoggerAdapter verbose: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.802Z [INFO] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: pipelining replication: peer="{Voter server-3994d176-a39a-d04f-3682-d70777352990 3994d176-a39a-d04f-3682-d70777352990}" + testing.go:149: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.802Z [INFO] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: pipelining replication: peer="{Voter server-3994d176-a39a-d04f-3682-d70777352990 3994d176-a39a-d04f-3682-d70777352990}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:01.802Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:01.802Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:01.907Z [INFO] stable state for Leader reached at 2024-01-08 00:31:01.802255276 +0000 UTC m=+7.849144360 (1 nodes), 2.054µs from start of poll, 275.215466ms from cluster start. Timeout at 2024-01-08 00:31:01.907771546 +0000 UTC m=+7.954660687, 105.516327ms after stability + testing.go:149: cluster: 2024-01-08T00:31:01.907Z [INFO] stable state for Leader reached at 2024-01-08 00:31:01.802255276 +0000 UTC m=+7.849144360 (1 nodes), 2.054µs from start of poll, 275.215466ms from cluster start. Timeout at 2024-01-08 00:31:01.907771546 +0000 UTC m=+7.954660687, 105.516327ms after stability +testLoggerAdapter verbose: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.907Z [INFO] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: aborting pipeline replication: peer="{Voter server-3994d176-a39a-d04f-3682-d70777352990 3994d176-a39a-d04f-3682-d70777352990}" + testing.go:149: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.907Z [INFO] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: aborting pipeline replication: peer="{Voter server-3994d176-a39a-d04f-3682-d70777352990 3994d176-a39a-d04f-3682-d70777352990}" +testLoggerAdapter verbose: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.908Z [INFO] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: aborting pipeline replication: peer="{Voter server-f04316e6-db58-f69e-9ee1-2882377e6fff f04316e6-db58-f69e-9ee1-2882377e6fff}" + testing.go:149: server-db8a847a-b3d9-9eb0-022b-79c71f04809d: 2024-01-08T00:31:01.908Z [INFO] server-db8a847a-b3d9-9eb0-022b-79c71f04809d: aborting pipeline replication: peer="{Voter server-f04316e6-db58-f69e-9ee1-2882377e6fff f04316e6-db58-f69e-9ee1-2882377e6fff}" +--- PASS: TestRaft_JoinNode_ConfigStore (0.38s) === RUN TestRaft_RemoveFollower -testLoggerAdapter verbose: server-0f854936-6a58-c968-3e32-225fc0cb9cfd: 2025-02-09T06:52:26.889Z [INFO] server-0f854936-6a58-c968-3e32-225fc0cb9cfd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0f854936-6a58-c968-3e32-225fc0cb9cfd Address:0f854936-6a58-c968-3e32-225fc0cb9cfd} {Suffrage:Voter ID:server-f1dda9c9-5b2b-4c8f-c9da-043485878064 Address:f1dda9c9-5b2b-4c8f-c9da-043485878064} {Suffrage:Voter ID:server-c265bb91-4990-e65e-6aca-05e246ace2a1 Address:c265bb91-4990-e65e-6aca-05e246ace2a1}]" - testing.go:149: server-0f854936-6a58-c968-3e32-225fc0cb9cfd: 2025-02-09T06:52:26.889Z [INFO] server-0f854936-6a58-c968-3e32-225fc0cb9cfd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0f854936-6a58-c968-3e32-225fc0cb9cfd Address:0f854936-6a58-c968-3e32-225fc0cb9cfd} {Suffrage:Voter ID:server-f1dda9c9-5b2b-4c8f-c9da-043485878064 Address:f1dda9c9-5b2b-4c8f-c9da-043485878064} {Suffrage:Voter ID:server-c265bb91-4990-e65e-6aca-05e246ace2a1 Address:c265bb91-4990-e65e-6aca-05e246ace2a1}]" -testLoggerAdapter verbose: server-0f854936-6a58-c968-3e32-225fc0cb9cfd: 2025-02-09T06:52:26.889Z [INFO] server-0f854936-6a58-c968-3e32-225fc0cb9cfd: entering follower state: follower="Node at 0f854936-6a58-c968-3e32-225fc0cb9cfd [Follower]" leader= - testing.go:149: server-0f854936-6a58-c968-3e32-225fc0cb9cfd: 2025-02-09T06:52:26.889Z [INFO] server-0f854936-6a58-c968-3e32-225fc0cb9cfd: entering follower state: follower="Node at 0f854936-6a58-c968-3e32-225fc0cb9cfd [Follower]" leader= -testLoggerAdapter verbose: server-f1dda9c9-5b2b-4c8f-c9da-043485878064: 2025-02-09T06:52:26.889Z [INFO] server-f1dda9c9-5b2b-4c8f-c9da-043485878064: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0f854936-6a58-c968-3e32-225fc0cb9cfd Address:0f854936-6a58-c968-3e32-225fc0cb9cfd} {Suffrage:Voter ID:server-f1dda9c9-5b2b-4c8f-c9da-043485878064 Address:f1dda9c9-5b2b-4c8f-c9da-043485878064} {Suffrage:Voter ID:server-c265bb91-4990-e65e-6aca-05e246ace2a1 Address:c265bb91-4990-e65e-6aca-05e246ace2a1}]" - testing.go:149: server-f1dda9c9-5b2b-4c8f-c9da-043485878064: 2025-02-09T06:52:26.889Z [INFO] server-f1dda9c9-5b2b-4c8f-c9da-043485878064: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0f854936-6a58-c968-3e32-225fc0cb9cfd Address:0f854936-6a58-c968-3e32-225fc0cb9cfd} {Suffrage:Voter ID:server-f1dda9c9-5b2b-4c8f-c9da-043485878064 Address:f1dda9c9-5b2b-4c8f-c9da-043485878064} {Suffrage:Voter ID:server-c265bb91-4990-e65e-6aca-05e246ace2a1 Address:c265bb91-4990-e65e-6aca-05e246ace2a1}]" -testLoggerAdapter verbose: server-f1dda9c9-5b2b-4c8f-c9da-043485878064: 2025-02-09T06:52:26.889Z [INFO] server-f1dda9c9-5b2b-4c8f-c9da-043485878064: entering follower state: follower="Node at f1dda9c9-5b2b-4c8f-c9da-043485878064 [Follower]" leader= - testing.go:149: server-f1dda9c9-5b2b-4c8f-c9da-043485878064: 2025-02-09T06:52:26.889Z [INFO] server-f1dda9c9-5b2b-4c8f-c9da-043485878064: entering follower state: follower="Node at f1dda9c9-5b2b-4c8f-c9da-043485878064 [Follower]" leader= -testLoggerAdapter verbose: server-c265bb91-4990-e65e-6aca-05e246ace2a1: 2025-02-09T06:52:26.890Z [INFO] server-c265bb91-4990-e65e-6aca-05e246ace2a1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0f854936-6a58-c968-3e32-225fc0cb9cfd Address:0f854936-6a58-c968-3e32-225fc0cb9cfd} {Suffrage:Voter ID:server-f1dda9c9-5b2b-4c8f-c9da-043485878064 Address:f1dda9c9-5b2b-4c8f-c9da-043485878064} {Suffrage:Voter ID:server-c265bb91-4990-e65e-6aca-05e246ace2a1 Address:c265bb91-4990-e65e-6aca-05e246ace2a1}]" - testing.go:149: server-c265bb91-4990-e65e-6aca-05e246ace2a1: 2025-02-09T06:52:26.890Z [INFO] server-c265bb91-4990-e65e-6aca-05e246ace2a1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0f854936-6a58-c968-3e32-225fc0cb9cfd Address:0f854936-6a58-c968-3e32-225fc0cb9cfd} {Suffrage:Voter ID:server-f1dda9c9-5b2b-4c8f-c9da-043485878064 Address:f1dda9c9-5b2b-4c8f-c9da-043485878064} {Suffrage:Voter ID:server-c265bb91-4990-e65e-6aca-05e246ace2a1 Address:c265bb91-4990-e65e-6aca-05e246ace2a1}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:26.890Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:26.890Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-c265bb91-4990-e65e-6aca-05e246ace2a1: 2025-02-09T06:52:26.890Z [INFO] server-c265bb91-4990-e65e-6aca-05e246ace2a1: entering follower state: follower="Node at c265bb91-4990-e65e-6aca-05e246ace2a1 [Follower]" leader= - testing.go:149: server-c265bb91-4990-e65e-6aca-05e246ace2a1: 2025-02-09T06:52:26.890Z [INFO] server-c265bb91-4990-e65e-6aca-05e246ace2a1: entering follower state: follower="Node at c265bb91-4990-e65e-6aca-05e246ace2a1 [Follower]" leader= -testLoggerAdapter verbose: server-c265bb91-4990-e65e-6aca-05e246ace2a1: 2025-02-09T06:52:26.969Z [WARN] server-c265bb91-4990-e65e-6aca-05e246ace2a1: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-c265bb91-4990-e65e-6aca-05e246ace2a1: 2025-02-09T06:52:26.969Z [WARN] server-c265bb91-4990-e65e-6aca-05e246ace2a1: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-c265bb91-4990-e65e-6aca-05e246ace2a1: 2025-02-09T06:52:26.969Z [INFO] server-c265bb91-4990-e65e-6aca-05e246ace2a1: entering candidate state: node="Node at c265bb91-4990-e65e-6aca-05e246ace2a1 [Candidate]" term=2 - testing.go:149: server-c265bb91-4990-e65e-6aca-05e246ace2a1: 2025-02-09T06:52:26.969Z [INFO] server-c265bb91-4990-e65e-6aca-05e246ace2a1: entering candidate state: node="Node at c265bb91-4990-e65e-6aca-05e246ace2a1 [Candidate]" term=2 -testLoggerAdapter verbose: server-c265bb91-4990-e65e-6aca-05e246ace2a1: 2025-02-09T06:52:26.969Z [INFO] server-c265bb91-4990-e65e-6aca-05e246ace2a1: election won: tally=2 - testing.go:149: server-c265bb91-4990-e65e-6aca-05e246ace2a1: 2025-02-09T06:52:26.969Z [INFO] server-c265bb91-4990-e65e-6aca-05e246ace2a1: election won: tally=2 -testLoggerAdapter verbose: server-c265bb91-4990-e65e-6aca-05e246ace2a1: 2025-02-09T06:52:26.969Z [INFO] server-c265bb91-4990-e65e-6aca-05e246ace2a1: entering leader state: leader="Node at c265bb91-4990-e65e-6aca-05e246ace2a1 [Leader]" - testing.go:149: server-c265bb91-4990-e65e-6aca-05e246ace2a1: 2025-02-09T06:52:26.969Z [INFO] server-c265bb91-4990-e65e-6aca-05e246ace2a1: entering leader state: leader="Node at c265bb91-4990-e65e-6aca-05e246ace2a1 [Leader]" -testLoggerAdapter verbose: server-c265bb91-4990-e65e-6aca-05e246ace2a1: 2025-02-09T06:52:26.969Z [INFO] server-c265bb91-4990-e65e-6aca-05e246ace2a1: added peer, starting replication: peer=server-0f854936-6a58-c968-3e32-225fc0cb9cfd - testing.go:149: server-c265bb91-4990-e65e-6aca-05e246ace2a1: 2025-02-09T06:52:26.969Z [INFO] server-c265bb91-4990-e65e-6aca-05e246ace2a1: added peer, starting replication: peer=server-0f854936-6a58-c968-3e32-225fc0cb9cfd -testLoggerAdapter verbose: server-c265bb91-4990-e65e-6aca-05e246ace2a1: 2025-02-09T06:52:26.969Z [INFO] server-c265bb91-4990-e65e-6aca-05e246ace2a1: added peer, starting replication: peer=server-f1dda9c9-5b2b-4c8f-c9da-043485878064 - testing.go:149: server-c265bb91-4990-e65e-6aca-05e246ace2a1: 2025-02-09T06:52:26.969Z [INFO] server-c265bb91-4990-e65e-6aca-05e246ace2a1: added peer, starting replication: peer=server-f1dda9c9-5b2b-4c8f-c9da-043485878064 -testLoggerAdapter verbose: server-c265bb91-4990-e65e-6aca-05e246ace2a1: 2025-02-09T06:52:26.969Z [INFO] server-c265bb91-4990-e65e-6aca-05e246ace2a1: pipelining replication: peer="{Voter server-f1dda9c9-5b2b-4c8f-c9da-043485878064 f1dda9c9-5b2b-4c8f-c9da-043485878064}" - testing.go:149: server-c265bb91-4990-e65e-6aca-05e246ace2a1: 2025-02-09T06:52:26.969Z [INFO] server-c265bb91-4990-e65e-6aca-05e246ace2a1: pipelining replication: peer="{Voter server-f1dda9c9-5b2b-4c8f-c9da-043485878064 f1dda9c9-5b2b-4c8f-c9da-043485878064}" -testLoggerAdapter verbose: server-c265bb91-4990-e65e-6aca-05e246ace2a1: 2025-02-09T06:52:26.969Z [INFO] server-c265bb91-4990-e65e-6aca-05e246ace2a1: pipelining replication: peer="{Voter server-0f854936-6a58-c968-3e32-225fc0cb9cfd 0f854936-6a58-c968-3e32-225fc0cb9cfd}" - testing.go:149: server-c265bb91-4990-e65e-6aca-05e246ace2a1: 2025-02-09T06:52:26.969Z [INFO] server-c265bb91-4990-e65e-6aca-05e246ace2a1: pipelining replication: peer="{Voter server-0f854936-6a58-c968-3e32-225fc0cb9cfd 0f854936-6a58-c968-3e32-225fc0cb9cfd}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:27.075Z [INFO] stable state for Leader reached at 2025-02-09 06:52:26.969841427 +0000 UTC m=+7.948650192 (1 nodes), 79.76955ms from start of poll, 80.172886ms from cluster start. Timeout at 2025-02-09 06:52:27.075767719 +0000 UTC m=+8.054576486, 105.926294ms after stability - testing.go:149: cluster: 2025-02-09T06:52:27.075Z [INFO] stable state for Leader reached at 2025-02-09 06:52:26.969841427 +0000 UTC m=+7.948650192 (1 nodes), 79.76955ms from start of poll, 80.172886ms from cluster start. Timeout at 2025-02-09 06:52:27.075767719 +0000 UTC m=+8.054576486, 105.926294ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:27.081Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T06:52:27.081Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:27.186Z [INFO] stable state for Follower reached at 2025-02-09 06:52:27.081187911 +0000 UTC m=+8.059996674 (2 nodes), 750ns from start of poll, 191.519368ms from cluster start. Timeout at 2025-02-09 06:52:27.186791861 +0000 UTC m=+8.165600618, 105.603944ms after stability - testing.go:149: cluster: 2025-02-09T06:52:27.186Z [INFO] stable state for Follower reached at 2025-02-09 06:52:27.081187911 +0000 UTC m=+8.059996674 (2 nodes), 750ns from start of poll, 191.519368ms from cluster start. Timeout at 2025-02-09 06:52:27.186791861 +0000 UTC m=+8.165600618, 105.603944ms after stability -testLoggerAdapter verbose: server-c265bb91-4990-e65e-6aca-05e246ace2a1: 2025-02-09T06:52:27.186Z [INFO] server-c265bb91-4990-e65e-6aca-05e246ace2a1: updating configuration: command=RemoveServer server-id=server-0f854936-6a58-c968-3e32-225fc0cb9cfd server-addr= servers="[{Suffrage:Voter ID:server-f1dda9c9-5b2b-4c8f-c9da-043485878064 Address:f1dda9c9-5b2b-4c8f-c9da-043485878064} {Suffrage:Voter ID:server-c265bb91-4990-e65e-6aca-05e246ace2a1 Address:c265bb91-4990-e65e-6aca-05e246ace2a1}]" - testing.go:149: server-c265bb91-4990-e65e-6aca-05e246ace2a1: 2025-02-09T06:52:27.186Z [INFO] server-c265bb91-4990-e65e-6aca-05e246ace2a1: updating configuration: command=RemoveServer server-id=server-0f854936-6a58-c968-3e32-225fc0cb9cfd server-addr= servers="[{Suffrage:Voter ID:server-f1dda9c9-5b2b-4c8f-c9da-043485878064 Address:f1dda9c9-5b2b-4c8f-c9da-043485878064} {Suffrage:Voter ID:server-c265bb91-4990-e65e-6aca-05e246ace2a1 Address:c265bb91-4990-e65e-6aca-05e246ace2a1}]" -testLoggerAdapter verbose: server-c265bb91-4990-e65e-6aca-05e246ace2a1: 2025-02-09T06:52:27.186Z [INFO] server-c265bb91-4990-e65e-6aca-05e246ace2a1: removed peer, stopping replication: peer=server-0f854936-6a58-c968-3e32-225fc0cb9cfd last-index=3 - testing.go:149: server-c265bb91-4990-e65e-6aca-05e246ace2a1: 2025-02-09T06:52:27.186Z [INFO] server-c265bb91-4990-e65e-6aca-05e246ace2a1: removed peer, stopping replication: peer=server-0f854936-6a58-c968-3e32-225fc0cb9cfd last-index=3 -testLoggerAdapter verbose: server-c265bb91-4990-e65e-6aca-05e246ace2a1: 2025-02-09T06:52:27.186Z [INFO] server-c265bb91-4990-e65e-6aca-05e246ace2a1: aborting pipeline replication: peer="{Voter server-0f854936-6a58-c968-3e32-225fc0cb9cfd 0f854936-6a58-c968-3e32-225fc0cb9cfd}" - testing.go:149: server-c265bb91-4990-e65e-6aca-05e246ace2a1: 2025-02-09T06:52:27.186Z [INFO] server-c265bb91-4990-e65e-6aca-05e246ace2a1: aborting pipeline replication: peer="{Voter server-0f854936-6a58-c968-3e32-225fc0cb9cfd 0f854936-6a58-c968-3e32-225fc0cb9cfd}" -testLoggerAdapter verbose: server-c265bb91-4990-e65e-6aca-05e246ace2a1: 2025-02-09T06:52:27.292Z [INFO] server-c265bb91-4990-e65e-6aca-05e246ace2a1: aborting pipeline replication: peer="{Voter server-f1dda9c9-5b2b-4c8f-c9da-043485878064 f1dda9c9-5b2b-4c8f-c9da-043485878064}" - testing.go:149: server-c265bb91-4990-e65e-6aca-05e246ace2a1: 2025-02-09T06:52:27.292Z [INFO] server-c265bb91-4990-e65e-6aca-05e246ace2a1: aborting pipeline replication: peer="{Voter server-f1dda9c9-5b2b-4c8f-c9da-043485878064 f1dda9c9-5b2b-4c8f-c9da-043485878064}" ---- PASS: TestRaft_RemoveFollower (0.40s) +testLoggerAdapter verbose: server-3f9779dc-bb02-61c2-80be-2e80e1a434c5: 2024-01-08T00:31:01.909Z [INFO] server-3f9779dc-bb02-61c2-80be-2e80e1a434c5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3f9779dc-bb02-61c2-80be-2e80e1a434c5 Address:3f9779dc-bb02-61c2-80be-2e80e1a434c5} {Suffrage:Voter ID:server-8734b349-06b6-45f6-9f8f-b52a9e96fbd8 Address:8734b349-06b6-45f6-9f8f-b52a9e96fbd8} {Suffrage:Voter ID:server-3acca375-1934-dbdd-b9cd-bef661b10ce0 Address:3acca375-1934-dbdd-b9cd-bef661b10ce0}]" + testing.go:149: server-3f9779dc-bb02-61c2-80be-2e80e1a434c5: 2024-01-08T00:31:01.909Z [INFO] server-3f9779dc-bb02-61c2-80be-2e80e1a434c5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3f9779dc-bb02-61c2-80be-2e80e1a434c5 Address:3f9779dc-bb02-61c2-80be-2e80e1a434c5} {Suffrage:Voter ID:server-8734b349-06b6-45f6-9f8f-b52a9e96fbd8 Address:8734b349-06b6-45f6-9f8f-b52a9e96fbd8} {Suffrage:Voter ID:server-3acca375-1934-dbdd-b9cd-bef661b10ce0 Address:3acca375-1934-dbdd-b9cd-bef661b10ce0}]" +testLoggerAdapter verbose: server-8734b349-06b6-45f6-9f8f-b52a9e96fbd8: 2024-01-08T00:31:01.909Z [INFO] server-8734b349-06b6-45f6-9f8f-b52a9e96fbd8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3f9779dc-bb02-61c2-80be-2e80e1a434c5 Address:3f9779dc-bb02-61c2-80be-2e80e1a434c5} {Suffrage:Voter ID:server-8734b349-06b6-45f6-9f8f-b52a9e96fbd8 Address:8734b349-06b6-45f6-9f8f-b52a9e96fbd8} {Suffrage:Voter ID:server-3acca375-1934-dbdd-b9cd-bef661b10ce0 Address:3acca375-1934-dbdd-b9cd-bef661b10ce0}]" + testing.go:149: server-8734b349-06b6-45f6-9f8f-b52a9e96fbd8: 2024-01-08T00:31:01.909Z [INFO] server-8734b349-06b6-45f6-9f8f-b52a9e96fbd8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3f9779dc-bb02-61c2-80be-2e80e1a434c5 Address:3f9779dc-bb02-61c2-80be-2e80e1a434c5} {Suffrage:Voter ID:server-8734b349-06b6-45f6-9f8f-b52a9e96fbd8 Address:8734b349-06b6-45f6-9f8f-b52a9e96fbd8} {Suffrage:Voter ID:server-3acca375-1934-dbdd-b9cd-bef661b10ce0 Address:3acca375-1934-dbdd-b9cd-bef661b10ce0}]" +testLoggerAdapter verbose: server-3acca375-1934-dbdd-b9cd-bef661b10ce0: 2024-01-08T00:31:01.909Z [INFO] server-3acca375-1934-dbdd-b9cd-bef661b10ce0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3f9779dc-bb02-61c2-80be-2e80e1a434c5 Address:3f9779dc-bb02-61c2-80be-2e80e1a434c5} {Suffrage:Voter ID:server-8734b349-06b6-45f6-9f8f-b52a9e96fbd8 Address:8734b349-06b6-45f6-9f8f-b52a9e96fbd8} {Suffrage:Voter ID:server-3acca375-1934-dbdd-b9cd-bef661b10ce0 Address:3acca375-1934-dbdd-b9cd-bef661b10ce0}]" + testing.go:149: server-3acca375-1934-dbdd-b9cd-bef661b10ce0: 2024-01-08T00:31:01.909Z [INFO] server-3acca375-1934-dbdd-b9cd-bef661b10ce0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3f9779dc-bb02-61c2-80be-2e80e1a434c5 Address:3f9779dc-bb02-61c2-80be-2e80e1a434c5} {Suffrage:Voter ID:server-8734b349-06b6-45f6-9f8f-b52a9e96fbd8 Address:8734b349-06b6-45f6-9f8f-b52a9e96fbd8} {Suffrage:Voter ID:server-3acca375-1934-dbdd-b9cd-bef661b10ce0 Address:3acca375-1934-dbdd-b9cd-bef661b10ce0}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:01.909Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:01.909Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-3f9779dc-bb02-61c2-80be-2e80e1a434c5: 2024-01-08T00:31:01.909Z [INFO] server-3f9779dc-bb02-61c2-80be-2e80e1a434c5: entering follower state: follower="Node at 3f9779dc-bb02-61c2-80be-2e80e1a434c5 [Follower]" leader= + testing.go:149: server-3f9779dc-bb02-61c2-80be-2e80e1a434c5: 2024-01-08T00:31:01.909Z [INFO] server-3f9779dc-bb02-61c2-80be-2e80e1a434c5: entering follower state: follower="Node at 3f9779dc-bb02-61c2-80be-2e80e1a434c5 [Follower]" leader= +testLoggerAdapter verbose: server-8734b349-06b6-45f6-9f8f-b52a9e96fbd8: 2024-01-08T00:31:01.910Z [INFO] server-8734b349-06b6-45f6-9f8f-b52a9e96fbd8: entering follower state: follower="Node at 8734b349-06b6-45f6-9f8f-b52a9e96fbd8 [Follower]" leader= + testing.go:149: server-8734b349-06b6-45f6-9f8f-b52a9e96fbd8: 2024-01-08T00:31:01.910Z [INFO] server-8734b349-06b6-45f6-9f8f-b52a9e96fbd8: entering follower state: follower="Node at 8734b349-06b6-45f6-9f8f-b52a9e96fbd8 [Follower]" leader= +testLoggerAdapter verbose: server-3acca375-1934-dbdd-b9cd-bef661b10ce0: 2024-01-08T00:31:01.910Z [INFO] server-3acca375-1934-dbdd-b9cd-bef661b10ce0: entering follower state: follower="Node at 3acca375-1934-dbdd-b9cd-bef661b10ce0 [Follower]" leader= + testing.go:149: server-3acca375-1934-dbdd-b9cd-bef661b10ce0: 2024-01-08T00:31:01.910Z [INFO] server-3acca375-1934-dbdd-b9cd-bef661b10ce0: entering follower state: follower="Node at 3acca375-1934-dbdd-b9cd-bef661b10ce0 [Follower]" leader= +testLoggerAdapter verbose: server-3acca375-1934-dbdd-b9cd-bef661b10ce0: 2024-01-08T00:31:01.970Z [WARN] server-3acca375-1934-dbdd-b9cd-bef661b10ce0: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-3acca375-1934-dbdd-b9cd-bef661b10ce0: 2024-01-08T00:31:01.970Z [WARN] server-3acca375-1934-dbdd-b9cd-bef661b10ce0: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-3acca375-1934-dbdd-b9cd-bef661b10ce0: 2024-01-08T00:31:01.970Z [INFO] server-3acca375-1934-dbdd-b9cd-bef661b10ce0: entering candidate state: node="Node at 3acca375-1934-dbdd-b9cd-bef661b10ce0 [Candidate]" term=2 + testing.go:149: server-3acca375-1934-dbdd-b9cd-bef661b10ce0: 2024-01-08T00:31:01.970Z [INFO] server-3acca375-1934-dbdd-b9cd-bef661b10ce0: entering candidate state: node="Node at 3acca375-1934-dbdd-b9cd-bef661b10ce0 [Candidate]" term=2 +testLoggerAdapter verbose: server-3acca375-1934-dbdd-b9cd-bef661b10ce0: 2024-01-08T00:31:01.970Z [INFO] server-3acca375-1934-dbdd-b9cd-bef661b10ce0: election won: tally=2 + testing.go:149: server-3acca375-1934-dbdd-b9cd-bef661b10ce0: 2024-01-08T00:31:01.970Z [INFO] server-3acca375-1934-dbdd-b9cd-bef661b10ce0: election won: tally=2 +testLoggerAdapter verbose: server-3acca375-1934-dbdd-b9cd-bef661b10ce0: 2024-01-08T00:31:01.970Z [INFO] server-3acca375-1934-dbdd-b9cd-bef661b10ce0: entering leader state: leader="Node at 3acca375-1934-dbdd-b9cd-bef661b10ce0 [Leader]" + testing.go:149: server-3acca375-1934-dbdd-b9cd-bef661b10ce0: 2024-01-08T00:31:01.970Z [INFO] server-3acca375-1934-dbdd-b9cd-bef661b10ce0: entering leader state: leader="Node at 3acca375-1934-dbdd-b9cd-bef661b10ce0 [Leader]" +testLoggerAdapter verbose: server-3acca375-1934-dbdd-b9cd-bef661b10ce0: 2024-01-08T00:31:01.970Z [INFO] server-3acca375-1934-dbdd-b9cd-bef661b10ce0: added peer, starting replication: peer=server-3f9779dc-bb02-61c2-80be-2e80e1a434c5 + testing.go:149: server-3acca375-1934-dbdd-b9cd-bef661b10ce0: 2024-01-08T00:31:01.970Z [INFO] server-3acca375-1934-dbdd-b9cd-bef661b10ce0: added peer, starting replication: peer=server-3f9779dc-bb02-61c2-80be-2e80e1a434c5 +testLoggerAdapter verbose: server-3acca375-1934-dbdd-b9cd-bef661b10ce0: 2024-01-08T00:31:01.971Z [INFO] server-3acca375-1934-dbdd-b9cd-bef661b10ce0: added peer, starting replication: peer=server-8734b349-06b6-45f6-9f8f-b52a9e96fbd8 + testing.go:149: server-3acca375-1934-dbdd-b9cd-bef661b10ce0: 2024-01-08T00:31:01.971Z [INFO] server-3acca375-1934-dbdd-b9cd-bef661b10ce0: added peer, starting replication: peer=server-8734b349-06b6-45f6-9f8f-b52a9e96fbd8 +testLoggerAdapter verbose: server-3acca375-1934-dbdd-b9cd-bef661b10ce0: 2024-01-08T00:31:01.971Z [INFO] server-3acca375-1934-dbdd-b9cd-bef661b10ce0: pipelining replication: peer="{Voter server-8734b349-06b6-45f6-9f8f-b52a9e96fbd8 8734b349-06b6-45f6-9f8f-b52a9e96fbd8}" + testing.go:149: server-3acca375-1934-dbdd-b9cd-bef661b10ce0: 2024-01-08T00:31:01.971Z [INFO] server-3acca375-1934-dbdd-b9cd-bef661b10ce0: pipelining replication: peer="{Voter server-8734b349-06b6-45f6-9f8f-b52a9e96fbd8 8734b349-06b6-45f6-9f8f-b52a9e96fbd8}" +testLoggerAdapter verbose: server-3acca375-1934-dbdd-b9cd-bef661b10ce0: 2024-01-08T00:31:01.971Z [INFO] server-3acca375-1934-dbdd-b9cd-bef661b10ce0: pipelining replication: peer="{Voter server-3f9779dc-bb02-61c2-80be-2e80e1a434c5 3f9779dc-bb02-61c2-80be-2e80e1a434c5}" + testing.go:149: server-3acca375-1934-dbdd-b9cd-bef661b10ce0: 2024-01-08T00:31:01.971Z [INFO] server-3acca375-1934-dbdd-b9cd-bef661b10ce0: pipelining replication: peer="{Voter server-3f9779dc-bb02-61c2-80be-2e80e1a434c5 3f9779dc-bb02-61c2-80be-2e80e1a434c5}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:02.076Z [INFO] stable state for Leader reached at 2024-01-08 00:31:01.971339231 +0000 UTC m=+8.018228321 (1 nodes), 61.361401ms from start of poll, 61.898196ms from cluster start. Timeout at 2024-01-08 00:31:02.076610168 +0000 UTC m=+8.123499302, 105.270981ms after stability + testing.go:149: cluster: 2024-01-08T00:31:02.076Z [INFO] stable state for Leader reached at 2024-01-08 00:31:01.971339231 +0000 UTC m=+8.018228321 (1 nodes), 61.361401ms from start of poll, 61.898196ms from cluster start. Timeout at 2024-01-08 00:31:02.076610168 +0000 UTC m=+8.123499302, 105.270981ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:02.083Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T00:31:02.083Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:02.188Z [INFO] stable state for Follower reached at 2024-01-08 00:31:02.083347634 +0000 UTC m=+8.130236728 (2 nodes), 2.457µs from start of poll, 173.906603ms from cluster start. Timeout at 2024-01-08 00:31:02.188657989 +0000 UTC m=+8.235547099, 105.310371ms after stability + testing.go:149: cluster: 2024-01-08T00:31:02.188Z [INFO] stable state for Follower reached at 2024-01-08 00:31:02.083347634 +0000 UTC m=+8.130236728 (2 nodes), 2.457µs from start of poll, 173.906603ms from cluster start. Timeout at 2024-01-08 00:31:02.188657989 +0000 UTC m=+8.235547099, 105.310371ms after stability +testLoggerAdapter verbose: server-3acca375-1934-dbdd-b9cd-bef661b10ce0: 2024-01-08T00:31:02.188Z [INFO] server-3acca375-1934-dbdd-b9cd-bef661b10ce0: updating configuration: command=RemoveServer server-id=server-3f9779dc-bb02-61c2-80be-2e80e1a434c5 server-addr= servers="[{Suffrage:Voter ID:server-8734b349-06b6-45f6-9f8f-b52a9e96fbd8 Address:8734b349-06b6-45f6-9f8f-b52a9e96fbd8} {Suffrage:Voter ID:server-3acca375-1934-dbdd-b9cd-bef661b10ce0 Address:3acca375-1934-dbdd-b9cd-bef661b10ce0}]" + testing.go:149: server-3acca375-1934-dbdd-b9cd-bef661b10ce0: 2024-01-08T00:31:02.188Z [INFO] server-3acca375-1934-dbdd-b9cd-bef661b10ce0: updating configuration: command=RemoveServer server-id=server-3f9779dc-bb02-61c2-80be-2e80e1a434c5 server-addr= servers="[{Suffrage:Voter ID:server-8734b349-06b6-45f6-9f8f-b52a9e96fbd8 Address:8734b349-06b6-45f6-9f8f-b52a9e96fbd8} {Suffrage:Voter ID:server-3acca375-1934-dbdd-b9cd-bef661b10ce0 Address:3acca375-1934-dbdd-b9cd-bef661b10ce0}]" +testLoggerAdapter verbose: server-3acca375-1934-dbdd-b9cd-bef661b10ce0: 2024-01-08T00:31:02.188Z [INFO] server-3acca375-1934-dbdd-b9cd-bef661b10ce0: removed peer, stopping replication: peer=server-3f9779dc-bb02-61c2-80be-2e80e1a434c5 last-index=3 + testing.go:149: server-3acca375-1934-dbdd-b9cd-bef661b10ce0: 2024-01-08T00:31:02.188Z [INFO] server-3acca375-1934-dbdd-b9cd-bef661b10ce0: removed peer, stopping replication: peer=server-3f9779dc-bb02-61c2-80be-2e80e1a434c5 last-index=3 +testLoggerAdapter verbose: server-3acca375-1934-dbdd-b9cd-bef661b10ce0: 2024-01-08T00:31:02.188Z [INFO] server-3acca375-1934-dbdd-b9cd-bef661b10ce0: aborting pipeline replication: peer="{Voter server-3f9779dc-bb02-61c2-80be-2e80e1a434c5 3f9779dc-bb02-61c2-80be-2e80e1a434c5}" + testing.go:149: server-3acca375-1934-dbdd-b9cd-bef661b10ce0: 2024-01-08T00:31:02.188Z [INFO] server-3acca375-1934-dbdd-b9cd-bef661b10ce0: aborting pipeline replication: peer="{Voter server-3f9779dc-bb02-61c2-80be-2e80e1a434c5 3f9779dc-bb02-61c2-80be-2e80e1a434c5}" +testLoggerAdapter verbose: server-3f9779dc-bb02-61c2-80be-2e80e1a434c5: 2024-01-08T00:31:02.291Z [WARN] server-3f9779dc-bb02-61c2-80be-2e80e1a434c5: heartbeat timeout reached, starting election: last-leader=3acca375-1934-dbdd-b9cd-bef661b10ce0 + testing.go:149: server-3f9779dc-bb02-61c2-80be-2e80e1a434c5: 2024-01-08T00:31:02.291Z [WARN] server-3f9779dc-bb02-61c2-80be-2e80e1a434c5: heartbeat timeout reached, starting election: last-leader=3acca375-1934-dbdd-b9cd-bef661b10ce0 +testLoggerAdapter verbose: server-3f9779dc-bb02-61c2-80be-2e80e1a434c5: 2024-01-08T00:31:02.291Z [INFO] server-3f9779dc-bb02-61c2-80be-2e80e1a434c5: entering candidate state: node="Node at 3f9779dc-bb02-61c2-80be-2e80e1a434c5 [Candidate]" term=3 + testing.go:149: server-3f9779dc-bb02-61c2-80be-2e80e1a434c5: 2024-01-08T00:31:02.291Z [INFO] server-3f9779dc-bb02-61c2-80be-2e80e1a434c5: entering candidate state: node="Node at 3f9779dc-bb02-61c2-80be-2e80e1a434c5 [Candidate]" term=3 +testLoggerAdapter verbose: server-3acca375-1934-dbdd-b9cd-bef661b10ce0: 2024-01-08T00:31:02.291Z [WARN] server-3acca375-1934-dbdd-b9cd-bef661b10ce0: rejecting vote request since we have a leader: from=3f9779dc-bb02-61c2-80be-2e80e1a434c5 leader=3acca375-1934-dbdd-b9cd-bef661b10ce0 + testing.go:149: server-3acca375-1934-dbdd-b9cd-bef661b10ce0: 2024-01-08T00:31:02.291Z [WARN] server-3acca375-1934-dbdd-b9cd-bef661b10ce0: rejecting vote request since we have a leader: from=3f9779dc-bb02-61c2-80be-2e80e1a434c5 leader=3acca375-1934-dbdd-b9cd-bef661b10ce0 +testLoggerAdapter verbose: server-8734b349-06b6-45f6-9f8f-b52a9e96fbd8: 2024-01-08T00:31:02.292Z [WARN] server-8734b349-06b6-45f6-9f8f-b52a9e96fbd8: rejecting vote request since we have a leader: from=3f9779dc-bb02-61c2-80be-2e80e1a434c5 leader=3acca375-1934-dbdd-b9cd-bef661b10ce0 + testing.go:149: server-8734b349-06b6-45f6-9f8f-b52a9e96fbd8: 2024-01-08T00:31:02.292Z [WARN] server-8734b349-06b6-45f6-9f8f-b52a9e96fbd8: rejecting vote request since we have a leader: from=3f9779dc-bb02-61c2-80be-2e80e1a434c5 leader=3acca375-1934-dbdd-b9cd-bef661b10ce0 +testLoggerAdapter verbose: server-3acca375-1934-dbdd-b9cd-bef661b10ce0: 2024-01-08T00:31:02.294Z [INFO] server-3acca375-1934-dbdd-b9cd-bef661b10ce0: aborting pipeline replication: peer="{Voter server-8734b349-06b6-45f6-9f8f-b52a9e96fbd8 8734b349-06b6-45f6-9f8f-b52a9e96fbd8}" + testing.go:149: server-3acca375-1934-dbdd-b9cd-bef661b10ce0: 2024-01-08T00:31:02.294Z [INFO] server-3acca375-1934-dbdd-b9cd-bef661b10ce0: aborting pipeline replication: peer="{Voter server-8734b349-06b6-45f6-9f8f-b52a9e96fbd8 8734b349-06b6-45f6-9f8f-b52a9e96fbd8}" +--- PASS: TestRaft_RemoveFollower (0.39s) === RUN TestRaft_RemoveLeader -testLoggerAdapter verbose: server-ba268597-bfc2-1596-3cdc-c543ef2f8346: 2025-02-09T06:52:27.293Z [INFO] server-ba268597-bfc2-1596-3cdc-c543ef2f8346: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ba268597-bfc2-1596-3cdc-c543ef2f8346 Address:ba268597-bfc2-1596-3cdc-c543ef2f8346} {Suffrage:Voter ID:server-4adad467-cf70-40ba-a03f-54e74fe7c051 Address:4adad467-cf70-40ba-a03f-54e74fe7c051} {Suffrage:Voter ID:server-8555462e-559d-a74b-2a63-5f27b8de9a78 Address:8555462e-559d-a74b-2a63-5f27b8de9a78}]" - testing.go:149: server-ba268597-bfc2-1596-3cdc-c543ef2f8346: 2025-02-09T06:52:27.293Z [INFO] server-ba268597-bfc2-1596-3cdc-c543ef2f8346: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ba268597-bfc2-1596-3cdc-c543ef2f8346 Address:ba268597-bfc2-1596-3cdc-c543ef2f8346} {Suffrage:Voter ID:server-4adad467-cf70-40ba-a03f-54e74fe7c051 Address:4adad467-cf70-40ba-a03f-54e74fe7c051} {Suffrage:Voter ID:server-8555462e-559d-a74b-2a63-5f27b8de9a78 Address:8555462e-559d-a74b-2a63-5f27b8de9a78}]" -testLoggerAdapter verbose: server-ba268597-bfc2-1596-3cdc-c543ef2f8346: 2025-02-09T06:52:27.293Z [INFO] server-ba268597-bfc2-1596-3cdc-c543ef2f8346: entering follower state: follower="Node at ba268597-bfc2-1596-3cdc-c543ef2f8346 [Follower]" leader= - testing.go:149: server-ba268597-bfc2-1596-3cdc-c543ef2f8346: 2025-02-09T06:52:27.293Z [INFO] server-ba268597-bfc2-1596-3cdc-c543ef2f8346: entering follower state: follower="Node at ba268597-bfc2-1596-3cdc-c543ef2f8346 [Follower]" leader= -testLoggerAdapter verbose: server-4adad467-cf70-40ba-a03f-54e74fe7c051: 2025-02-09T06:52:27.293Z [INFO] server-4adad467-cf70-40ba-a03f-54e74fe7c051: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ba268597-bfc2-1596-3cdc-c543ef2f8346 Address:ba268597-bfc2-1596-3cdc-c543ef2f8346} {Suffrage:Voter ID:server-4adad467-cf70-40ba-a03f-54e74fe7c051 Address:4adad467-cf70-40ba-a03f-54e74fe7c051} {Suffrage:Voter ID:server-8555462e-559d-a74b-2a63-5f27b8de9a78 Address:8555462e-559d-a74b-2a63-5f27b8de9a78}]" - testing.go:149: server-4adad467-cf70-40ba-a03f-54e74fe7c051: 2025-02-09T06:52:27.293Z [INFO] server-4adad467-cf70-40ba-a03f-54e74fe7c051: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ba268597-bfc2-1596-3cdc-c543ef2f8346 Address:ba268597-bfc2-1596-3cdc-c543ef2f8346} {Suffrage:Voter ID:server-4adad467-cf70-40ba-a03f-54e74fe7c051 Address:4adad467-cf70-40ba-a03f-54e74fe7c051} {Suffrage:Voter ID:server-8555462e-559d-a74b-2a63-5f27b8de9a78 Address:8555462e-559d-a74b-2a63-5f27b8de9a78}]" -testLoggerAdapter verbose: server-4adad467-cf70-40ba-a03f-54e74fe7c051: 2025-02-09T06:52:27.293Z [INFO] server-4adad467-cf70-40ba-a03f-54e74fe7c051: entering follower state: follower="Node at 4adad467-cf70-40ba-a03f-54e74fe7c051 [Follower]" leader= - testing.go:149: server-4adad467-cf70-40ba-a03f-54e74fe7c051: 2025-02-09T06:52:27.293Z [INFO] server-4adad467-cf70-40ba-a03f-54e74fe7c051: entering follower state: follower="Node at 4adad467-cf70-40ba-a03f-54e74fe7c051 [Follower]" leader= -testLoggerAdapter verbose: server-8555462e-559d-a74b-2a63-5f27b8de9a78: 2025-02-09T06:52:27.293Z [INFO] server-8555462e-559d-a74b-2a63-5f27b8de9a78: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ba268597-bfc2-1596-3cdc-c543ef2f8346 Address:ba268597-bfc2-1596-3cdc-c543ef2f8346} {Suffrage:Voter ID:server-4adad467-cf70-40ba-a03f-54e74fe7c051 Address:4adad467-cf70-40ba-a03f-54e74fe7c051} {Suffrage:Voter ID:server-8555462e-559d-a74b-2a63-5f27b8de9a78 Address:8555462e-559d-a74b-2a63-5f27b8de9a78}]" - testing.go:149: server-8555462e-559d-a74b-2a63-5f27b8de9a78: 2025-02-09T06:52:27.293Z [INFO] server-8555462e-559d-a74b-2a63-5f27b8de9a78: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ba268597-bfc2-1596-3cdc-c543ef2f8346 Address:ba268597-bfc2-1596-3cdc-c543ef2f8346} {Suffrage:Voter ID:server-4adad467-cf70-40ba-a03f-54e74fe7c051 Address:4adad467-cf70-40ba-a03f-54e74fe7c051} {Suffrage:Voter ID:server-8555462e-559d-a74b-2a63-5f27b8de9a78 Address:8555462e-559d-a74b-2a63-5f27b8de9a78}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:27.293Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-8555462e-559d-a74b-2a63-5f27b8de9a78: 2025-02-09T06:52:27.293Z [INFO] server-8555462e-559d-a74b-2a63-5f27b8de9a78: entering follower state: follower="Node at 8555462e-559d-a74b-2a63-5f27b8de9a78 [Follower]" leader= - testing.go:149: cluster: 2025-02-09T06:52:27.293Z [INFO] starting stability test: raft-state=Leader - testing.go:149: server-8555462e-559d-a74b-2a63-5f27b8de9a78: 2025-02-09T06:52:27.293Z [INFO] server-8555462e-559d-a74b-2a63-5f27b8de9a78: entering follower state: follower="Node at 8555462e-559d-a74b-2a63-5f27b8de9a78 [Follower]" leader= -testLoggerAdapter verbose: server-4adad467-cf70-40ba-a03f-54e74fe7c051: 2025-02-09T06:52:27.350Z [WARN] server-4adad467-cf70-40ba-a03f-54e74fe7c051: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-4adad467-cf70-40ba-a03f-54e74fe7c051: 2025-02-09T06:52:27.350Z [WARN] server-4adad467-cf70-40ba-a03f-54e74fe7c051: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-4adad467-cf70-40ba-a03f-54e74fe7c051: 2025-02-09T06:52:27.350Z [INFO] server-4adad467-cf70-40ba-a03f-54e74fe7c051: entering candidate state: node="Node at 4adad467-cf70-40ba-a03f-54e74fe7c051 [Candidate]" term=2 - testing.go:149: server-4adad467-cf70-40ba-a03f-54e74fe7c051: 2025-02-09T06:52:27.350Z [INFO] server-4adad467-cf70-40ba-a03f-54e74fe7c051: entering candidate state: node="Node at 4adad467-cf70-40ba-a03f-54e74fe7c051 [Candidate]" term=2 -testLoggerAdapter verbose: server-4adad467-cf70-40ba-a03f-54e74fe7c051: 2025-02-09T06:52:27.350Z [INFO] server-4adad467-cf70-40ba-a03f-54e74fe7c051: election won: tally=2 - testing.go:149: server-4adad467-cf70-40ba-a03f-54e74fe7c051: 2025-02-09T06:52:27.350Z [INFO] server-4adad467-cf70-40ba-a03f-54e74fe7c051: election won: tally=2 -testLoggerAdapter verbose: server-4adad467-cf70-40ba-a03f-54e74fe7c051: 2025-02-09T06:52:27.350Z [INFO] server-4adad467-cf70-40ba-a03f-54e74fe7c051: entering leader state: leader="Node at 4adad467-cf70-40ba-a03f-54e74fe7c051 [Leader]" - testing.go:149: server-4adad467-cf70-40ba-a03f-54e74fe7c051: 2025-02-09T06:52:27.350Z [INFO] server-4adad467-cf70-40ba-a03f-54e74fe7c051: entering leader state: leader="Node at 4adad467-cf70-40ba-a03f-54e74fe7c051 [Leader]" -testLoggerAdapter verbose: server-4adad467-cf70-40ba-a03f-54e74fe7c051: 2025-02-09T06:52:27.350Z [INFO] server-4adad467-cf70-40ba-a03f-54e74fe7c051: added peer, starting replication: peer=server-ba268597-bfc2-1596-3cdc-c543ef2f8346 - testing.go:149: server-4adad467-cf70-40ba-a03f-54e74fe7c051: 2025-02-09T06:52:27.350Z [INFO] server-4adad467-cf70-40ba-a03f-54e74fe7c051: added peer, starting replication: peer=server-ba268597-bfc2-1596-3cdc-c543ef2f8346 -testLoggerAdapter verbose: server-4adad467-cf70-40ba-a03f-54e74fe7c051: 2025-02-09T06:52:27.350Z [INFO] server-4adad467-cf70-40ba-a03f-54e74fe7c051: added peer, starting replication: peer=server-8555462e-559d-a74b-2a63-5f27b8de9a78 - testing.go:149: server-4adad467-cf70-40ba-a03f-54e74fe7c051: 2025-02-09T06:52:27.350Z [INFO] server-4adad467-cf70-40ba-a03f-54e74fe7c051: added peer, starting replication: peer=server-8555462e-559d-a74b-2a63-5f27b8de9a78 -testLoggerAdapter verbose: server-4adad467-cf70-40ba-a03f-54e74fe7c051: 2025-02-09T06:52:27.351Z [INFO] server-4adad467-cf70-40ba-a03f-54e74fe7c051: pipelining replication: peer="{Voter server-ba268597-bfc2-1596-3cdc-c543ef2f8346 ba268597-bfc2-1596-3cdc-c543ef2f8346}" - testing.go:149: server-4adad467-cf70-40ba-a03f-54e74fe7c051: 2025-02-09T06:52:27.351Z [INFO] server-4adad467-cf70-40ba-a03f-54e74fe7c051: pipelining replication: peer="{Voter server-ba268597-bfc2-1596-3cdc-c543ef2f8346 ba268597-bfc2-1596-3cdc-c543ef2f8346}" -testLoggerAdapter verbose: server-4adad467-cf70-40ba-a03f-54e74fe7c051: 2025-02-09T06:52:27.351Z [INFO] server-4adad467-cf70-40ba-a03f-54e74fe7c051: pipelining replication: peer="{Voter server-8555462e-559d-a74b-2a63-5f27b8de9a78 8555462e-559d-a74b-2a63-5f27b8de9a78}" - testing.go:149: server-4adad467-cf70-40ba-a03f-54e74fe7c051: 2025-02-09T06:52:27.351Z [INFO] server-4adad467-cf70-40ba-a03f-54e74fe7c051: pipelining replication: peer="{Voter server-8555462e-559d-a74b-2a63-5f27b8de9a78 8555462e-559d-a74b-2a63-5f27b8de9a78}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:27.456Z [INFO] stable state for Leader reached at 2025-02-09 06:52:27.350907673 +0000 UTC m=+8.329716437 (1 nodes), 57.599477ms from start of poll, 57.952249ms from cluster start. Timeout at 2025-02-09 06:52:27.456580153 +0000 UTC m=+8.435388921, 105.672484ms after stability - testing.go:149: cluster: 2025-02-09T06:52:27.456Z [INFO] stable state for Leader reached at 2025-02-09 06:52:27.350907673 +0000 UTC m=+8.329716437 (1 nodes), 57.599477ms from start of poll, 57.952249ms from cluster start. Timeout at 2025-02-09 06:52:27.456580153 +0000 UTC m=+8.435388921, 105.672484ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:27.461Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T06:52:27.461Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:27.567Z [INFO] stable state for Follower reached at 2025-02-09 06:52:27.461986758 +0000 UTC m=+8.440795523 (2 nodes), 650ns from start of poll, 169.031335ms from cluster start. Timeout at 2025-02-09 06:52:27.567482722 +0000 UTC m=+8.546291492, 105.495969ms after stability - testing.go:149: cluster: 2025-02-09T06:52:27.567Z [INFO] stable state for Follower reached at 2025-02-09 06:52:27.461986758 +0000 UTC m=+8.440795523 (2 nodes), 650ns from start of poll, 169.031335ms from cluster start. Timeout at 2025-02-09 06:52:27.567482722 +0000 UTC m=+8.546291492, 105.495969ms after stability -testLoggerAdapter verbose: server-4adad467-cf70-40ba-a03f-54e74fe7c051: 2025-02-09T06:52:27.567Z [INFO] server-4adad467-cf70-40ba-a03f-54e74fe7c051: updating configuration: command=RemoveServer server-id=server-4adad467-cf70-40ba-a03f-54e74fe7c051 server-addr= servers="[{Suffrage:Voter ID:server-ba268597-bfc2-1596-3cdc-c543ef2f8346 Address:ba268597-bfc2-1596-3cdc-c543ef2f8346} {Suffrage:Voter ID:server-8555462e-559d-a74b-2a63-5f27b8de9a78 Address:8555462e-559d-a74b-2a63-5f27b8de9a78}]" - testing.go:149: server-4adad467-cf70-40ba-a03f-54e74fe7c051: 2025-02-09T06:52:27.567Z [INFO] server-4adad467-cf70-40ba-a03f-54e74fe7c051: updating configuration: command=RemoveServer server-id=server-4adad467-cf70-40ba-a03f-54e74fe7c051 server-addr= servers="[{Suffrage:Voter ID:server-ba268597-bfc2-1596-3cdc-c543ef2f8346 Address:ba268597-bfc2-1596-3cdc-c543ef2f8346} {Suffrage:Voter ID:server-8555462e-559d-a74b-2a63-5f27b8de9a78 Address:8555462e-559d-a74b-2a63-5f27b8de9a78}]" -testLoggerAdapter verbose: server-4adad467-cf70-40ba-a03f-54e74fe7c051: 2025-02-09T06:52:27.567Z [INFO] server-4adad467-cf70-40ba-a03f-54e74fe7c051: removed ourself, shutting down - testing.go:149: server-4adad467-cf70-40ba-a03f-54e74fe7c051: 2025-02-09T06:52:27.567Z [INFO] server-4adad467-cf70-40ba-a03f-54e74fe7c051: removed ourself, shutting down -testLoggerAdapter verbose: server-4adad467-cf70-40ba-a03f-54e74fe7c051: 2025-02-09T06:52:27.567Z [INFO] server-4adad467-cf70-40ba-a03f-54e74fe7c051: aborting pipeline replication: peer="{Voter server-8555462e-559d-a74b-2a63-5f27b8de9a78 8555462e-559d-a74b-2a63-5f27b8de9a78}" - testing.go:149: server-4adad467-cf70-40ba-a03f-54e74fe7c051: 2025-02-09T06:52:27.567Z [INFO] server-4adad467-cf70-40ba-a03f-54e74fe7c051: aborting pipeline replication: peer="{Voter server-8555462e-559d-a74b-2a63-5f27b8de9a78 8555462e-559d-a74b-2a63-5f27b8de9a78}" -testLoggerAdapter verbose: server-4adad467-cf70-40ba-a03f-54e74fe7c051: 2025-02-09T06:52:27.567Z [INFO] server-4adad467-cf70-40ba-a03f-54e74fe7c051: aborting pipeline replication: peer="{Voter server-ba268597-bfc2-1596-3cdc-c543ef2f8346 ba268597-bfc2-1596-3cdc-c543ef2f8346}" - testing.go:149: server-4adad467-cf70-40ba-a03f-54e74fe7c051: 2025-02-09T06:52:27.567Z [INFO] server-4adad467-cf70-40ba-a03f-54e74fe7c051: aborting pipeline replication: peer="{Voter server-ba268597-bfc2-1596-3cdc-c543ef2f8346 ba268597-bfc2-1596-3cdc-c543ef2f8346}" -testLoggerAdapter verbose: server-8555462e-559d-a74b-2a63-5f27b8de9a78: 2025-02-09T06:52:27.641Z [WARN] server-8555462e-559d-a74b-2a63-5f27b8de9a78: heartbeat timeout reached, starting election: last-leader=4adad467-cf70-40ba-a03f-54e74fe7c051 - testing.go:149: server-8555462e-559d-a74b-2a63-5f27b8de9a78: 2025-02-09T06:52:27.641Z [WARN] server-8555462e-559d-a74b-2a63-5f27b8de9a78: heartbeat timeout reached, starting election: last-leader=4adad467-cf70-40ba-a03f-54e74fe7c051 -testLoggerAdapter verbose: server-8555462e-559d-a74b-2a63-5f27b8de9a78: 2025-02-09T06:52:27.641Z [INFO] server-8555462e-559d-a74b-2a63-5f27b8de9a78: entering candidate state: node="Node at 8555462e-559d-a74b-2a63-5f27b8de9a78 [Candidate]" term=3 - testing.go:149: server-8555462e-559d-a74b-2a63-5f27b8de9a78: 2025-02-09T06:52:27.641Z [INFO] server-8555462e-559d-a74b-2a63-5f27b8de9a78: entering candidate state: node="Node at 8555462e-559d-a74b-2a63-5f27b8de9a78 [Candidate]" term=3 -testLoggerAdapter verbose: server-ba268597-bfc2-1596-3cdc-c543ef2f8346: 2025-02-09T06:52:27.641Z [WARN] server-ba268597-bfc2-1596-3cdc-c543ef2f8346: rejecting vote request since we have a leader: from=8555462e-559d-a74b-2a63-5f27b8de9a78 leader=4adad467-cf70-40ba-a03f-54e74fe7c051 - testing.go:149: server-ba268597-bfc2-1596-3cdc-c543ef2f8346: 2025-02-09T06:52:27.641Z [WARN] server-ba268597-bfc2-1596-3cdc-c543ef2f8346: rejecting vote request since we have a leader: from=8555462e-559d-a74b-2a63-5f27b8de9a78 leader=4adad467-cf70-40ba-a03f-54e74fe7c051 -testLoggerAdapter verbose: server-ba268597-bfc2-1596-3cdc-c543ef2f8346: 2025-02-09T06:52:27.674Z [WARN] server-ba268597-bfc2-1596-3cdc-c543ef2f8346: heartbeat timeout reached, starting election: last-leader=4adad467-cf70-40ba-a03f-54e74fe7c051 - testing.go:149: server-ba268597-bfc2-1596-3cdc-c543ef2f8346: 2025-02-09T06:52:27.674Z [WARN] server-ba268597-bfc2-1596-3cdc-c543ef2f8346: heartbeat timeout reached, starting election: last-leader=4adad467-cf70-40ba-a03f-54e74fe7c051 -testLoggerAdapter verbose: server-ba268597-bfc2-1596-3cdc-c543ef2f8346: 2025-02-09T06:52:27.674Z [INFO] server-ba268597-bfc2-1596-3cdc-c543ef2f8346: entering candidate state: node="Node at ba268597-bfc2-1596-3cdc-c543ef2f8346 [Candidate]" term=3 - testing.go:149: server-ba268597-bfc2-1596-3cdc-c543ef2f8346: 2025-02-09T06:52:27.674Z [INFO] server-ba268597-bfc2-1596-3cdc-c543ef2f8346: entering candidate state: node="Node at ba268597-bfc2-1596-3cdc-c543ef2f8346 [Candidate]" term=3 -testLoggerAdapter verbose: server-8555462e-559d-a74b-2a63-5f27b8de9a78: 2025-02-09T06:52:27.674Z [INFO] server-8555462e-559d-a74b-2a63-5f27b8de9a78: duplicate requestVote for same term: term=3 - testing.go:149: server-8555462e-559d-a74b-2a63-5f27b8de9a78: 2025-02-09T06:52:27.674Z [INFO] server-8555462e-559d-a74b-2a63-5f27b8de9a78: duplicate requestVote for same term: term=3 -testLoggerAdapter verbose: server-8555462e-559d-a74b-2a63-5f27b8de9a78: 2025-02-09T06:52:27.731Z [WARN] server-8555462e-559d-a74b-2a63-5f27b8de9a78: Election timeout reached, restarting election - testing.go:149: server-8555462e-559d-a74b-2a63-5f27b8de9a78: 2025-02-09T06:52:27.731Z [WARN] server-8555462e-559d-a74b-2a63-5f27b8de9a78: Election timeout reached, restarting election -testLoggerAdapter verbose: server-8555462e-559d-a74b-2a63-5f27b8de9a78: 2025-02-09T06:52:27.731Z [INFO] server-8555462e-559d-a74b-2a63-5f27b8de9a78: entering candidate state: node="Node at 8555462e-559d-a74b-2a63-5f27b8de9a78 [Candidate]" term=4 - testing.go:149: server-8555462e-559d-a74b-2a63-5f27b8de9a78: 2025-02-09T06:52:27.731Z [INFO] server-8555462e-559d-a74b-2a63-5f27b8de9a78: entering candidate state: node="Node at 8555462e-559d-a74b-2a63-5f27b8de9a78 [Candidate]" term=4 -testLoggerAdapter verbose: server-ba268597-bfc2-1596-3cdc-c543ef2f8346: 2025-02-09T06:52:27.731Z [INFO] server-ba268597-bfc2-1596-3cdc-c543ef2f8346: entering follower state: follower="Node at ba268597-bfc2-1596-3cdc-c543ef2f8346 [Follower]" leader= - testing.go:149: server-ba268597-bfc2-1596-3cdc-c543ef2f8346: 2025-02-09T06:52:27.731Z [INFO] server-ba268597-bfc2-1596-3cdc-c543ef2f8346: entering follower state: follower="Node at ba268597-bfc2-1596-3cdc-c543ef2f8346 [Follower]" leader= -testLoggerAdapter verbose: server-8555462e-559d-a74b-2a63-5f27b8de9a78: 2025-02-09T06:52:27.731Z [INFO] server-8555462e-559d-a74b-2a63-5f27b8de9a78: election won: tally=2 - testing.go:149: server-8555462e-559d-a74b-2a63-5f27b8de9a78: 2025-02-09T06:52:27.731Z [INFO] server-8555462e-559d-a74b-2a63-5f27b8de9a78: election won: tally=2 -testLoggerAdapter verbose: server-8555462e-559d-a74b-2a63-5f27b8de9a78: 2025-02-09T06:52:27.731Z [INFO] server-8555462e-559d-a74b-2a63-5f27b8de9a78: entering leader state: leader="Node at 8555462e-559d-a74b-2a63-5f27b8de9a78 [Leader]" - testing.go:149: server-8555462e-559d-a74b-2a63-5f27b8de9a78: 2025-02-09T06:52:27.731Z [INFO] server-8555462e-559d-a74b-2a63-5f27b8de9a78: entering leader state: leader="Node at 8555462e-559d-a74b-2a63-5f27b8de9a78 [Leader]" -testLoggerAdapter verbose: server-8555462e-559d-a74b-2a63-5f27b8de9a78: 2025-02-09T06:52:27.731Z [INFO] server-8555462e-559d-a74b-2a63-5f27b8de9a78: added peer, starting replication: peer=server-ba268597-bfc2-1596-3cdc-c543ef2f8346 - testing.go:149: server-8555462e-559d-a74b-2a63-5f27b8de9a78: 2025-02-09T06:52:27.731Z [INFO] server-8555462e-559d-a74b-2a63-5f27b8de9a78: added peer, starting replication: peer=server-ba268597-bfc2-1596-3cdc-c543ef2f8346 -testLoggerAdapter verbose: server-8555462e-559d-a74b-2a63-5f27b8de9a78: 2025-02-09T06:52:27.731Z [INFO] server-8555462e-559d-a74b-2a63-5f27b8de9a78: pipelining replication: peer="{Voter server-ba268597-bfc2-1596-3cdc-c543ef2f8346 ba268597-bfc2-1596-3cdc-c543ef2f8346}" - testing.go:149: server-8555462e-559d-a74b-2a63-5f27b8de9a78: 2025-02-09T06:52:27.731Z [INFO] server-8555462e-559d-a74b-2a63-5f27b8de9a78: pipelining replication: peer="{Voter server-ba268597-bfc2-1596-3cdc-c543ef2f8346 ba268597-bfc2-1596-3cdc-c543ef2f8346}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:27.778Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:27.778Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:27.884Z [INFO] stable state for Leader reached at 2025-02-09 06:52:27.778862307 +0000 UTC m=+8.757671070 (1 nodes), 21.925µs from start of poll, 485.906882ms from cluster start. Timeout at 2025-02-09 06:52:27.884204181 +0000 UTC m=+8.863012950, 105.34188ms after stability - testing.go:149: cluster: 2025-02-09T06:52:27.884Z [INFO] stable state for Leader reached at 2025-02-09 06:52:27.778862307 +0000 UTC m=+8.757671070 (1 nodes), 21.925µs from start of poll, 485.906882ms from cluster start. Timeout at 2025-02-09 06:52:27.884204181 +0000 UTC m=+8.863012950, 105.34188ms after stability -testLoggerAdapter verbose: server-8555462e-559d-a74b-2a63-5f27b8de9a78: 2025-02-09T06:52:27.884Z [INFO] server-8555462e-559d-a74b-2a63-5f27b8de9a78: aborting pipeline replication: peer="{Voter server-ba268597-bfc2-1596-3cdc-c543ef2f8346 ba268597-bfc2-1596-3cdc-c543ef2f8346}" - testing.go:149: server-8555462e-559d-a74b-2a63-5f27b8de9a78: 2025-02-09T06:52:27.884Z [INFO] server-8555462e-559d-a74b-2a63-5f27b8de9a78: aborting pipeline replication: peer="{Voter server-ba268597-bfc2-1596-3cdc-c543ef2f8346 ba268597-bfc2-1596-3cdc-c543ef2f8346}" -testLoggerAdapter verbose: server-8555462e-559d-a74b-2a63-5f27b8de9a78: 2025-02-09T06:52:27.934Z [ERROR] server-8555462e-559d-a74b-2a63-5f27b8de9a78: failed to heartbeat to: peer=ba268597-bfc2-1596-3cdc-c543ef2f8346 error="command timed out" - testing.go:149: server-8555462e-559d-a74b-2a63-5f27b8de9a78: 2025-02-09T06:52:27.934Z [ERROR] server-8555462e-559d-a74b-2a63-5f27b8de9a78: failed to heartbeat to: peer=ba268597-bfc2-1596-3cdc-c543ef2f8346 error="command timed out" ---- PASS: TestRaft_RemoveLeader (0.64s) +testLoggerAdapter verbose: server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: 2024-01-08T00:31:02.295Z [INFO] server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31a1b4e1-a672-c703-b03b-4730ec8b4a57 Address:31a1b4e1-a672-c703-b03b-4730ec8b4a57} {Suffrage:Voter ID:server-8176d594-816f-b691-12da-1cf5ae6bae90 Address:8176d594-816f-b691-12da-1cf5ae6bae90} {Suffrage:Voter ID:server-13f348d1-8e65-864d-3378-178df005bf6d Address:13f348d1-8e65-864d-3378-178df005bf6d}]" + testing.go:149: server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: 2024-01-08T00:31:02.295Z [INFO] server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31a1b4e1-a672-c703-b03b-4730ec8b4a57 Address:31a1b4e1-a672-c703-b03b-4730ec8b4a57} {Suffrage:Voter ID:server-8176d594-816f-b691-12da-1cf5ae6bae90 Address:8176d594-816f-b691-12da-1cf5ae6bae90} {Suffrage:Voter ID:server-13f348d1-8e65-864d-3378-178df005bf6d Address:13f348d1-8e65-864d-3378-178df005bf6d}]" +testLoggerAdapter verbose: server-8176d594-816f-b691-12da-1cf5ae6bae90: 2024-01-08T00:31:02.295Z [INFO] server-8176d594-816f-b691-12da-1cf5ae6bae90: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31a1b4e1-a672-c703-b03b-4730ec8b4a57 Address:31a1b4e1-a672-c703-b03b-4730ec8b4a57} {Suffrage:Voter ID:server-8176d594-816f-b691-12da-1cf5ae6bae90 Address:8176d594-816f-b691-12da-1cf5ae6bae90} {Suffrage:Voter ID:server-13f348d1-8e65-864d-3378-178df005bf6d Address:13f348d1-8e65-864d-3378-178df005bf6d}]" + testing.go:149: server-8176d594-816f-b691-12da-1cf5ae6bae90: 2024-01-08T00:31:02.295Z [INFO] server-8176d594-816f-b691-12da-1cf5ae6bae90: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31a1b4e1-a672-c703-b03b-4730ec8b4a57 Address:31a1b4e1-a672-c703-b03b-4730ec8b4a57} {Suffrage:Voter ID:server-8176d594-816f-b691-12da-1cf5ae6bae90 Address:8176d594-816f-b691-12da-1cf5ae6bae90} {Suffrage:Voter ID:server-13f348d1-8e65-864d-3378-178df005bf6d Address:13f348d1-8e65-864d-3378-178df005bf6d}]" +testLoggerAdapter verbose: server-13f348d1-8e65-864d-3378-178df005bf6d: 2024-01-08T00:31:02.295Z [INFO] server-13f348d1-8e65-864d-3378-178df005bf6d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31a1b4e1-a672-c703-b03b-4730ec8b4a57 Address:31a1b4e1-a672-c703-b03b-4730ec8b4a57} {Suffrage:Voter ID:server-8176d594-816f-b691-12da-1cf5ae6bae90 Address:8176d594-816f-b691-12da-1cf5ae6bae90} {Suffrage:Voter ID:server-13f348d1-8e65-864d-3378-178df005bf6d Address:13f348d1-8e65-864d-3378-178df005bf6d}]" + testing.go:149: server-13f348d1-8e65-864d-3378-178df005bf6d: 2024-01-08T00:31:02.295Z [INFO] server-13f348d1-8e65-864d-3378-178df005bf6d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31a1b4e1-a672-c703-b03b-4730ec8b4a57 Address:31a1b4e1-a672-c703-b03b-4730ec8b4a57} {Suffrage:Voter ID:server-8176d594-816f-b691-12da-1cf5ae6bae90 Address:8176d594-816f-b691-12da-1cf5ae6bae90} {Suffrage:Voter ID:server-13f348d1-8e65-864d-3378-178df005bf6d Address:13f348d1-8e65-864d-3378-178df005bf6d}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:02.295Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:02.295Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: 2024-01-08T00:31:02.295Z [INFO] server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: entering follower state: follower="Node at 31a1b4e1-a672-c703-b03b-4730ec8b4a57 [Follower]" leader= + testing.go:149: server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: 2024-01-08T00:31:02.295Z [INFO] server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: entering follower state: follower="Node at 31a1b4e1-a672-c703-b03b-4730ec8b4a57 [Follower]" leader= +testLoggerAdapter verbose: server-8176d594-816f-b691-12da-1cf5ae6bae90: 2024-01-08T00:31:02.295Z [INFO] server-8176d594-816f-b691-12da-1cf5ae6bae90: entering follower state: follower="Node at 8176d594-816f-b691-12da-1cf5ae6bae90 [Follower]" leader= + testing.go:149: server-8176d594-816f-b691-12da-1cf5ae6bae90: 2024-01-08T00:31:02.295Z [INFO] server-8176d594-816f-b691-12da-1cf5ae6bae90: entering follower state: follower="Node at 8176d594-816f-b691-12da-1cf5ae6bae90 [Follower]" leader= +testLoggerAdapter verbose: server-13f348d1-8e65-864d-3378-178df005bf6d: 2024-01-08T00:31:02.296Z [INFO] server-13f348d1-8e65-864d-3378-178df005bf6d: entering follower state: follower="Node at 13f348d1-8e65-864d-3378-178df005bf6d [Follower]" leader= + testing.go:149: server-13f348d1-8e65-864d-3378-178df005bf6d: 2024-01-08T00:31:02.296Z [INFO] server-13f348d1-8e65-864d-3378-178df005bf6d: entering follower state: follower="Node at 13f348d1-8e65-864d-3378-178df005bf6d [Follower]" leader= +testLoggerAdapter verbose: server-8176d594-816f-b691-12da-1cf5ae6bae90: 2024-01-08T00:31:02.349Z [WARN] server-8176d594-816f-b691-12da-1cf5ae6bae90: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-8176d594-816f-b691-12da-1cf5ae6bae90: 2024-01-08T00:31:02.349Z [WARN] server-8176d594-816f-b691-12da-1cf5ae6bae90: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-8176d594-816f-b691-12da-1cf5ae6bae90: 2024-01-08T00:31:02.349Z [INFO] server-8176d594-816f-b691-12da-1cf5ae6bae90: entering candidate state: node="Node at 8176d594-816f-b691-12da-1cf5ae6bae90 [Candidate]" term=2 + testing.go:149: server-8176d594-816f-b691-12da-1cf5ae6bae90: 2024-01-08T00:31:02.349Z [INFO] server-8176d594-816f-b691-12da-1cf5ae6bae90: entering candidate state: node="Node at 8176d594-816f-b691-12da-1cf5ae6bae90 [Candidate]" term=2 +testLoggerAdapter verbose: server-8176d594-816f-b691-12da-1cf5ae6bae90: 2024-01-08T00:31:02.349Z [INFO] server-8176d594-816f-b691-12da-1cf5ae6bae90: election won: tally=2 + testing.go:149: server-8176d594-816f-b691-12da-1cf5ae6bae90: 2024-01-08T00:31:02.349Z [INFO] server-8176d594-816f-b691-12da-1cf5ae6bae90: election won: tally=2 +testLoggerAdapter verbose: server-8176d594-816f-b691-12da-1cf5ae6bae90: 2024-01-08T00:31:02.349Z [INFO] server-8176d594-816f-b691-12da-1cf5ae6bae90: entering leader state: leader="Node at 8176d594-816f-b691-12da-1cf5ae6bae90 [Leader]" + testing.go:149: server-8176d594-816f-b691-12da-1cf5ae6bae90: 2024-01-08T00:31:02.349Z [INFO] server-8176d594-816f-b691-12da-1cf5ae6bae90: entering leader state: leader="Node at 8176d594-816f-b691-12da-1cf5ae6bae90 [Leader]" +testLoggerAdapter verbose: server-8176d594-816f-b691-12da-1cf5ae6bae90: 2024-01-08T00:31:02.350Z [INFO] server-8176d594-816f-b691-12da-1cf5ae6bae90: added peer, starting replication: peer=server-31a1b4e1-a672-c703-b03b-4730ec8b4a57 + testing.go:149: server-8176d594-816f-b691-12da-1cf5ae6bae90: 2024-01-08T00:31:02.350Z [INFO] server-8176d594-816f-b691-12da-1cf5ae6bae90: added peer, starting replication: peer=server-31a1b4e1-a672-c703-b03b-4730ec8b4a57 +testLoggerAdapter verbose: server-8176d594-816f-b691-12da-1cf5ae6bae90: 2024-01-08T00:31:02.350Z [INFO] server-8176d594-816f-b691-12da-1cf5ae6bae90: added peer, starting replication: peer=server-13f348d1-8e65-864d-3378-178df005bf6d + testing.go:149: server-8176d594-816f-b691-12da-1cf5ae6bae90: 2024-01-08T00:31:02.350Z [INFO] server-8176d594-816f-b691-12da-1cf5ae6bae90: added peer, starting replication: peer=server-13f348d1-8e65-864d-3378-178df005bf6d +testLoggerAdapter verbose: server-8176d594-816f-b691-12da-1cf5ae6bae90: 2024-01-08T00:31:02.350Z [INFO] server-8176d594-816f-b691-12da-1cf5ae6bae90: pipelining replication: peer="{Voter server-13f348d1-8e65-864d-3378-178df005bf6d 13f348d1-8e65-864d-3378-178df005bf6d}" + testing.go:149: server-8176d594-816f-b691-12da-1cf5ae6bae90: 2024-01-08T00:31:02.350Z [INFO] server-8176d594-816f-b691-12da-1cf5ae6bae90: pipelining replication: peer="{Voter server-13f348d1-8e65-864d-3378-178df005bf6d 13f348d1-8e65-864d-3378-178df005bf6d}" +testLoggerAdapter verbose: server-8176d594-816f-b691-12da-1cf5ae6bae90: 2024-01-08T00:31:02.350Z [INFO] server-8176d594-816f-b691-12da-1cf5ae6bae90: pipelining replication: peer="{Voter server-31a1b4e1-a672-c703-b03b-4730ec8b4a57 31a1b4e1-a672-c703-b03b-4730ec8b4a57}" + testing.go:149: server-8176d594-816f-b691-12da-1cf5ae6bae90: 2024-01-08T00:31:02.350Z [INFO] server-8176d594-816f-b691-12da-1cf5ae6bae90: pipelining replication: peer="{Voter server-31a1b4e1-a672-c703-b03b-4730ec8b4a57 31a1b4e1-a672-c703-b03b-4730ec8b4a57}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:02.455Z [INFO] stable state for Leader reached at 2024-01-08 00:31:02.350359518 +0000 UTC m=+8.397248609 (1 nodes), 54.471426ms from start of poll, 54.896596ms from cluster start. Timeout at 2024-01-08 00:31:02.455839037 +0000 UTC m=+8.502728160, 105.479551ms after stability + testing.go:149: cluster: 2024-01-08T00:31:02.455Z [INFO] stable state for Leader reached at 2024-01-08 00:31:02.350359518 +0000 UTC m=+8.397248609 (1 nodes), 54.471426ms from start of poll, 54.896596ms from cluster start. Timeout at 2024-01-08 00:31:02.455839037 +0000 UTC m=+8.502728160, 105.479551ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:02.461Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T00:31:02.461Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:02.566Z [INFO] stable state for Follower reached at 2024-01-08 00:31:02.461600292 +0000 UTC m=+8.508489380 (2 nodes), 2.473µs from start of poll, 166.137367ms from cluster start. Timeout at 2024-01-08 00:31:02.566645189 +0000 UTC m=+8.613534291, 105.044911ms after stability + testing.go:149: cluster: 2024-01-08T00:31:02.566Z [INFO] stable state for Follower reached at 2024-01-08 00:31:02.461600292 +0000 UTC m=+8.508489380 (2 nodes), 2.473µs from start of poll, 166.137367ms from cluster start. Timeout at 2024-01-08 00:31:02.566645189 +0000 UTC m=+8.613534291, 105.044911ms after stability +testLoggerAdapter verbose: server-8176d594-816f-b691-12da-1cf5ae6bae90: 2024-01-08T00:31:02.566Z [INFO] server-8176d594-816f-b691-12da-1cf5ae6bae90: updating configuration: command=RemoveServer server-id=server-8176d594-816f-b691-12da-1cf5ae6bae90 server-addr= servers="[{Suffrage:Voter ID:server-31a1b4e1-a672-c703-b03b-4730ec8b4a57 Address:31a1b4e1-a672-c703-b03b-4730ec8b4a57} {Suffrage:Voter ID:server-13f348d1-8e65-864d-3378-178df005bf6d Address:13f348d1-8e65-864d-3378-178df005bf6d}]" + testing.go:149: server-8176d594-816f-b691-12da-1cf5ae6bae90: 2024-01-08T00:31:02.566Z [INFO] server-8176d594-816f-b691-12da-1cf5ae6bae90: updating configuration: command=RemoveServer server-id=server-8176d594-816f-b691-12da-1cf5ae6bae90 server-addr= servers="[{Suffrage:Voter ID:server-31a1b4e1-a672-c703-b03b-4730ec8b4a57 Address:31a1b4e1-a672-c703-b03b-4730ec8b4a57} {Suffrage:Voter ID:server-13f348d1-8e65-864d-3378-178df005bf6d Address:13f348d1-8e65-864d-3378-178df005bf6d}]" +testLoggerAdapter verbose: server-8176d594-816f-b691-12da-1cf5ae6bae90: 2024-01-08T00:31:02.566Z [INFO] server-8176d594-816f-b691-12da-1cf5ae6bae90: removed ourself, shutting down + testing.go:149: server-8176d594-816f-b691-12da-1cf5ae6bae90: 2024-01-08T00:31:02.566Z [INFO] server-8176d594-816f-b691-12da-1cf5ae6bae90: removed ourself, shutting down +testLoggerAdapter verbose: server-8176d594-816f-b691-12da-1cf5ae6bae90: 2024-01-08T00:31:02.567Z [INFO] server-8176d594-816f-b691-12da-1cf5ae6bae90: aborting pipeline replication: peer="{Voter server-13f348d1-8e65-864d-3378-178df005bf6d 13f348d1-8e65-864d-3378-178df005bf6d}" + testing.go:149: server-8176d594-816f-b691-12da-1cf5ae6bae90: 2024-01-08T00:31:02.567Z [INFO] server-8176d594-816f-b691-12da-1cf5ae6bae90: aborting pipeline replication: peer="{Voter server-13f348d1-8e65-864d-3378-178df005bf6d 13f348d1-8e65-864d-3378-178df005bf6d}" +testLoggerAdapter verbose: server-8176d594-816f-b691-12da-1cf5ae6bae90: 2024-01-08T00:31:02.567Z [INFO] server-8176d594-816f-b691-12da-1cf5ae6bae90: aborting pipeline replication: peer="{Voter server-31a1b4e1-a672-c703-b03b-4730ec8b4a57 31a1b4e1-a672-c703-b03b-4730ec8b4a57}" + testing.go:149: server-8176d594-816f-b691-12da-1cf5ae6bae90: 2024-01-08T00:31:02.567Z [INFO] server-8176d594-816f-b691-12da-1cf5ae6bae90: aborting pipeline replication: peer="{Voter server-31a1b4e1-a672-c703-b03b-4730ec8b4a57 31a1b4e1-a672-c703-b03b-4730ec8b4a57}" +testLoggerAdapter verbose: server-13f348d1-8e65-864d-3378-178df005bf6d: 2024-01-08T00:31:02.629Z [WARN] server-13f348d1-8e65-864d-3378-178df005bf6d: heartbeat timeout reached, starting election: last-leader=8176d594-816f-b691-12da-1cf5ae6bae90 + testing.go:149: server-13f348d1-8e65-864d-3378-178df005bf6d: 2024-01-08T00:31:02.629Z [WARN] server-13f348d1-8e65-864d-3378-178df005bf6d: heartbeat timeout reached, starting election: last-leader=8176d594-816f-b691-12da-1cf5ae6bae90 +testLoggerAdapter verbose: server-13f348d1-8e65-864d-3378-178df005bf6d: 2024-01-08T00:31:02.629Z [INFO] server-13f348d1-8e65-864d-3378-178df005bf6d: entering candidate state: node="Node at 13f348d1-8e65-864d-3378-178df005bf6d [Candidate]" term=3 + testing.go:149: server-13f348d1-8e65-864d-3378-178df005bf6d: 2024-01-08T00:31:02.629Z [INFO] server-13f348d1-8e65-864d-3378-178df005bf6d: entering candidate state: node="Node at 13f348d1-8e65-864d-3378-178df005bf6d [Candidate]" term=3 +testLoggerAdapter verbose: server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: 2024-01-08T00:31:02.629Z [WARN] server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: rejecting vote request since we have a leader: from=13f348d1-8e65-864d-3378-178df005bf6d leader=8176d594-816f-b691-12da-1cf5ae6bae90 + testing.go:149: server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: 2024-01-08T00:31:02.629Z [WARN] server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: rejecting vote request since we have a leader: from=13f348d1-8e65-864d-3378-178df005bf6d leader=8176d594-816f-b691-12da-1cf5ae6bae90 +testLoggerAdapter verbose: server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: 2024-01-08T00:31:02.657Z [WARN] server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: heartbeat timeout reached, starting election: last-leader=8176d594-816f-b691-12da-1cf5ae6bae90 + testing.go:149: server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: 2024-01-08T00:31:02.657Z [WARN] server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: heartbeat timeout reached, starting election: last-leader=8176d594-816f-b691-12da-1cf5ae6bae90 +testLoggerAdapter verbose: server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: 2024-01-08T00:31:02.657Z [INFO] server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: entering candidate state: node="Node at 31a1b4e1-a672-c703-b03b-4730ec8b4a57 [Candidate]" term=3 + testing.go:149: server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: 2024-01-08T00:31:02.657Z [INFO] server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: entering candidate state: node="Node at 31a1b4e1-a672-c703-b03b-4730ec8b4a57 [Candidate]" term=3 +testLoggerAdapter verbose: server-13f348d1-8e65-864d-3378-178df005bf6d: 2024-01-08T00:31:02.657Z [INFO] server-13f348d1-8e65-864d-3378-178df005bf6d: duplicate requestVote for same term: term=3 + testing.go:149: server-13f348d1-8e65-864d-3378-178df005bf6d: 2024-01-08T00:31:02.657Z [INFO] server-13f348d1-8e65-864d-3378-178df005bf6d: duplicate requestVote for same term: term=3 +testLoggerAdapter verbose: server-13f348d1-8e65-864d-3378-178df005bf6d: 2024-01-08T00:31:02.728Z [WARN] server-13f348d1-8e65-864d-3378-178df005bf6d: Election timeout reached, restarting election + testing.go:149: server-13f348d1-8e65-864d-3378-178df005bf6d: 2024-01-08T00:31:02.728Z [WARN] server-13f348d1-8e65-864d-3378-178df005bf6d: Election timeout reached, restarting election +testLoggerAdapter verbose: server-13f348d1-8e65-864d-3378-178df005bf6d: 2024-01-08T00:31:02.728Z [INFO] server-13f348d1-8e65-864d-3378-178df005bf6d: entering candidate state: node="Node at 13f348d1-8e65-864d-3378-178df005bf6d [Candidate]" term=4 + testing.go:149: server-13f348d1-8e65-864d-3378-178df005bf6d: 2024-01-08T00:31:02.728Z [INFO] server-13f348d1-8e65-864d-3378-178df005bf6d: entering candidate state: node="Node at 13f348d1-8e65-864d-3378-178df005bf6d [Candidate]" term=4 +testLoggerAdapter verbose: server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: 2024-01-08T00:31:02.728Z [WARN] server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: Election timeout reached, restarting election + testing.go:149: server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: 2024-01-08T00:31:02.728Z [WARN] server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: Election timeout reached, restarting election +testLoggerAdapter verbose: server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: 2024-01-08T00:31:02.729Z [INFO] server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: entering candidate state: node="Node at 31a1b4e1-a672-c703-b03b-4730ec8b4a57 [Candidate]" term=4 + testing.go:149: server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: 2024-01-08T00:31:02.729Z [INFO] server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: entering candidate state: node="Node at 31a1b4e1-a672-c703-b03b-4730ec8b4a57 [Candidate]" term=4 +testLoggerAdapter verbose: server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: 2024-01-08T00:31:02.729Z [INFO] server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: duplicate requestVote for same term: term=4 + testing.go:149: server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: 2024-01-08T00:31:02.729Z [INFO] server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: duplicate requestVote for same term: term=4 +testLoggerAdapter verbose: server-13f348d1-8e65-864d-3378-178df005bf6d: 2024-01-08T00:31:02.729Z [INFO] server-13f348d1-8e65-864d-3378-178df005bf6d: duplicate requestVote for same term: term=4 + testing.go:149: server-13f348d1-8e65-864d-3378-178df005bf6d: 2024-01-08T00:31:02.729Z [INFO] server-13f348d1-8e65-864d-3378-178df005bf6d: duplicate requestVote for same term: term=4 +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:02.778Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:02.778Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: 2024-01-08T00:31:02.804Z [WARN] server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: Election timeout reached, restarting election + testing.go:149: server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: 2024-01-08T00:31:02.804Z [WARN] server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: Election timeout reached, restarting election +testLoggerAdapter verbose: server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: 2024-01-08T00:31:02.804Z [INFO] server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: entering candidate state: node="Node at 31a1b4e1-a672-c703-b03b-4730ec8b4a57 [Candidate]" term=5 + testing.go:149: server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: 2024-01-08T00:31:02.804Z [INFO] server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: entering candidate state: node="Node at 31a1b4e1-a672-c703-b03b-4730ec8b4a57 [Candidate]" term=5 +testLoggerAdapter verbose: server-13f348d1-8e65-864d-3378-178df005bf6d: 2024-01-08T00:31:02.804Z [INFO] server-13f348d1-8e65-864d-3378-178df005bf6d: entering follower state: follower="Node at 13f348d1-8e65-864d-3378-178df005bf6d [Follower]" leader= + testing.go:149: server-13f348d1-8e65-864d-3378-178df005bf6d: 2024-01-08T00:31:02.804Z [INFO] server-13f348d1-8e65-864d-3378-178df005bf6d: entering follower state: follower="Node at 13f348d1-8e65-864d-3378-178df005bf6d [Follower]" leader= +testLoggerAdapter verbose: server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: 2024-01-08T00:31:02.804Z [INFO] server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: election won: tally=2 + testing.go:149: server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: 2024-01-08T00:31:02.804Z [INFO] server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: election won: tally=2 +testLoggerAdapter verbose: server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: 2024-01-08T00:31:02.804Z [INFO] server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: entering leader state: leader="Node at 31a1b4e1-a672-c703-b03b-4730ec8b4a57 [Leader]" + testing.go:149: server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: 2024-01-08T00:31:02.804Z [INFO] server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: entering leader state: leader="Node at 31a1b4e1-a672-c703-b03b-4730ec8b4a57 [Leader]" +testLoggerAdapter verbose: server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: 2024-01-08T00:31:02.804Z [INFO] server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: added peer, starting replication: peer=server-13f348d1-8e65-864d-3378-178df005bf6d + testing.go:149: server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: 2024-01-08T00:31:02.804Z [INFO] server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: added peer, starting replication: peer=server-13f348d1-8e65-864d-3378-178df005bf6d +testLoggerAdapter verbose: server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: 2024-01-08T00:31:02.804Z [INFO] server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: pipelining replication: peer="{Voter server-13f348d1-8e65-864d-3378-178df005bf6d 13f348d1-8e65-864d-3378-178df005bf6d}" + testing.go:149: server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: 2024-01-08T00:31:02.804Z [INFO] server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: pipelining replication: peer="{Voter server-13f348d1-8e65-864d-3378-178df005bf6d 13f348d1-8e65-864d-3378-178df005bf6d}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:02.909Z [INFO] stable state for Leader reached at 2024-01-08 00:31:02.804549042 +0000 UTC m=+8.851438128 (1 nodes), 25.935905ms from start of poll, 509.086115ms from cluster start. Timeout at 2024-01-08 00:31:02.909691093 +0000 UTC m=+8.956580165, 105.142037ms after stability + testing.go:149: cluster: 2024-01-08T00:31:02.909Z [INFO] stable state for Leader reached at 2024-01-08 00:31:02.804549042 +0000 UTC m=+8.851438128 (1 nodes), 25.935905ms from start of poll, 509.086115ms from cluster start. Timeout at 2024-01-08 00:31:02.909691093 +0000 UTC m=+8.956580165, 105.142037ms after stability +testLoggerAdapter verbose: server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: 2024-01-08T00:31:02.909Z [INFO] server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: aborting pipeline replication: peer="{Voter server-13f348d1-8e65-864d-3378-178df005bf6d 13f348d1-8e65-864d-3378-178df005bf6d}" + testing.go:149: server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: 2024-01-08T00:31:02.909Z [INFO] server-31a1b4e1-a672-c703-b03b-4730ec8b4a57: aborting pipeline replication: peer="{Voter server-13f348d1-8e65-864d-3378-178df005bf6d 13f348d1-8e65-864d-3378-178df005bf6d}" +--- PASS: TestRaft_RemoveLeader (0.62s) === RUN TestRaft_RemoveLeader_NoShutdown -testLoggerAdapter verbose: server-c677d058-06e6-6ad9-91fa-9babc09d6d1c: 2025-02-09T06:52:27.935Z [INFO] server-c677d058-06e6-6ad9-91fa-9babc09d6d1c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c677d058-06e6-6ad9-91fa-9babc09d6d1c Address:c677d058-06e6-6ad9-91fa-9babc09d6d1c} {Suffrage:Voter ID:server-0838c6e9-239d-13e4-fd19-9e5d97774dfa Address:0838c6e9-239d-13e4-fd19-9e5d97774dfa} {Suffrage:Voter ID:server-ae580321-9259-4b53-4ab2-b11f6c17aaa8 Address:ae580321-9259-4b53-4ab2-b11f6c17aaa8}]" - testing.go:149: server-c677d058-06e6-6ad9-91fa-9babc09d6d1c: 2025-02-09T06:52:27.935Z [INFO] server-c677d058-06e6-6ad9-91fa-9babc09d6d1c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c677d058-06e6-6ad9-91fa-9babc09d6d1c Address:c677d058-06e6-6ad9-91fa-9babc09d6d1c} {Suffrage:Voter ID:server-0838c6e9-239d-13e4-fd19-9e5d97774dfa Address:0838c6e9-239d-13e4-fd19-9e5d97774dfa} {Suffrage:Voter ID:server-ae580321-9259-4b53-4ab2-b11f6c17aaa8 Address:ae580321-9259-4b53-4ab2-b11f6c17aaa8}]" -testLoggerAdapter verbose: server-c677d058-06e6-6ad9-91fa-9babc09d6d1c: 2025-02-09T06:52:27.935Z [INFO] server-c677d058-06e6-6ad9-91fa-9babc09d6d1c: entering follower state: follower="Node at c677d058-06e6-6ad9-91fa-9babc09d6d1c [Follower]" leader= - testing.go:149: server-c677d058-06e6-6ad9-91fa-9babc09d6d1c: 2025-02-09T06:52:27.935Z [INFO] server-c677d058-06e6-6ad9-91fa-9babc09d6d1c: entering follower state: follower="Node at c677d058-06e6-6ad9-91fa-9babc09d6d1c [Follower]" leader= -testLoggerAdapter verbose: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:27.935Z [INFO] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c677d058-06e6-6ad9-91fa-9babc09d6d1c Address:c677d058-06e6-6ad9-91fa-9babc09d6d1c} {Suffrage:Voter ID:server-0838c6e9-239d-13e4-fd19-9e5d97774dfa Address:0838c6e9-239d-13e4-fd19-9e5d97774dfa} {Suffrage:Voter ID:server-ae580321-9259-4b53-4ab2-b11f6c17aaa8 Address:ae580321-9259-4b53-4ab2-b11f6c17aaa8}]" - testing.go:149: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:27.935Z [INFO] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c677d058-06e6-6ad9-91fa-9babc09d6d1c Address:c677d058-06e6-6ad9-91fa-9babc09d6d1c} {Suffrage:Voter ID:server-0838c6e9-239d-13e4-fd19-9e5d97774dfa Address:0838c6e9-239d-13e4-fd19-9e5d97774dfa} {Suffrage:Voter ID:server-ae580321-9259-4b53-4ab2-b11f6c17aaa8 Address:ae580321-9259-4b53-4ab2-b11f6c17aaa8}]" -testLoggerAdapter verbose: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:27.935Z [INFO] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: entering follower state: follower="Node at 0838c6e9-239d-13e4-fd19-9e5d97774dfa [Follower]" leader= - testing.go:149: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:27.935Z [INFO] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: entering follower state: follower="Node at 0838c6e9-239d-13e4-fd19-9e5d97774dfa [Follower]" leader= -testLoggerAdapter verbose: server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: 2025-02-09T06:52:27.935Z [INFO] server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c677d058-06e6-6ad9-91fa-9babc09d6d1c Address:c677d058-06e6-6ad9-91fa-9babc09d6d1c} {Suffrage:Voter ID:server-0838c6e9-239d-13e4-fd19-9e5d97774dfa Address:0838c6e9-239d-13e4-fd19-9e5d97774dfa} {Suffrage:Voter ID:server-ae580321-9259-4b53-4ab2-b11f6c17aaa8 Address:ae580321-9259-4b53-4ab2-b11f6c17aaa8}]" - testing.go:149: server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: 2025-02-09T06:52:27.935Z [INFO] server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c677d058-06e6-6ad9-91fa-9babc09d6d1c Address:c677d058-06e6-6ad9-91fa-9babc09d6d1c} {Suffrage:Voter ID:server-0838c6e9-239d-13e4-fd19-9e5d97774dfa Address:0838c6e9-239d-13e4-fd19-9e5d97774dfa} {Suffrage:Voter ID:server-ae580321-9259-4b53-4ab2-b11f6c17aaa8 Address:ae580321-9259-4b53-4ab2-b11f6c17aaa8}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:27.935Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T06:52:27.935Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: 2025-02-09T06:52:27.935Z [INFO] server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: entering follower state: follower="Node at ae580321-9259-4b53-4ab2-b11f6c17aaa8 [Follower]" leader= - testing.go:149: server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: 2025-02-09T06:52:27.935Z [INFO] server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: entering follower state: follower="Node at ae580321-9259-4b53-4ab2-b11f6c17aaa8 [Follower]" leader= -testLoggerAdapter verbose: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:27.989Z [WARN] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:27.989Z [WARN] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:27.989Z [INFO] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: entering candidate state: node="Node at 0838c6e9-239d-13e4-fd19-9e5d97774dfa [Candidate]" term=2 - testing.go:149: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:27.989Z [INFO] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: entering candidate state: node="Node at 0838c6e9-239d-13e4-fd19-9e5d97774dfa [Candidate]" term=2 -testLoggerAdapter verbose: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:27.990Z [INFO] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: election won: tally=2 - testing.go:149: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:27.990Z [INFO] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: election won: tally=2 -testLoggerAdapter verbose: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:27.990Z [INFO] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: entering leader state: leader="Node at 0838c6e9-239d-13e4-fd19-9e5d97774dfa [Leader]" - testing.go:149: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:27.990Z [INFO] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: entering leader state: leader="Node at 0838c6e9-239d-13e4-fd19-9e5d97774dfa [Leader]" -testLoggerAdapter verbose: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:27.990Z [INFO] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: added peer, starting replication: peer=server-c677d058-06e6-6ad9-91fa-9babc09d6d1c - testing.go:149: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:27.990Z [INFO] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: added peer, starting replication: peer=server-c677d058-06e6-6ad9-91fa-9babc09d6d1c -testLoggerAdapter verbose: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:27.990Z [INFO] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: added peer, starting replication: peer=server-ae580321-9259-4b53-4ab2-b11f6c17aaa8 - testing.go:149: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:27.990Z [INFO] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: added peer, starting replication: peer=server-ae580321-9259-4b53-4ab2-b11f6c17aaa8 -testLoggerAdapter verbose: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:27.990Z [INFO] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: pipelining replication: peer="{Voter server-ae580321-9259-4b53-4ab2-b11f6c17aaa8 ae580321-9259-4b53-4ab2-b11f6c17aaa8}" - testing.go:149: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:27.990Z [INFO] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: pipelining replication: peer="{Voter server-ae580321-9259-4b53-4ab2-b11f6c17aaa8 ae580321-9259-4b53-4ab2-b11f6c17aaa8}" -testLoggerAdapter verbose: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:27.990Z [INFO] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: pipelining replication: peer="{Voter server-c677d058-06e6-6ad9-91fa-9babc09d6d1c c677d058-06e6-6ad9-91fa-9babc09d6d1c}" - testing.go:149: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:27.990Z [INFO] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: pipelining replication: peer="{Voter server-c677d058-06e6-6ad9-91fa-9babc09d6d1c c677d058-06e6-6ad9-91fa-9babc09d6d1c}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:28.095Z [INFO] stable state for Follower reached at 2025-02-09 06:52:27.990103778 +0000 UTC m=+8.968912543 (2 nodes), 54.473464ms from start of poll, 54.853943ms from cluster start. Timeout at 2025-02-09 06:52:28.095424295 +0000 UTC m=+9.074233052, 105.320509ms after stability - testing.go:149: cluster: 2025-02-09T06:52:28.095Z [INFO] stable state for Follower reached at 2025-02-09 06:52:27.990103778 +0000 UTC m=+8.968912543 (2 nodes), 54.473464ms from start of poll, 54.853943ms from cluster start. Timeout at 2025-02-09 06:52:28.095424295 +0000 UTC m=+9.074233052, 105.320509ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:28.095Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:28.095Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:28.200Z [INFO] stable state for Leader reached at 2025-02-09 06:52:28.095485378 +0000 UTC m=+9.074294144 (1 nodes), 465ns from start of poll, 160.235544ms from cluster start. Timeout at 2025-02-09 06:52:28.200883484 +0000 UTC m=+9.179692252, 105.398108ms after stability - testing.go:149: cluster: 2025-02-09T06:52:28.200Z [INFO] stable state for Leader reached at 2025-02-09 06:52:28.095485378 +0000 UTC m=+9.074294144 (1 nodes), 465ns from start of poll, 160.235544ms from cluster start. Timeout at 2025-02-09 06:52:28.200883484 +0000 UTC m=+9.179692252, 105.398108ms after stability -testLoggerAdapter verbose: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:28.201Z [INFO] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: updating configuration: command=RemoveServer server-id=server-0838c6e9-239d-13e4-fd19-9e5d97774dfa server-addr= servers="[{Suffrage:Voter ID:server-c677d058-06e6-6ad9-91fa-9babc09d6d1c Address:c677d058-06e6-6ad9-91fa-9babc09d6d1c} {Suffrage:Voter ID:server-ae580321-9259-4b53-4ab2-b11f6c17aaa8 Address:ae580321-9259-4b53-4ab2-b11f6c17aaa8}]" - testing.go:149: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:28.201Z [INFO] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: updating configuration: command=RemoveServer server-id=server-0838c6e9-239d-13e4-fd19-9e5d97774dfa server-addr= servers="[{Suffrage:Voter ID:server-c677d058-06e6-6ad9-91fa-9babc09d6d1c Address:c677d058-06e6-6ad9-91fa-9babc09d6d1c} {Suffrage:Voter ID:server-ae580321-9259-4b53-4ab2-b11f6c17aaa8 Address:ae580321-9259-4b53-4ab2-b11f6c17aaa8}]" -testLoggerAdapter verbose: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:28.202Z [INFO] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: removed ourself, transitioning to follower - testing.go:149: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:28.202Z [INFO] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: removed ourself, transitioning to follower -testLoggerAdapter verbose: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:28.202Z [INFO] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: entering follower state: follower="Node at 0838c6e9-239d-13e4-fd19-9e5d97774dfa [Follower]" leader= - testing.go:149: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:28.202Z [INFO] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: entering follower state: follower="Node at 0838c6e9-239d-13e4-fd19-9e5d97774dfa [Follower]" leader= -testLoggerAdapter verbose: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:28.202Z [INFO] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: aborting pipeline replication: peer="{Voter server-c677d058-06e6-6ad9-91fa-9babc09d6d1c c677d058-06e6-6ad9-91fa-9babc09d6d1c}" - testing.go:149: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:28.202Z [INFO] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: aborting pipeline replication: peer="{Voter server-c677d058-06e6-6ad9-91fa-9babc09d6d1c c677d058-06e6-6ad9-91fa-9babc09d6d1c}" -testLoggerAdapter verbose: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:28.202Z [INFO] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: aborting pipeline replication: peer="{Voter server-ae580321-9259-4b53-4ab2-b11f6c17aaa8 ae580321-9259-4b53-4ab2-b11f6c17aaa8}" - testing.go:149: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:28.202Z [INFO] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: aborting pipeline replication: peer="{Voter server-ae580321-9259-4b53-4ab2-b11f6c17aaa8 ae580321-9259-4b53-4ab2-b11f6c17aaa8}" -testLoggerAdapter verbose: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:28.293Z [WARN] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: not part of stable configuration, aborting election - testing.go:149: server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: 2025-02-09T06:52:28.293Z [WARN] server-0838c6e9-239d-13e4-fd19-9e5d97774dfa: not part of stable configuration, aborting election -testLoggerAdapter verbose: server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: 2025-02-09T06:52:28.298Z [WARN] server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: heartbeat timeout reached, starting election: last-leader=0838c6e9-239d-13e4-fd19-9e5d97774dfa - testing.go:149: server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: 2025-02-09T06:52:28.298Z [WARN] server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: heartbeat timeout reached, starting election: last-leader=0838c6e9-239d-13e4-fd19-9e5d97774dfa -testLoggerAdapter verbose: server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: 2025-02-09T06:52:28.298Z [INFO] server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: entering candidate state: node="Node at ae580321-9259-4b53-4ab2-b11f6c17aaa8 [Candidate]" term=3 - testing.go:149: server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: 2025-02-09T06:52:28.298Z [INFO] server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: entering candidate state: node="Node at ae580321-9259-4b53-4ab2-b11f6c17aaa8 [Candidate]" term=3 -testLoggerAdapter verbose: server-c677d058-06e6-6ad9-91fa-9babc09d6d1c: 2025-02-09T06:52:28.298Z [WARN] server-c677d058-06e6-6ad9-91fa-9babc09d6d1c: rejecting vote request since we have a leader: from=ae580321-9259-4b53-4ab2-b11f6c17aaa8 leader=0838c6e9-239d-13e4-fd19-9e5d97774dfa - testing.go:149: server-c677d058-06e6-6ad9-91fa-9babc09d6d1c: 2025-02-09T06:52:28.298Z [WARN] server-c677d058-06e6-6ad9-91fa-9babc09d6d1c: rejecting vote request since we have a leader: from=ae580321-9259-4b53-4ab2-b11f6c17aaa8 leader=0838c6e9-239d-13e4-fd19-9e5d97774dfa -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:28.307Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:28.307Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-c677d058-06e6-6ad9-91fa-9babc09d6d1c: 2025-02-09T06:52:28.308Z [WARN] server-c677d058-06e6-6ad9-91fa-9babc09d6d1c: heartbeat timeout reached, starting election: last-leader=0838c6e9-239d-13e4-fd19-9e5d97774dfa - testing.go:149: server-c677d058-06e6-6ad9-91fa-9babc09d6d1c: 2025-02-09T06:52:28.308Z [WARN] server-c677d058-06e6-6ad9-91fa-9babc09d6d1c: heartbeat timeout reached, starting election: last-leader=0838c6e9-239d-13e4-fd19-9e5d97774dfa -testLoggerAdapter verbose: server-c677d058-06e6-6ad9-91fa-9babc09d6d1c: 2025-02-09T06:52:28.308Z [INFO] server-c677d058-06e6-6ad9-91fa-9babc09d6d1c: entering candidate state: node="Node at c677d058-06e6-6ad9-91fa-9babc09d6d1c [Candidate]" term=3 - testing.go:149: server-c677d058-06e6-6ad9-91fa-9babc09d6d1c: 2025-02-09T06:52:28.308Z [INFO] server-c677d058-06e6-6ad9-91fa-9babc09d6d1c: entering candidate state: node="Node at c677d058-06e6-6ad9-91fa-9babc09d6d1c [Candidate]" term=3 -testLoggerAdapter verbose: server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: 2025-02-09T06:52:28.308Z [INFO] server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: duplicate requestVote for same term: term=3 - testing.go:149: server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: 2025-02-09T06:52:28.308Z [INFO] server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: duplicate requestVote for same term: term=3 -testLoggerAdapter verbose: server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: 2025-02-09T06:52:28.348Z [WARN] server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: Election timeout reached, restarting election - testing.go:149: server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: 2025-02-09T06:52:28.348Z [WARN] server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: Election timeout reached, restarting election -testLoggerAdapter verbose: server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: 2025-02-09T06:52:28.348Z [INFO] server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: entering candidate state: node="Node at ae580321-9259-4b53-4ab2-b11f6c17aaa8 [Candidate]" term=4 - testing.go:149: server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: 2025-02-09T06:52:28.348Z [INFO] server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: entering candidate state: node="Node at ae580321-9259-4b53-4ab2-b11f6c17aaa8 [Candidate]" term=4 -testLoggerAdapter verbose: server-c677d058-06e6-6ad9-91fa-9babc09d6d1c: 2025-02-09T06:52:28.349Z [INFO] server-c677d058-06e6-6ad9-91fa-9babc09d6d1c: entering follower state: follower="Node at c677d058-06e6-6ad9-91fa-9babc09d6d1c [Follower]" leader= - testing.go:149: server-c677d058-06e6-6ad9-91fa-9babc09d6d1c: 2025-02-09T06:52:28.349Z [INFO] server-c677d058-06e6-6ad9-91fa-9babc09d6d1c: entering follower state: follower="Node at c677d058-06e6-6ad9-91fa-9babc09d6d1c [Follower]" leader= -testLoggerAdapter verbose: server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: 2025-02-09T06:52:28.349Z [INFO] server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: election won: tally=2 - testing.go:149: server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: 2025-02-09T06:52:28.349Z [INFO] server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: election won: tally=2 -testLoggerAdapter verbose: server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: 2025-02-09T06:52:28.349Z [INFO] server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: entering leader state: leader="Node at ae580321-9259-4b53-4ab2-b11f6c17aaa8 [Leader]" - testing.go:149: server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: 2025-02-09T06:52:28.349Z [INFO] server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: entering leader state: leader="Node at ae580321-9259-4b53-4ab2-b11f6c17aaa8 [Leader]" -testLoggerAdapter verbose: server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: 2025-02-09T06:52:28.349Z [INFO] server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: added peer, starting replication: peer=server-c677d058-06e6-6ad9-91fa-9babc09d6d1c - testing.go:149: server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: 2025-02-09T06:52:28.349Z [INFO] server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: added peer, starting replication: peer=server-c677d058-06e6-6ad9-91fa-9babc09d6d1c -testLoggerAdapter verbose: server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: 2025-02-09T06:52:28.349Z [INFO] server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: pipelining replication: peer="{Voter server-c677d058-06e6-6ad9-91fa-9babc09d6d1c c677d058-06e6-6ad9-91fa-9babc09d6d1c}" - testing.go:149: server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: 2025-02-09T06:52:28.349Z [INFO] server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: pipelining replication: peer="{Voter server-c677d058-06e6-6ad9-91fa-9babc09d6d1c c677d058-06e6-6ad9-91fa-9babc09d6d1c}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:28.454Z [INFO] stable state for Leader reached at 2025-02-09 06:52:28.3490993 +0000 UTC m=+9.327908075 (1 nodes), 41.599008ms from start of poll, 413.849475ms from cluster start. Timeout at 2025-02-09 06:52:28.454773371 +0000 UTC m=+9.433582124, 105.674049ms after stability - testing.go:149: cluster: 2025-02-09T06:52:28.454Z [INFO] stable state for Leader reached at 2025-02-09 06:52:28.3490993 +0000 UTC m=+9.327908075 (1 nodes), 41.599008ms from start of poll, 413.849475ms from cluster start. Timeout at 2025-02-09 06:52:28.454773371 +0000 UTC m=+9.433582124, 105.674049ms after stability -testLoggerAdapter verbose: server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: 2025-02-09T06:52:28.560Z [INFO] server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: aborting pipeline replication: peer="{Voter server-c677d058-06e6-6ad9-91fa-9babc09d6d1c c677d058-06e6-6ad9-91fa-9babc09d6d1c}" - testing.go:149: server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: 2025-02-09T06:52:28.560Z [INFO] server-ae580321-9259-4b53-4ab2-b11f6c17aaa8: aborting pipeline replication: peer="{Voter server-c677d058-06e6-6ad9-91fa-9babc09d6d1c c677d058-06e6-6ad9-91fa-9babc09d6d1c}" ---- PASS: TestRaft_RemoveLeader_NoShutdown (0.63s) +testLoggerAdapter verbose: server-c8916d4d-c692-c08c-342f-b49dac6d77de: 2024-01-08T00:31:02.911Z [INFO] server-c8916d4d-c692-c08c-342f-b49dac6d77de: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c8916d4d-c692-c08c-342f-b49dac6d77de Address:c8916d4d-c692-c08c-342f-b49dac6d77de} {Suffrage:Voter ID:server-ba02d1da-3554-f86d-33cd-a66d1e967adb Address:ba02d1da-3554-f86d-33cd-a66d1e967adb} {Suffrage:Voter ID:server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61 Address:5ea0aa5b-efe8-e78c-befc-caf4b98dfd61}]" + testing.go:149: server-c8916d4d-c692-c08c-342f-b49dac6d77de: 2024-01-08T00:31:02.911Z [INFO] server-c8916d4d-c692-c08c-342f-b49dac6d77de: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c8916d4d-c692-c08c-342f-b49dac6d77de Address:c8916d4d-c692-c08c-342f-b49dac6d77de} {Suffrage:Voter ID:server-ba02d1da-3554-f86d-33cd-a66d1e967adb Address:ba02d1da-3554-f86d-33cd-a66d1e967adb} {Suffrage:Voter ID:server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61 Address:5ea0aa5b-efe8-e78c-befc-caf4b98dfd61}]" +testLoggerAdapter verbose: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:02.911Z [INFO] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c8916d4d-c692-c08c-342f-b49dac6d77de Address:c8916d4d-c692-c08c-342f-b49dac6d77de} {Suffrage:Voter ID:server-ba02d1da-3554-f86d-33cd-a66d1e967adb Address:ba02d1da-3554-f86d-33cd-a66d1e967adb} {Suffrage:Voter ID:server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61 Address:5ea0aa5b-efe8-e78c-befc-caf4b98dfd61}]" + testing.go:149: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:02.911Z [INFO] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c8916d4d-c692-c08c-342f-b49dac6d77de Address:c8916d4d-c692-c08c-342f-b49dac6d77de} {Suffrage:Voter ID:server-ba02d1da-3554-f86d-33cd-a66d1e967adb Address:ba02d1da-3554-f86d-33cd-a66d1e967adb} {Suffrage:Voter ID:server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61 Address:5ea0aa5b-efe8-e78c-befc-caf4b98dfd61}]" +testLoggerAdapter verbose: server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: 2024-01-08T00:31:02.911Z [INFO] server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c8916d4d-c692-c08c-342f-b49dac6d77de Address:c8916d4d-c692-c08c-342f-b49dac6d77de} {Suffrage:Voter ID:server-ba02d1da-3554-f86d-33cd-a66d1e967adb Address:ba02d1da-3554-f86d-33cd-a66d1e967adb} {Suffrage:Voter ID:server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61 Address:5ea0aa5b-efe8-e78c-befc-caf4b98dfd61}]" + testing.go:149: server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: 2024-01-08T00:31:02.911Z [INFO] server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c8916d4d-c692-c08c-342f-b49dac6d77de Address:c8916d4d-c692-c08c-342f-b49dac6d77de} {Suffrage:Voter ID:server-ba02d1da-3554-f86d-33cd-a66d1e967adb Address:ba02d1da-3554-f86d-33cd-a66d1e967adb} {Suffrage:Voter ID:server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61 Address:5ea0aa5b-efe8-e78c-befc-caf4b98dfd61}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:02.911Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T00:31:02.911Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: server-c8916d4d-c692-c08c-342f-b49dac6d77de: 2024-01-08T00:31:02.911Z [INFO] server-c8916d4d-c692-c08c-342f-b49dac6d77de: entering follower state: follower="Node at c8916d4d-c692-c08c-342f-b49dac6d77de [Follower]" leader= + testing.go:149: server-c8916d4d-c692-c08c-342f-b49dac6d77de: 2024-01-08T00:31:02.911Z [INFO] server-c8916d4d-c692-c08c-342f-b49dac6d77de: entering follower state: follower="Node at c8916d4d-c692-c08c-342f-b49dac6d77de [Follower]" leader= +testLoggerAdapter verbose: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:02.911Z [INFO] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: entering follower state: follower="Node at ba02d1da-3554-f86d-33cd-a66d1e967adb [Follower]" leader= + testing.go:149: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:02.911Z [INFO] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: entering follower state: follower="Node at ba02d1da-3554-f86d-33cd-a66d1e967adb [Follower]" leader= +testLoggerAdapter verbose: server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: 2024-01-08T00:31:02.911Z [INFO] server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: entering follower state: follower="Node at 5ea0aa5b-efe8-e78c-befc-caf4b98dfd61 [Follower]" leader= + testing.go:149: server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: 2024-01-08T00:31:02.911Z [INFO] server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: entering follower state: follower="Node at 5ea0aa5b-efe8-e78c-befc-caf4b98dfd61 [Follower]" leader= +testLoggerAdapter verbose: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:02.979Z [WARN] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:02.979Z [WARN] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:02.979Z [INFO] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: entering candidate state: node="Node at ba02d1da-3554-f86d-33cd-a66d1e967adb [Candidate]" term=2 + testing.go:149: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:02.979Z [INFO] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: entering candidate state: node="Node at ba02d1da-3554-f86d-33cd-a66d1e967adb [Candidate]" term=2 +testLoggerAdapter verbose: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:02.979Z [INFO] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: election won: tally=2 + testing.go:149: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:02.979Z [INFO] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: election won: tally=2 +testLoggerAdapter verbose: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:02.979Z [INFO] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: entering leader state: leader="Node at ba02d1da-3554-f86d-33cd-a66d1e967adb [Leader]" + testing.go:149: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:02.979Z [INFO] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: entering leader state: leader="Node at ba02d1da-3554-f86d-33cd-a66d1e967adb [Leader]" +testLoggerAdapter verbose: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:02.979Z [INFO] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: added peer, starting replication: peer=server-c8916d4d-c692-c08c-342f-b49dac6d77de + testing.go:149: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:02.979Z [INFO] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: added peer, starting replication: peer=server-c8916d4d-c692-c08c-342f-b49dac6d77de +testLoggerAdapter verbose: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:02.979Z [INFO] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: added peer, starting replication: peer=server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61 + testing.go:149: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:02.979Z [INFO] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: added peer, starting replication: peer=server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61 +testLoggerAdapter verbose: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:02.979Z [INFO] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: pipelining replication: peer="{Voter server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61 5ea0aa5b-efe8-e78c-befc-caf4b98dfd61}" + testing.go:149: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:02.979Z [INFO] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: pipelining replication: peer="{Voter server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61 5ea0aa5b-efe8-e78c-befc-caf4b98dfd61}" +testLoggerAdapter verbose: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:02.979Z [INFO] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: pipelining replication: peer="{Voter server-c8916d4d-c692-c08c-342f-b49dac6d77de c8916d4d-c692-c08c-342f-b49dac6d77de}" + testing.go:149: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:02.979Z [INFO] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: pipelining replication: peer="{Voter server-c8916d4d-c692-c08c-342f-b49dac6d77de c8916d4d-c692-c08c-342f-b49dac6d77de}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:03.085Z [INFO] stable state for Follower reached at 2024-01-08 00:31:02.979678301 +0000 UTC m=+9.026567346 (2 nodes), 68.395986ms from start of poll, 68.767979ms from cluster start. Timeout at 2024-01-08 00:31:03.085207872 +0000 UTC m=+9.132096948, 105.529602ms after stability + testing.go:149: cluster: 2024-01-08T00:31:03.085Z [INFO] stable state for Follower reached at 2024-01-08 00:31:02.979678301 +0000 UTC m=+9.026567346 (2 nodes), 68.395986ms from start of poll, 68.767979ms from cluster start. Timeout at 2024-01-08 00:31:03.085207872 +0000 UTC m=+9.132096948, 105.529602ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:03.085Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:03.085Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:03.191Z [INFO] stable state for Leader reached at 2024-01-08 00:31:03.085325084 +0000 UTC m=+9.132214121 (1 nodes), 2.169µs from start of poll, 174.414754ms from cluster start. Timeout at 2024-01-08 00:31:03.19126752 +0000 UTC m=+9.238156588, 105.942467ms after stability + testing.go:149: cluster: 2024-01-08T00:31:03.191Z [INFO] stable state for Leader reached at 2024-01-08 00:31:03.085325084 +0000 UTC m=+9.132214121 (1 nodes), 2.169µs from start of poll, 174.414754ms from cluster start. Timeout at 2024-01-08 00:31:03.19126752 +0000 UTC m=+9.238156588, 105.942467ms after stability +testLoggerAdapter verbose: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:03.194Z [INFO] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: updating configuration: command=RemoveServer server-id=server-ba02d1da-3554-f86d-33cd-a66d1e967adb server-addr= servers="[{Suffrage:Voter ID:server-c8916d4d-c692-c08c-342f-b49dac6d77de Address:c8916d4d-c692-c08c-342f-b49dac6d77de} {Suffrage:Voter ID:server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61 Address:5ea0aa5b-efe8-e78c-befc-caf4b98dfd61}]" + testing.go:149: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:03.194Z [INFO] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: updating configuration: command=RemoveServer server-id=server-ba02d1da-3554-f86d-33cd-a66d1e967adb server-addr= servers="[{Suffrage:Voter ID:server-c8916d4d-c692-c08c-342f-b49dac6d77de Address:c8916d4d-c692-c08c-342f-b49dac6d77de} {Suffrage:Voter ID:server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61 Address:5ea0aa5b-efe8-e78c-befc-caf4b98dfd61}]" +testLoggerAdapter verbose: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:03.194Z [INFO] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: removed ourself, transitioning to follower + testing.go:149: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:03.194Z [INFO] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: removed ourself, transitioning to follower +testLoggerAdapter verbose: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:03.194Z [INFO] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: entering follower state: follower="Node at ba02d1da-3554-f86d-33cd-a66d1e967adb [Follower]" leader= + testing.go:149: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:03.194Z [INFO] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: entering follower state: follower="Node at ba02d1da-3554-f86d-33cd-a66d1e967adb [Follower]" leader= +testLoggerAdapter verbose: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:03.194Z [INFO] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: aborting pipeline replication: peer="{Voter server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61 5ea0aa5b-efe8-e78c-befc-caf4b98dfd61}" + testing.go:149: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:03.194Z [INFO] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: aborting pipeline replication: peer="{Voter server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61 5ea0aa5b-efe8-e78c-befc-caf4b98dfd61}" +testLoggerAdapter verbose: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:03.194Z [INFO] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: aborting pipeline replication: peer="{Voter server-c8916d4d-c692-c08c-342f-b49dac6d77de c8916d4d-c692-c08c-342f-b49dac6d77de}" + testing.go:149: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:03.194Z [INFO] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: aborting pipeline replication: peer="{Voter server-c8916d4d-c692-c08c-342f-b49dac6d77de c8916d4d-c692-c08c-342f-b49dac6d77de}" +testLoggerAdapter verbose: server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: 2024-01-08T00:31:03.246Z [WARN] server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: heartbeat timeout reached, starting election: last-leader=ba02d1da-3554-f86d-33cd-a66d1e967adb + testing.go:149: server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: 2024-01-08T00:31:03.246Z [WARN] server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: heartbeat timeout reached, starting election: last-leader=ba02d1da-3554-f86d-33cd-a66d1e967adb +testLoggerAdapter verbose: server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: 2024-01-08T00:31:03.246Z [INFO] server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: entering candidate state: node="Node at 5ea0aa5b-efe8-e78c-befc-caf4b98dfd61 [Candidate]" term=3 + testing.go:149: server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: 2024-01-08T00:31:03.246Z [INFO] server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: entering candidate state: node="Node at 5ea0aa5b-efe8-e78c-befc-caf4b98dfd61 [Candidate]" term=3 +testLoggerAdapter verbose: server-c8916d4d-c692-c08c-342f-b49dac6d77de: 2024-01-08T00:31:03.246Z [WARN] server-c8916d4d-c692-c08c-342f-b49dac6d77de: rejecting vote request since we have a leader: from=5ea0aa5b-efe8-e78c-befc-caf4b98dfd61 leader=ba02d1da-3554-f86d-33cd-a66d1e967adb + testing.go:149: server-c8916d4d-c692-c08c-342f-b49dac6d77de: 2024-01-08T00:31:03.246Z [WARN] server-c8916d4d-c692-c08c-342f-b49dac6d77de: rejecting vote request since we have a leader: from=5ea0aa5b-efe8-e78c-befc-caf4b98dfd61 leader=ba02d1da-3554-f86d-33cd-a66d1e967adb +testLoggerAdapter verbose: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:03.279Z [WARN] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: not part of stable configuration, aborting election + testing.go:149: server-ba02d1da-3554-f86d-33cd-a66d1e967adb: 2024-01-08T00:31:03.279Z [WARN] server-ba02d1da-3554-f86d-33cd-a66d1e967adb: not part of stable configuration, aborting election +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:03.300Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:03.300Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-c8916d4d-c692-c08c-342f-b49dac6d77de: 2024-01-08T00:31:03.311Z [WARN] server-c8916d4d-c692-c08c-342f-b49dac6d77de: heartbeat timeout reached, starting election: last-leader=ba02d1da-3554-f86d-33cd-a66d1e967adb + testing.go:149: server-c8916d4d-c692-c08c-342f-b49dac6d77de: 2024-01-08T00:31:03.311Z [WARN] server-c8916d4d-c692-c08c-342f-b49dac6d77de: heartbeat timeout reached, starting election: last-leader=ba02d1da-3554-f86d-33cd-a66d1e967adb +testLoggerAdapter verbose: server-c8916d4d-c692-c08c-342f-b49dac6d77de: 2024-01-08T00:31:03.311Z [INFO] server-c8916d4d-c692-c08c-342f-b49dac6d77de: entering candidate state: node="Node at c8916d4d-c692-c08c-342f-b49dac6d77de [Candidate]" term=3 + testing.go:149: server-c8916d4d-c692-c08c-342f-b49dac6d77de: 2024-01-08T00:31:03.311Z [INFO] server-c8916d4d-c692-c08c-342f-b49dac6d77de: entering candidate state: node="Node at c8916d4d-c692-c08c-342f-b49dac6d77de [Candidate]" term=3 +testLoggerAdapter verbose: server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: 2024-01-08T00:31:03.311Z [INFO] server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: duplicate requestVote for same term: term=3 + testing.go:149: server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: 2024-01-08T00:31:03.311Z [INFO] server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: duplicate requestVote for same term: term=3 +testLoggerAdapter verbose: server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: 2024-01-08T00:31:03.333Z [WARN] server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: Election timeout reached, restarting election + testing.go:149: server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: 2024-01-08T00:31:03.333Z [WARN] server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: Election timeout reached, restarting election +testLoggerAdapter verbose: server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: 2024-01-08T00:31:03.333Z [INFO] server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: entering candidate state: node="Node at 5ea0aa5b-efe8-e78c-befc-caf4b98dfd61 [Candidate]" term=4 + testing.go:149: server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: 2024-01-08T00:31:03.333Z [INFO] server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: entering candidate state: node="Node at 5ea0aa5b-efe8-e78c-befc-caf4b98dfd61 [Candidate]" term=4 +testLoggerAdapter verbose: server-c8916d4d-c692-c08c-342f-b49dac6d77de: 2024-01-08T00:31:03.333Z [INFO] server-c8916d4d-c692-c08c-342f-b49dac6d77de: entering follower state: follower="Node at c8916d4d-c692-c08c-342f-b49dac6d77de [Follower]" leader= + testing.go:149: server-c8916d4d-c692-c08c-342f-b49dac6d77de: 2024-01-08T00:31:03.333Z [INFO] server-c8916d4d-c692-c08c-342f-b49dac6d77de: entering follower state: follower="Node at c8916d4d-c692-c08c-342f-b49dac6d77de [Follower]" leader= +testLoggerAdapter verbose: server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: 2024-01-08T00:31:03.333Z [INFO] server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: election won: tally=2 + testing.go:149: server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: 2024-01-08T00:31:03.333Z [INFO] server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: election won: tally=2 +testLoggerAdapter verbose: server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: 2024-01-08T00:31:03.333Z [INFO] server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: entering leader state: leader="Node at 5ea0aa5b-efe8-e78c-befc-caf4b98dfd61 [Leader]" + testing.go:149: server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: 2024-01-08T00:31:03.333Z [INFO] server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: entering leader state: leader="Node at 5ea0aa5b-efe8-e78c-befc-caf4b98dfd61 [Leader]" +testLoggerAdapter verbose: server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: 2024-01-08T00:31:03.333Z [INFO] server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: added peer, starting replication: peer=server-c8916d4d-c692-c08c-342f-b49dac6d77de + testing.go:149: server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: 2024-01-08T00:31:03.333Z [INFO] server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: added peer, starting replication: peer=server-c8916d4d-c692-c08c-342f-b49dac6d77de +testLoggerAdapter verbose: server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: 2024-01-08T00:31:03.333Z [INFO] server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: pipelining replication: peer="{Voter server-c8916d4d-c692-c08c-342f-b49dac6d77de c8916d4d-c692-c08c-342f-b49dac6d77de}" + testing.go:149: server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: 2024-01-08T00:31:03.333Z [INFO] server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: pipelining replication: peer="{Voter server-c8916d4d-c692-c08c-342f-b49dac6d77de c8916d4d-c692-c08c-342f-b49dac6d77de}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:03.439Z [INFO] stable state for Leader reached at 2024-01-08 00:31:03.333891048 +0000 UTC m=+9.380780083 (1 nodes), 33.229502ms from start of poll, 422.980716ms from cluster start. Timeout at 2024-01-08 00:31:03.43926069 +0000 UTC m=+9.486149748, 105.369665ms after stability + testing.go:149: cluster: 2024-01-08T00:31:03.439Z [INFO] stable state for Leader reached at 2024-01-08 00:31:03.333891048 +0000 UTC m=+9.380780083 (1 nodes), 33.229502ms from start of poll, 422.980716ms from cluster start. Timeout at 2024-01-08 00:31:03.43926069 +0000 UTC m=+9.486149748, 105.369665ms after stability +testLoggerAdapter verbose: server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: 2024-01-08T00:31:03.545Z [INFO] server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: aborting pipeline replication: peer="{Voter server-c8916d4d-c692-c08c-342f-b49dac6d77de c8916d4d-c692-c08c-342f-b49dac6d77de}" + testing.go:149: server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: 2024-01-08T00:31:03.545Z [INFO] server-5ea0aa5b-efe8-e78c-befc-caf4b98dfd61: aborting pipeline replication: peer="{Voter server-c8916d4d-c692-c08c-342f-b49dac6d77de c8916d4d-c692-c08c-342f-b49dac6d77de}" +--- PASS: TestRaft_RemoveLeader_NoShutdown (0.64s) === RUN TestRaft_RemoveFollower_SplitCluster -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.561Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-11d28c8d-42bd-914e-6e33-db5fae092536 Address:11d28c8d-42bd-914e-6e33-db5fae092536} {Suffrage:Voter ID:server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 Address:f9713e63-dadc-5a41-10b5-a8cb7d3e02c2} {Suffrage:Voter ID:server-bb88d534-4229-922a-decb-be8f58a65ec8 Address:bb88d534-4229-922a-decb-be8f58a65ec8} {Suffrage:Voter ID:server-73d42892-293a-5762-0208-8b9d8e465179 Address:73d42892-293a-5762-0208-8b9d8e465179}]" - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.561Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-11d28c8d-42bd-914e-6e33-db5fae092536 Address:11d28c8d-42bd-914e-6e33-db5fae092536} {Suffrage:Voter ID:server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 Address:f9713e63-dadc-5a41-10b5-a8cb7d3e02c2} {Suffrage:Voter ID:server-bb88d534-4229-922a-decb-be8f58a65ec8 Address:bb88d534-4229-922a-decb-be8f58a65ec8} {Suffrage:Voter ID:server-73d42892-293a-5762-0208-8b9d8e465179 Address:73d42892-293a-5762-0208-8b9d8e465179}]" -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.561Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: entering follower state: follower="Node at 11d28c8d-42bd-914e-6e33-db5fae092536 [Follower]" leader= - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.561Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: entering follower state: follower="Node at 11d28c8d-42bd-914e-6e33-db5fae092536 [Follower]" leader= -testLoggerAdapter verbose: server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2: 2025-02-09T06:52:28.561Z [INFO] server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-11d28c8d-42bd-914e-6e33-db5fae092536 Address:11d28c8d-42bd-914e-6e33-db5fae092536} {Suffrage:Voter ID:server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 Address:f9713e63-dadc-5a41-10b5-a8cb7d3e02c2} {Suffrage:Voter ID:server-bb88d534-4229-922a-decb-be8f58a65ec8 Address:bb88d534-4229-922a-decb-be8f58a65ec8} {Suffrage:Voter ID:server-73d42892-293a-5762-0208-8b9d8e465179 Address:73d42892-293a-5762-0208-8b9d8e465179}]" - testing.go:149: server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2: 2025-02-09T06:52:28.561Z [INFO] server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-11d28c8d-42bd-914e-6e33-db5fae092536 Address:11d28c8d-42bd-914e-6e33-db5fae092536} {Suffrage:Voter ID:server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 Address:f9713e63-dadc-5a41-10b5-a8cb7d3e02c2} {Suffrage:Voter ID:server-bb88d534-4229-922a-decb-be8f58a65ec8 Address:bb88d534-4229-922a-decb-be8f58a65ec8} {Suffrage:Voter ID:server-73d42892-293a-5762-0208-8b9d8e465179 Address:73d42892-293a-5762-0208-8b9d8e465179}]" -testLoggerAdapter verbose: server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2: 2025-02-09T06:52:28.561Z [INFO] server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2: entering follower state: follower="Node at f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 [Follower]" leader= - testing.go:149: server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2: 2025-02-09T06:52:28.561Z [INFO] server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2: entering follower state: follower="Node at f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 [Follower]" leader= -testLoggerAdapter verbose: server-bb88d534-4229-922a-decb-be8f58a65ec8: 2025-02-09T06:52:28.561Z [INFO] server-bb88d534-4229-922a-decb-be8f58a65ec8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-11d28c8d-42bd-914e-6e33-db5fae092536 Address:11d28c8d-42bd-914e-6e33-db5fae092536} {Suffrage:Voter ID:server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 Address:f9713e63-dadc-5a41-10b5-a8cb7d3e02c2} {Suffrage:Voter ID:server-bb88d534-4229-922a-decb-be8f58a65ec8 Address:bb88d534-4229-922a-decb-be8f58a65ec8} {Suffrage:Voter ID:server-73d42892-293a-5762-0208-8b9d8e465179 Address:73d42892-293a-5762-0208-8b9d8e465179}]" - testing.go:149: server-bb88d534-4229-922a-decb-be8f58a65ec8: 2025-02-09T06:52:28.561Z [INFO] server-bb88d534-4229-922a-decb-be8f58a65ec8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-11d28c8d-42bd-914e-6e33-db5fae092536 Address:11d28c8d-42bd-914e-6e33-db5fae092536} {Suffrage:Voter ID:server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 Address:f9713e63-dadc-5a41-10b5-a8cb7d3e02c2} {Suffrage:Voter ID:server-bb88d534-4229-922a-decb-be8f58a65ec8 Address:bb88d534-4229-922a-decb-be8f58a65ec8} {Suffrage:Voter ID:server-73d42892-293a-5762-0208-8b9d8e465179 Address:73d42892-293a-5762-0208-8b9d8e465179}]" -testLoggerAdapter verbose: server-bb88d534-4229-922a-decb-be8f58a65ec8: 2025-02-09T06:52:28.561Z [INFO] server-bb88d534-4229-922a-decb-be8f58a65ec8: entering follower state: follower="Node at bb88d534-4229-922a-decb-be8f58a65ec8 [Follower]" leader= - testing.go:149: server-bb88d534-4229-922a-decb-be8f58a65ec8: 2025-02-09T06:52:28.561Z [INFO] server-bb88d534-4229-922a-decb-be8f58a65ec8: entering follower state: follower="Node at bb88d534-4229-922a-decb-be8f58a65ec8 [Follower]" leader= -testLoggerAdapter verbose: server-73d42892-293a-5762-0208-8b9d8e465179: 2025-02-09T06:52:28.561Z [INFO] server-73d42892-293a-5762-0208-8b9d8e465179: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-11d28c8d-42bd-914e-6e33-db5fae092536 Address:11d28c8d-42bd-914e-6e33-db5fae092536} {Suffrage:Voter ID:server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 Address:f9713e63-dadc-5a41-10b5-a8cb7d3e02c2} {Suffrage:Voter ID:server-bb88d534-4229-922a-decb-be8f58a65ec8 Address:bb88d534-4229-922a-decb-be8f58a65ec8} {Suffrage:Voter ID:server-73d42892-293a-5762-0208-8b9d8e465179 Address:73d42892-293a-5762-0208-8b9d8e465179}]" - testing.go:149: server-73d42892-293a-5762-0208-8b9d8e465179: 2025-02-09T06:52:28.561Z [INFO] server-73d42892-293a-5762-0208-8b9d8e465179: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-11d28c8d-42bd-914e-6e33-db5fae092536 Address:11d28c8d-42bd-914e-6e33-db5fae092536} {Suffrage:Voter ID:server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 Address:f9713e63-dadc-5a41-10b5-a8cb7d3e02c2} {Suffrage:Voter ID:server-bb88d534-4229-922a-decb-be8f58a65ec8 Address:bb88d534-4229-922a-decb-be8f58a65ec8} {Suffrage:Voter ID:server-73d42892-293a-5762-0208-8b9d8e465179 Address:73d42892-293a-5762-0208-8b9d8e465179}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:28.561Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:28.561Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-73d42892-293a-5762-0208-8b9d8e465179: 2025-02-09T06:52:28.561Z [INFO] server-73d42892-293a-5762-0208-8b9d8e465179: entering follower state: follower="Node at 73d42892-293a-5762-0208-8b9d8e465179 [Follower]" leader= - testing.go:149: server-73d42892-293a-5762-0208-8b9d8e465179: 2025-02-09T06:52:28.561Z [INFO] server-73d42892-293a-5762-0208-8b9d8e465179: entering follower state: follower="Node at 73d42892-293a-5762-0208-8b9d8e465179 [Follower]" leader= -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.624Z [WARN] server-11d28c8d-42bd-914e-6e33-db5fae092536: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.624Z [WARN] server-11d28c8d-42bd-914e-6e33-db5fae092536: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.624Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: entering candidate state: node="Node at 11d28c8d-42bd-914e-6e33-db5fae092536 [Candidate]" term=2 - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.624Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: entering candidate state: node="Node at 11d28c8d-42bd-914e-6e33-db5fae092536 [Candidate]" term=2 -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.624Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: election won: tally=3 - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.624Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: election won: tally=3 -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.624Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: entering leader state: leader="Node at 11d28c8d-42bd-914e-6e33-db5fae092536 [Leader]" - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.624Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: entering leader state: leader="Node at 11d28c8d-42bd-914e-6e33-db5fae092536 [Leader]" -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.624Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: added peer, starting replication: peer=server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.624Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: added peer, starting replication: peer=server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.624Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: added peer, starting replication: peer=server-bb88d534-4229-922a-decb-be8f58a65ec8 - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.624Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: added peer, starting replication: peer=server-bb88d534-4229-922a-decb-be8f58a65ec8 -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.624Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: added peer, starting replication: peer=server-73d42892-293a-5762-0208-8b9d8e465179 - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.624Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: added peer, starting replication: peer=server-73d42892-293a-5762-0208-8b9d8e465179 -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.624Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: pipelining replication: peer="{Voter server-73d42892-293a-5762-0208-8b9d8e465179 73d42892-293a-5762-0208-8b9d8e465179}" - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.624Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: pipelining replication: peer="{Voter server-73d42892-293a-5762-0208-8b9d8e465179 73d42892-293a-5762-0208-8b9d8e465179}" -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.624Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: pipelining replication: peer="{Voter server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 f9713e63-dadc-5a41-10b5-a8cb7d3e02c2}" - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.624Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: pipelining replication: peer="{Voter server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 f9713e63-dadc-5a41-10b5-a8cb7d3e02c2}" -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.624Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: pipelining replication: peer="{Voter server-bb88d534-4229-922a-decb-be8f58a65ec8 bb88d534-4229-922a-decb-be8f58a65ec8}" - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.624Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: pipelining replication: peer="{Voter server-bb88d534-4229-922a-decb-be8f58a65ec8 bb88d534-4229-922a-decb-be8f58a65ec8}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:28.729Z [INFO] stable state for Leader reached at 2025-02-09 06:52:28.62439515 +0000 UTC m=+9.603203925 (1 nodes), 62.875896ms from start of poll, 63.319865ms from cluster start. Timeout at 2025-02-09 06:52:28.729943959 +0000 UTC m=+9.708752730, 105.548805ms after stability - testing.go:149: cluster: 2025-02-09T06:52:28.729Z [INFO] stable state for Leader reached at 2025-02-09 06:52:28.62439515 +0000 UTC m=+9.603203925 (1 nodes), 62.875896ms from start of poll, 63.319865ms from cluster start. Timeout at 2025-02-09 06:52:28.729943959 +0000 UTC m=+9.708752730, 105.548805ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:28.835Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T06:52:28.835Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:28.940Z [INFO] stable state for Follower reached at 2025-02-09 06:52:28.835471061 +0000 UTC m=+9.814279827 (3 nodes), 908ns from start of poll, 274.395767ms from cluster start. Timeout at 2025-02-09 06:52:28.94065374 +0000 UTC m=+9.919462497, 105.18267ms after stability - testing.go:149: cluster: 2025-02-09T06:52:28.940Z [INFO] stable state for Follower reached at 2025-02-09 06:52:28.835471061 +0000 UTC m=+9.814279827 (3 nodes), 908ns from start of poll, 274.395767ms from cluster start. Timeout at 2025-02-09 06:52:28.94065374 +0000 UTC m=+9.919462497, 105.18267ms after stability -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.940Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: updating configuration: command=RemoveServer server-id=server-73d42892-293a-5762-0208-8b9d8e465179 server-addr= servers="[{Suffrage:Voter ID:server-11d28c8d-42bd-914e-6e33-db5fae092536 Address:11d28c8d-42bd-914e-6e33-db5fae092536} {Suffrage:Voter ID:server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 Address:f9713e63-dadc-5a41-10b5-a8cb7d3e02c2} {Suffrage:Voter ID:server-bb88d534-4229-922a-decb-be8f58a65ec8 Address:bb88d534-4229-922a-decb-be8f58a65ec8}]" - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.940Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: updating configuration: command=RemoveServer server-id=server-73d42892-293a-5762-0208-8b9d8e465179 server-addr= servers="[{Suffrage:Voter ID:server-11d28c8d-42bd-914e-6e33-db5fae092536 Address:11d28c8d-42bd-914e-6e33-db5fae092536} {Suffrage:Voter ID:server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 Address:f9713e63-dadc-5a41-10b5-a8cb7d3e02c2} {Suffrage:Voter ID:server-bb88d534-4229-922a-decb-be8f58a65ec8 Address:bb88d534-4229-922a-decb-be8f58a65ec8}]" -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.940Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: removed peer, stopping replication: peer=server-73d42892-293a-5762-0208-8b9d8e465179 last-index=3 - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.940Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: removed peer, stopping replication: peer=server-73d42892-293a-5762-0208-8b9d8e465179 last-index=3 -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.940Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to pipeline appendEntries: peer="{Voter server-bb88d534-4229-922a-decb-be8f58a65ec8 bb88d534-4229-922a-decb-be8f58a65ec8}" error="append pipeline closed" - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.940Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to pipeline appendEntries: peer="{Voter server-bb88d534-4229-922a-decb-be8f58a65ec8 bb88d534-4229-922a-decb-be8f58a65ec8}" error="append pipeline closed" -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.940Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: aborting pipeline replication: peer="{Voter server-bb88d534-4229-922a-decb-be8f58a65ec8 bb88d534-4229-922a-decb-be8f58a65ec8}" - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.940Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: aborting pipeline replication: peer="{Voter server-bb88d534-4229-922a-decb-be8f58a65ec8 bb88d534-4229-922a-decb-be8f58a65ec8}" -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.940Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to pipeline appendEntries: peer="{Voter server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 f9713e63-dadc-5a41-10b5-a8cb7d3e02c2}" error="append pipeline closed" - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.940Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to pipeline appendEntries: peer="{Voter server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 f9713e63-dadc-5a41-10b5-a8cb7d3e02c2}" error="append pipeline closed" -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.940Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: aborting pipeline replication: peer="{Voter server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 f9713e63-dadc-5a41-10b5-a8cb7d3e02c2}" - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.940Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: aborting pipeline replication: peer="{Voter server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 f9713e63-dadc-5a41-10b5-a8cb7d3e02c2}" -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.940Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: aborting pipeline replication: peer="{Voter server-73d42892-293a-5762-0208-8b9d8e465179 73d42892-293a-5762-0208-8b9d8e465179}" - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.940Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: aborting pipeline replication: peer="{Voter server-73d42892-293a-5762-0208-8b9d8e465179 73d42892-293a-5762-0208-8b9d8e465179}" -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.945Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to heartbeat to: peer=f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 error="failed to connect to peer: f9713e63-dadc-5a41-10b5-a8cb7d3e02c2" - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.945Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to heartbeat to: peer=f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 error="failed to connect to peer: f9713e63-dadc-5a41-10b5-a8cb7d3e02c2" -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.945Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to heartbeat to: peer=bb88d534-4229-922a-decb-be8f58a65ec8 error="failed to connect to peer: bb88d534-4229-922a-decb-be8f58a65ec8" - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.945Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to heartbeat to: peer=bb88d534-4229-922a-decb-be8f58a65ec8 error="failed to connect to peer: bb88d534-4229-922a-decb-be8f58a65ec8" -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.950Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to appendEntries to: peer="{Voter server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 f9713e63-dadc-5a41-10b5-a8cb7d3e02c2}" error="failed to connect to peer: f9713e63-dadc-5a41-10b5-a8cb7d3e02c2" - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.950Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to appendEntries to: peer="{Voter server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 f9713e63-dadc-5a41-10b5-a8cb7d3e02c2}" error="failed to connect to peer: f9713e63-dadc-5a41-10b5-a8cb7d3e02c2" -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.950Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to appendEntries to: peer="{Voter server-bb88d534-4229-922a-decb-be8f58a65ec8 bb88d534-4229-922a-decb-be8f58a65ec8}" error="failed to connect to peer: bb88d534-4229-922a-decb-be8f58a65ec8" - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.950Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to appendEntries to: peer="{Voter server-bb88d534-4229-922a-decb-be8f58a65ec8 bb88d534-4229-922a-decb-be8f58a65ec8}" error="failed to connect to peer: bb88d534-4229-922a-decb-be8f58a65ec8" -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.962Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to heartbeat to: peer=f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 error="failed to connect to peer: f9713e63-dadc-5a41-10b5-a8cb7d3e02c2" - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.962Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to heartbeat to: peer=f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 error="failed to connect to peer: f9713e63-dadc-5a41-10b5-a8cb7d3e02c2" -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.965Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to heartbeat to: peer=bb88d534-4229-922a-decb-be8f58a65ec8 error="failed to connect to peer: bb88d534-4229-922a-decb-be8f58a65ec8" - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.965Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to heartbeat to: peer=bb88d534-4229-922a-decb-be8f58a65ec8 error="failed to connect to peer: bb88d534-4229-922a-decb-be8f58a65ec8" -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.967Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to appendEntries to: peer="{Voter server-bb88d534-4229-922a-decb-be8f58a65ec8 bb88d534-4229-922a-decb-be8f58a65ec8}" error="failed to connect to peer: bb88d534-4229-922a-decb-be8f58a65ec8" - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.967Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to appendEntries to: peer="{Voter server-bb88d534-4229-922a-decb-be8f58a65ec8 bb88d534-4229-922a-decb-be8f58a65ec8}" error="failed to connect to peer: bb88d534-4229-922a-decb-be8f58a65ec8" -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.968Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to appendEntries to: peer="{Voter server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 f9713e63-dadc-5a41-10b5-a8cb7d3e02c2}" error="failed to connect to peer: f9713e63-dadc-5a41-10b5-a8cb7d3e02c2" - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.968Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to appendEntries to: peer="{Voter server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 f9713e63-dadc-5a41-10b5-a8cb7d3e02c2}" error="failed to connect to peer: f9713e63-dadc-5a41-10b5-a8cb7d3e02c2" -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.979Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to heartbeat to: peer=f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 error="failed to connect to peer: f9713e63-dadc-5a41-10b5-a8cb7d3e02c2" - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.979Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to heartbeat to: peer=f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 error="failed to connect to peer: f9713e63-dadc-5a41-10b5-a8cb7d3e02c2" -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.985Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to heartbeat to: peer=bb88d534-4229-922a-decb-be8f58a65ec8 error="failed to connect to peer: bb88d534-4229-922a-decb-be8f58a65ec8" - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.985Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to heartbeat to: peer=bb88d534-4229-922a-decb-be8f58a65ec8 error="failed to connect to peer: bb88d534-4229-922a-decb-be8f58a65ec8" -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.985Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to appendEntries to: peer="{Voter server-bb88d534-4229-922a-decb-be8f58a65ec8 bb88d534-4229-922a-decb-be8f58a65ec8}" error="failed to connect to peer: bb88d534-4229-922a-decb-be8f58a65ec8" - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.985Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to appendEntries to: peer="{Voter server-bb88d534-4229-922a-decb-be8f58a65ec8 bb88d534-4229-922a-decb-be8f58a65ec8}" error="failed to connect to peer: bb88d534-4229-922a-decb-be8f58a65ec8" -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.985Z [WARN] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to contact: server-id=server-bb88d534-4229-922a-decb-be8f58a65ec8 time=50.464849ms - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.985Z [WARN] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to contact: server-id=server-bb88d534-4229-922a-decb-be8f58a65ec8 time=50.464849ms -testLoggerAdapter verbose: server-bb88d534-4229-922a-decb-be8f58a65ec8: 2025-02-09T06:52:28.987Z [WARN] server-bb88d534-4229-922a-decb-be8f58a65ec8: heartbeat timeout reached, starting election: last-leader=11d28c8d-42bd-914e-6e33-db5fae092536 - testing.go:149: server-bb88d534-4229-922a-decb-be8f58a65ec8: 2025-02-09T06:52:28.987Z [WARN] server-bb88d534-4229-922a-decb-be8f58a65ec8: heartbeat timeout reached, starting election: last-leader=11d28c8d-42bd-914e-6e33-db5fae092536 -testLoggerAdapter verbose: server-bb88d534-4229-922a-decb-be8f58a65ec8: 2025-02-09T06:52:28.987Z [INFO] server-bb88d534-4229-922a-decb-be8f58a65ec8: entering candidate state: node="Node at bb88d534-4229-922a-decb-be8f58a65ec8 [Candidate]" term=3 - testing.go:149: server-bb88d534-4229-922a-decb-be8f58a65ec8: 2025-02-09T06:52:28.987Z [INFO] server-bb88d534-4229-922a-decb-be8f58a65ec8: entering candidate state: node="Node at bb88d534-4229-922a-decb-be8f58a65ec8 [Candidate]" term=3 -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.987Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to appendEntries to: peer="{Voter server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 f9713e63-dadc-5a41-10b5-a8cb7d3e02c2}" error="failed to connect to peer: f9713e63-dadc-5a41-10b5-a8cb7d3e02c2" - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.987Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to appendEntries to: peer="{Voter server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 f9713e63-dadc-5a41-10b5-a8cb7d3e02c2}" error="failed to connect to peer: f9713e63-dadc-5a41-10b5-a8cb7d3e02c2" -testLoggerAdapter verbose: server-bb88d534-4229-922a-decb-be8f58a65ec8: 2025-02-09T06:52:28.988Z [ERROR] server-bb88d534-4229-922a-decb-be8f58a65ec8: failed to make requestVote RPC: target="{Voter server-73d42892-293a-5762-0208-8b9d8e465179 73d42892-293a-5762-0208-8b9d8e465179}" error="failed to connect to peer: 73d42892-293a-5762-0208-8b9d8e465179" - testing.go:149: server-bb88d534-4229-922a-decb-be8f58a65ec8: 2025-02-09T06:52:28.988Z [ERROR] server-bb88d534-4229-922a-decb-be8f58a65ec8: failed to make requestVote RPC: target="{Voter server-73d42892-293a-5762-0208-8b9d8e465179 73d42892-293a-5762-0208-8b9d8e465179}" error="failed to connect to peer: 73d42892-293a-5762-0208-8b9d8e465179" -testLoggerAdapter verbose: server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2: 2025-02-09T06:52:28.988Z [WARN] server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2: rejecting vote request since we have a leader: from=bb88d534-4229-922a-decb-be8f58a65ec8 leader=11d28c8d-42bd-914e-6e33-db5fae092536 -testLoggerAdapter verbose: server-bb88d534-4229-922a-decb-be8f58a65ec8: 2025-02-09T06:52:28.988Z [ERROR] server-bb88d534-4229-922a-decb-be8f58a65ec8: failed to make requestVote RPC: target="{Voter server-11d28c8d-42bd-914e-6e33-db5fae092536 11d28c8d-42bd-914e-6e33-db5fae092536}" error="failed to connect to peer: 11d28c8d-42bd-914e-6e33-db5fae092536" - testing.go:149: server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2: 2025-02-09T06:52:28.988Z [WARN] server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2: rejecting vote request since we have a leader: from=bb88d534-4229-922a-decb-be8f58a65ec8 leader=11d28c8d-42bd-914e-6e33-db5fae092536 - testing.go:149: server-bb88d534-4229-922a-decb-be8f58a65ec8: 2025-02-09T06:52:28.988Z [ERROR] server-bb88d534-4229-922a-decb-be8f58a65ec8: failed to make requestVote RPC: target="{Voter server-11d28c8d-42bd-914e-6e33-db5fae092536 11d28c8d-42bd-914e-6e33-db5fae092536}" error="failed to connect to peer: 11d28c8d-42bd-914e-6e33-db5fae092536" -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.996Z [WARN] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to contact: server-id=server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 time=56.938455ms - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.996Z [WARN] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to contact: server-id=server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 time=56.938455ms -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.996Z [WARN] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to contact: server-id=server-bb88d534-4229-922a-decb-be8f58a65ec8 time=61.232702ms - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.996Z [WARN] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to contact: server-id=server-bb88d534-4229-922a-decb-be8f58a65ec8 time=61.232702ms -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.996Z [WARN] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to contact quorum of nodes, stepping down - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.996Z [WARN] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to contact quorum of nodes, stepping down -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.996Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: entering follower state: follower="Node at 11d28c8d-42bd-914e-6e33-db5fae092536 [Follower]" leader= - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.996Z [INFO] server-11d28c8d-42bd-914e-6e33-db5fae092536: entering follower state: follower="Node at 11d28c8d-42bd-914e-6e33-db5fae092536 [Follower]" leader= -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.996Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to appendEntries to: peer="{Voter server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 f9713e63-dadc-5a41-10b5-a8cb7d3e02c2}" error="failed to connect to peer: f9713e63-dadc-5a41-10b5-a8cb7d3e02c2" - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.996Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to appendEntries to: peer="{Voter server-f9713e63-dadc-5a41-10b5-a8cb7d3e02c2 f9713e63-dadc-5a41-10b5-a8cb7d3e02c2}" error="failed to connect to peer: f9713e63-dadc-5a41-10b5-a8cb7d3e02c2" -testLoggerAdapter verbose: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.996Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to appendEntries to: peer="{Voter server-bb88d534-4229-922a-decb-be8f58a65ec8 bb88d534-4229-922a-decb-be8f58a65ec8}" error="failed to connect to peer: bb88d534-4229-922a-decb-be8f58a65ec8" - testing.go:149: server-11d28c8d-42bd-914e-6e33-db5fae092536: 2025-02-09T06:52:28.996Z [ERROR] server-11d28c8d-42bd-914e-6e33-db5fae092536: failed to appendEntries to: peer="{Voter server-bb88d534-4229-922a-decb-be8f58a65ec8 bb88d534-4229-922a-decb-be8f58a65ec8}" error="failed to connect to peer: bb88d534-4229-922a-decb-be8f58a65ec8" ---- PASS: TestRaft_RemoveFollower_SplitCluster (0.44s) +testLoggerAdapter verbose: server-1a498f52-a6b7-3194-596f-5b8daeed77c9: 2024-01-08T00:31:03.546Z [INFO] server-1a498f52-a6b7-3194-596f-5b8daeed77c9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1a498f52-a6b7-3194-596f-5b8daeed77c9 Address:1a498f52-a6b7-3194-596f-5b8daeed77c9} {Suffrage:Voter ID:server-04abdece-8234-ac33-4ead-679b17f7c194 Address:04abdece-8234-ac33-4ead-679b17f7c194} {Suffrage:Voter ID:server-e5cb02de-0e10-6cc3-f95e-83e39bfbe826 Address:e5cb02de-0e10-6cc3-f95e-83e39bfbe826} {Suffrage:Voter ID:server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e Address:4d7088ed-c8b2-7193-7741-0cfc60f25d7e}]" + testing.go:149: server-1a498f52-a6b7-3194-596f-5b8daeed77c9: 2024-01-08T00:31:03.546Z [INFO] server-1a498f52-a6b7-3194-596f-5b8daeed77c9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1a498f52-a6b7-3194-596f-5b8daeed77c9 Address:1a498f52-a6b7-3194-596f-5b8daeed77c9} {Suffrage:Voter ID:server-04abdece-8234-ac33-4ead-679b17f7c194 Address:04abdece-8234-ac33-4ead-679b17f7c194} {Suffrage:Voter ID:server-e5cb02de-0e10-6cc3-f95e-83e39bfbe826 Address:e5cb02de-0e10-6cc3-f95e-83e39bfbe826} {Suffrage:Voter ID:server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e Address:4d7088ed-c8b2-7193-7741-0cfc60f25d7e}]" +testLoggerAdapter verbose: server-04abdece-8234-ac33-4ead-679b17f7c194: 2024-01-08T00:31:03.546Z [INFO] server-04abdece-8234-ac33-4ead-679b17f7c194: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1a498f52-a6b7-3194-596f-5b8daeed77c9 Address:1a498f52-a6b7-3194-596f-5b8daeed77c9} {Suffrage:Voter ID:server-04abdece-8234-ac33-4ead-679b17f7c194 Address:04abdece-8234-ac33-4ead-679b17f7c194} {Suffrage:Voter ID:server-e5cb02de-0e10-6cc3-f95e-83e39bfbe826 Address:e5cb02de-0e10-6cc3-f95e-83e39bfbe826} {Suffrage:Voter ID:server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e Address:4d7088ed-c8b2-7193-7741-0cfc60f25d7e}]" + testing.go:149: server-04abdece-8234-ac33-4ead-679b17f7c194: 2024-01-08T00:31:03.546Z [INFO] server-04abdece-8234-ac33-4ead-679b17f7c194: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1a498f52-a6b7-3194-596f-5b8daeed77c9 Address:1a498f52-a6b7-3194-596f-5b8daeed77c9} {Suffrage:Voter ID:server-04abdece-8234-ac33-4ead-679b17f7c194 Address:04abdece-8234-ac33-4ead-679b17f7c194} {Suffrage:Voter ID:server-e5cb02de-0e10-6cc3-f95e-83e39bfbe826 Address:e5cb02de-0e10-6cc3-f95e-83e39bfbe826} {Suffrage:Voter ID:server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e Address:4d7088ed-c8b2-7193-7741-0cfc60f25d7e}]" +testLoggerAdapter verbose: server-e5cb02de-0e10-6cc3-f95e-83e39bfbe826: 2024-01-08T00:31:03.546Z [INFO] server-e5cb02de-0e10-6cc3-f95e-83e39bfbe826: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1a498f52-a6b7-3194-596f-5b8daeed77c9 Address:1a498f52-a6b7-3194-596f-5b8daeed77c9} {Suffrage:Voter ID:server-04abdece-8234-ac33-4ead-679b17f7c194 Address:04abdece-8234-ac33-4ead-679b17f7c194} {Suffrage:Voter ID:server-e5cb02de-0e10-6cc3-f95e-83e39bfbe826 Address:e5cb02de-0e10-6cc3-f95e-83e39bfbe826} {Suffrage:Voter ID:server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e Address:4d7088ed-c8b2-7193-7741-0cfc60f25d7e}]" + testing.go:149: server-e5cb02de-0e10-6cc3-f95e-83e39bfbe826: 2024-01-08T00:31:03.546Z [INFO] server-e5cb02de-0e10-6cc3-f95e-83e39bfbe826: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1a498f52-a6b7-3194-596f-5b8daeed77c9 Address:1a498f52-a6b7-3194-596f-5b8daeed77c9} {Suffrage:Voter ID:server-04abdece-8234-ac33-4ead-679b17f7c194 Address:04abdece-8234-ac33-4ead-679b17f7c194} {Suffrage:Voter ID:server-e5cb02de-0e10-6cc3-f95e-83e39bfbe826 Address:e5cb02de-0e10-6cc3-f95e-83e39bfbe826} {Suffrage:Voter ID:server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e Address:4d7088ed-c8b2-7193-7741-0cfc60f25d7e}]" +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.546Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1a498f52-a6b7-3194-596f-5b8daeed77c9 Address:1a498f52-a6b7-3194-596f-5b8daeed77c9} {Suffrage:Voter ID:server-04abdece-8234-ac33-4ead-679b17f7c194 Address:04abdece-8234-ac33-4ead-679b17f7c194} {Suffrage:Voter ID:server-e5cb02de-0e10-6cc3-f95e-83e39bfbe826 Address:e5cb02de-0e10-6cc3-f95e-83e39bfbe826} {Suffrage:Voter ID:server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e Address:4d7088ed-c8b2-7193-7741-0cfc60f25d7e}]" + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.546Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1a498f52-a6b7-3194-596f-5b8daeed77c9 Address:1a498f52-a6b7-3194-596f-5b8daeed77c9} {Suffrage:Voter ID:server-04abdece-8234-ac33-4ead-679b17f7c194 Address:04abdece-8234-ac33-4ead-679b17f7c194} {Suffrage:Voter ID:server-e5cb02de-0e10-6cc3-f95e-83e39bfbe826 Address:e5cb02de-0e10-6cc3-f95e-83e39bfbe826} {Suffrage:Voter ID:server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e Address:4d7088ed-c8b2-7193-7741-0cfc60f25d7e}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:03.546Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:03.546Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-1a498f52-a6b7-3194-596f-5b8daeed77c9: 2024-01-08T00:31:03.547Z [INFO] server-1a498f52-a6b7-3194-596f-5b8daeed77c9: entering follower state: follower="Node at 1a498f52-a6b7-3194-596f-5b8daeed77c9 [Follower]" leader= + testing.go:149: server-1a498f52-a6b7-3194-596f-5b8daeed77c9: 2024-01-08T00:31:03.547Z [INFO] server-1a498f52-a6b7-3194-596f-5b8daeed77c9: entering follower state: follower="Node at 1a498f52-a6b7-3194-596f-5b8daeed77c9 [Follower]" leader= +testLoggerAdapter verbose: server-04abdece-8234-ac33-4ead-679b17f7c194: 2024-01-08T00:31:03.547Z [INFO] server-04abdece-8234-ac33-4ead-679b17f7c194: entering follower state: follower="Node at 04abdece-8234-ac33-4ead-679b17f7c194 [Follower]" leader= + testing.go:149: server-04abdece-8234-ac33-4ead-679b17f7c194: 2024-01-08T00:31:03.547Z [INFO] server-04abdece-8234-ac33-4ead-679b17f7c194: entering follower state: follower="Node at 04abdece-8234-ac33-4ead-679b17f7c194 [Follower]" leader= +testLoggerAdapter verbose: server-e5cb02de-0e10-6cc3-f95e-83e39bfbe826: 2024-01-08T00:31:03.547Z [INFO] server-e5cb02de-0e10-6cc3-f95e-83e39bfbe826: entering follower state: follower="Node at e5cb02de-0e10-6cc3-f95e-83e39bfbe826 [Follower]" leader= + testing.go:149: server-e5cb02de-0e10-6cc3-f95e-83e39bfbe826: 2024-01-08T00:31:03.547Z [INFO] server-e5cb02de-0e10-6cc3-f95e-83e39bfbe826: entering follower state: follower="Node at e5cb02de-0e10-6cc3-f95e-83e39bfbe826 [Follower]" leader= +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.547Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: entering follower state: follower="Node at 4d7088ed-c8b2-7193-7741-0cfc60f25d7e [Follower]" leader= + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.547Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: entering follower state: follower="Node at 4d7088ed-c8b2-7193-7741-0cfc60f25d7e [Follower]" leader= +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.599Z [WARN] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.599Z [WARN] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.599Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: entering candidate state: node="Node at 4d7088ed-c8b2-7193-7741-0cfc60f25d7e [Candidate]" term=2 + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.599Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: entering candidate state: node="Node at 4d7088ed-c8b2-7193-7741-0cfc60f25d7e [Candidate]" term=2 +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.599Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: election won: tally=3 + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.599Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: election won: tally=3 +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.599Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: entering leader state: leader="Node at 4d7088ed-c8b2-7193-7741-0cfc60f25d7e [Leader]" + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.599Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: entering leader state: leader="Node at 4d7088ed-c8b2-7193-7741-0cfc60f25d7e [Leader]" +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.599Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: added peer, starting replication: peer=server-1a498f52-a6b7-3194-596f-5b8daeed77c9 + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.599Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: added peer, starting replication: peer=server-1a498f52-a6b7-3194-596f-5b8daeed77c9 +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.599Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: added peer, starting replication: peer=server-04abdece-8234-ac33-4ead-679b17f7c194 + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.599Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: added peer, starting replication: peer=server-04abdece-8234-ac33-4ead-679b17f7c194 +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.599Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: added peer, starting replication: peer=server-e5cb02de-0e10-6cc3-f95e-83e39bfbe826 + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.599Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: added peer, starting replication: peer=server-e5cb02de-0e10-6cc3-f95e-83e39bfbe826 +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.599Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: pipelining replication: peer="{Voter server-e5cb02de-0e10-6cc3-f95e-83e39bfbe826 e5cb02de-0e10-6cc3-f95e-83e39bfbe826}" + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.599Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: pipelining replication: peer="{Voter server-e5cb02de-0e10-6cc3-f95e-83e39bfbe826 e5cb02de-0e10-6cc3-f95e-83e39bfbe826}" +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.600Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: pipelining replication: peer="{Voter server-1a498f52-a6b7-3194-596f-5b8daeed77c9 1a498f52-a6b7-3194-596f-5b8daeed77c9}" + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.600Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: pipelining replication: peer="{Voter server-1a498f52-a6b7-3194-596f-5b8daeed77c9 1a498f52-a6b7-3194-596f-5b8daeed77c9}" +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.600Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: pipelining replication: peer="{Voter server-04abdece-8234-ac33-4ead-679b17f7c194 04abdece-8234-ac33-4ead-679b17f7c194}" + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.600Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: pipelining replication: peer="{Voter server-04abdece-8234-ac33-4ead-679b17f7c194 04abdece-8234-ac33-4ead-679b17f7c194}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:03.705Z [INFO] stable state for Leader reached at 2024-01-08 00:31:03.600051891 +0000 UTC m=+9.646940930 (1 nodes), 53.052013ms from start of poll, 53.554535ms from cluster start. Timeout at 2024-01-08 00:31:03.705269581 +0000 UTC m=+9.752158732, 105.217802ms after stability + testing.go:149: cluster: 2024-01-08T00:31:03.705Z [INFO] stable state for Leader reached at 2024-01-08 00:31:03.600051891 +0000 UTC m=+9.646940930 (1 nodes), 53.052013ms from start of poll, 53.554535ms from cluster start. Timeout at 2024-01-08 00:31:03.705269581 +0000 UTC m=+9.752158732, 105.217802ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:03.811Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T00:31:03.811Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:03.917Z [INFO] stable state for Follower reached at 2024-01-08 00:31:03.811250756 +0000 UTC m=+9.858139854 (3 nodes), 2.563µs from start of poll, 264.753459ms from cluster start. Timeout at 2024-01-08 00:31:03.916979389 +0000 UTC m=+9.963868510, 105.728656ms after stability + testing.go:149: cluster: 2024-01-08T00:31:03.917Z [INFO] stable state for Follower reached at 2024-01-08 00:31:03.811250756 +0000 UTC m=+9.858139854 (3 nodes), 2.563µs from start of poll, 264.753459ms from cluster start. Timeout at 2024-01-08 00:31:03.916979389 +0000 UTC m=+9.963868510, 105.728656ms after stability +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.917Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: updating configuration: command=RemoveServer server-id=server-e5cb02de-0e10-6cc3-f95e-83e39bfbe826 server-addr= servers="[{Suffrage:Voter ID:server-1a498f52-a6b7-3194-596f-5b8daeed77c9 Address:1a498f52-a6b7-3194-596f-5b8daeed77c9} {Suffrage:Voter ID:server-04abdece-8234-ac33-4ead-679b17f7c194 Address:04abdece-8234-ac33-4ead-679b17f7c194} {Suffrage:Voter ID:server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e Address:4d7088ed-c8b2-7193-7741-0cfc60f25d7e}]" + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.917Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: updating configuration: command=RemoveServer server-id=server-e5cb02de-0e10-6cc3-f95e-83e39bfbe826 server-addr= servers="[{Suffrage:Voter ID:server-1a498f52-a6b7-3194-596f-5b8daeed77c9 Address:1a498f52-a6b7-3194-596f-5b8daeed77c9} {Suffrage:Voter ID:server-04abdece-8234-ac33-4ead-679b17f7c194 Address:04abdece-8234-ac33-4ead-679b17f7c194} {Suffrage:Voter ID:server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e Address:4d7088ed-c8b2-7193-7741-0cfc60f25d7e}]" +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.917Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: removed peer, stopping replication: peer=server-e5cb02de-0e10-6cc3-f95e-83e39bfbe826 last-index=3 + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.917Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: removed peer, stopping replication: peer=server-e5cb02de-0e10-6cc3-f95e-83e39bfbe826 last-index=3 +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.917Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: aborting pipeline replication: peer="{Voter server-e5cb02de-0e10-6cc3-f95e-83e39bfbe826 e5cb02de-0e10-6cc3-f95e-83e39bfbe826}" + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.917Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: aborting pipeline replication: peer="{Voter server-e5cb02de-0e10-6cc3-f95e-83e39bfbe826 e5cb02de-0e10-6cc3-f95e-83e39bfbe826}" +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.917Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to pipeline appendEntries: peer="{Voter server-1a498f52-a6b7-3194-596f-5b8daeed77c9 1a498f52-a6b7-3194-596f-5b8daeed77c9}" error="append pipeline closed" + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.917Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to pipeline appendEntries: peer="{Voter server-1a498f52-a6b7-3194-596f-5b8daeed77c9 1a498f52-a6b7-3194-596f-5b8daeed77c9}" error="append pipeline closed" +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.917Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: aborting pipeline replication: peer="{Voter server-1a498f52-a6b7-3194-596f-5b8daeed77c9 1a498f52-a6b7-3194-596f-5b8daeed77c9}" + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.917Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: aborting pipeline replication: peer="{Voter server-1a498f52-a6b7-3194-596f-5b8daeed77c9 1a498f52-a6b7-3194-596f-5b8daeed77c9}" +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.917Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to pipeline appendEntries: peer="{Voter server-04abdece-8234-ac33-4ead-679b17f7c194 04abdece-8234-ac33-4ead-679b17f7c194}" error="append pipeline closed" + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.917Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to pipeline appendEntries: peer="{Voter server-04abdece-8234-ac33-4ead-679b17f7c194 04abdece-8234-ac33-4ead-679b17f7c194}" error="append pipeline closed" +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.917Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: aborting pipeline replication: peer="{Voter server-04abdece-8234-ac33-4ead-679b17f7c194 04abdece-8234-ac33-4ead-679b17f7c194}" + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.917Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: aborting pipeline replication: peer="{Voter server-04abdece-8234-ac33-4ead-679b17f7c194 04abdece-8234-ac33-4ead-679b17f7c194}" +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.917Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to heartbeat to: peer=1a498f52-a6b7-3194-596f-5b8daeed77c9 error="failed to connect to peer: 1a498f52-a6b7-3194-596f-5b8daeed77c9" + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.917Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to heartbeat to: peer=1a498f52-a6b7-3194-596f-5b8daeed77c9 error="failed to connect to peer: 1a498f52-a6b7-3194-596f-5b8daeed77c9" +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.923Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to appendEntries to: peer="{Voter server-04abdece-8234-ac33-4ead-679b17f7c194 04abdece-8234-ac33-4ead-679b17f7c194}" error="failed to connect to peer: 04abdece-8234-ac33-4ead-679b17f7c194" + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.923Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to appendEntries to: peer="{Voter server-04abdece-8234-ac33-4ead-679b17f7c194 04abdece-8234-ac33-4ead-679b17f7c194}" error="failed to connect to peer: 04abdece-8234-ac33-4ead-679b17f7c194" +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.924Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to appendEntries to: peer="{Voter server-1a498f52-a6b7-3194-596f-5b8daeed77c9 1a498f52-a6b7-3194-596f-5b8daeed77c9}" error="failed to connect to peer: 1a498f52-a6b7-3194-596f-5b8daeed77c9" + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.924Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to appendEntries to: peer="{Voter server-1a498f52-a6b7-3194-596f-5b8daeed77c9 1a498f52-a6b7-3194-596f-5b8daeed77c9}" error="failed to connect to peer: 1a498f52-a6b7-3194-596f-5b8daeed77c9" +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.924Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to heartbeat to: peer=04abdece-8234-ac33-4ead-679b17f7c194 error="failed to connect to peer: 04abdece-8234-ac33-4ead-679b17f7c194" + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.924Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to heartbeat to: peer=04abdece-8234-ac33-4ead-679b17f7c194 error="failed to connect to peer: 04abdece-8234-ac33-4ead-679b17f7c194" +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.936Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to heartbeat to: peer=1a498f52-a6b7-3194-596f-5b8daeed77c9 error="failed to connect to peer: 1a498f52-a6b7-3194-596f-5b8daeed77c9" + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.936Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to heartbeat to: peer=1a498f52-a6b7-3194-596f-5b8daeed77c9 error="failed to connect to peer: 1a498f52-a6b7-3194-596f-5b8daeed77c9" +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.942Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to appendEntries to: peer="{Voter server-1a498f52-a6b7-3194-596f-5b8daeed77c9 1a498f52-a6b7-3194-596f-5b8daeed77c9}" error="failed to connect to peer: 1a498f52-a6b7-3194-596f-5b8daeed77c9" + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.942Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to appendEntries to: peer="{Voter server-1a498f52-a6b7-3194-596f-5b8daeed77c9 1a498f52-a6b7-3194-596f-5b8daeed77c9}" error="failed to connect to peer: 1a498f52-a6b7-3194-596f-5b8daeed77c9" +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.942Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to heartbeat to: peer=04abdece-8234-ac33-4ead-679b17f7c194 error="failed to connect to peer: 04abdece-8234-ac33-4ead-679b17f7c194" + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.942Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to heartbeat to: peer=04abdece-8234-ac33-4ead-679b17f7c194 error="failed to connect to peer: 04abdece-8234-ac33-4ead-679b17f7c194" +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.942Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to appendEntries to: peer="{Voter server-04abdece-8234-ac33-4ead-679b17f7c194 04abdece-8234-ac33-4ead-679b17f7c194}" error="failed to connect to peer: 04abdece-8234-ac33-4ead-679b17f7c194" + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.942Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to appendEntries to: peer="{Voter server-04abdece-8234-ac33-4ead-679b17f7c194 04abdece-8234-ac33-4ead-679b17f7c194}" error="failed to connect to peer: 04abdece-8234-ac33-4ead-679b17f7c194" +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.954Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to heartbeat to: peer=1a498f52-a6b7-3194-596f-5b8daeed77c9 error="failed to connect to peer: 1a498f52-a6b7-3194-596f-5b8daeed77c9" + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.954Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to heartbeat to: peer=1a498f52-a6b7-3194-596f-5b8daeed77c9 error="failed to connect to peer: 1a498f52-a6b7-3194-596f-5b8daeed77c9" +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.958Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to heartbeat to: peer=04abdece-8234-ac33-4ead-679b17f7c194 error="failed to connect to peer: 04abdece-8234-ac33-4ead-679b17f7c194" + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.958Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to heartbeat to: peer=04abdece-8234-ac33-4ead-679b17f7c194 error="failed to connect to peer: 04abdece-8234-ac33-4ead-679b17f7c194" +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.960Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to appendEntries to: peer="{Voter server-1a498f52-a6b7-3194-596f-5b8daeed77c9 1a498f52-a6b7-3194-596f-5b8daeed77c9}" error="failed to connect to peer: 1a498f52-a6b7-3194-596f-5b8daeed77c9" + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.960Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to appendEntries to: peer="{Voter server-1a498f52-a6b7-3194-596f-5b8daeed77c9 1a498f52-a6b7-3194-596f-5b8daeed77c9}" error="failed to connect to peer: 1a498f52-a6b7-3194-596f-5b8daeed77c9" +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.961Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to appendEntries to: peer="{Voter server-04abdece-8234-ac33-4ead-679b17f7c194 04abdece-8234-ac33-4ead-679b17f7c194}" error="failed to connect to peer: 04abdece-8234-ac33-4ead-679b17f7c194" + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.961Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to appendEntries to: peer="{Voter server-04abdece-8234-ac33-4ead-679b17f7c194 04abdece-8234-ac33-4ead-679b17f7c194}" error="failed to connect to peer: 04abdece-8234-ac33-4ead-679b17f7c194" +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.962Z [WARN] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to contact: server-id=server-1a498f52-a6b7-3194-596f-5b8daeed77c9 time=50.938364ms + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.962Z [WARN] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to contact: server-id=server-1a498f52-a6b7-3194-596f-5b8daeed77c9 time=50.938364ms +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.973Z [WARN] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to contact: server-id=server-1a498f52-a6b7-3194-596f-5b8daeed77c9 time=61.87283ms + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.973Z [WARN] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to contact: server-id=server-1a498f52-a6b7-3194-596f-5b8daeed77c9 time=61.87283ms +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.973Z [WARN] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to contact: server-id=server-04abdece-8234-ac33-4ead-679b17f7c194 time=59.624733ms + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.973Z [WARN] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to contact: server-id=server-04abdece-8234-ac33-4ead-679b17f7c194 time=59.624733ms +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.973Z [WARN] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to contact quorum of nodes, stepping down + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.973Z [WARN] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to contact quorum of nodes, stepping down +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.973Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: entering follower state: follower="Node at 4d7088ed-c8b2-7193-7741-0cfc60f25d7e [Follower]" leader= + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.973Z [INFO] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: entering follower state: follower="Node at 4d7088ed-c8b2-7193-7741-0cfc60f25d7e [Follower]" leader= +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.973Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to appendEntries to: peer="{Voter server-04abdece-8234-ac33-4ead-679b17f7c194 04abdece-8234-ac33-4ead-679b17f7c194}" error="failed to connect to peer: 04abdece-8234-ac33-4ead-679b17f7c194" + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.973Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to appendEntries to: peer="{Voter server-04abdece-8234-ac33-4ead-679b17f7c194 04abdece-8234-ac33-4ead-679b17f7c194}" error="failed to connect to peer: 04abdece-8234-ac33-4ead-679b17f7c194" +testLoggerAdapter verbose: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.973Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to appendEntries to: peer="{Voter server-1a498f52-a6b7-3194-596f-5b8daeed77c9 1a498f52-a6b7-3194-596f-5b8daeed77c9}" error="failed to connect to peer: 1a498f52-a6b7-3194-596f-5b8daeed77c9" + testing.go:149: server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: 2024-01-08T00:31:03.973Z [ERROR] server-4d7088ed-c8b2-7193-7741-0cfc60f25d7e: failed to appendEntries to: peer="{Voter server-1a498f52-a6b7-3194-596f-5b8daeed77c9 1a498f52-a6b7-3194-596f-5b8daeed77c9}" error="failed to connect to peer: 1a498f52-a6b7-3194-596f-5b8daeed77c9" +--- PASS: TestRaft_RemoveFollower_SplitCluster (0.43s) === RUN TestRaft_AddKnownPeer -testLoggerAdapter verbose: server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c: 2025-02-09T06:52:28.997Z [INFO] server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c Address:2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c} {Suffrage:Voter ID:server-fd343c0f-d0c9-813c-8b62-827445bc927e Address:fd343c0f-d0c9-813c-8b62-827445bc927e} {Suffrage:Voter ID:server-bf17a376-c094-ea84-3489-52ad4bf33ff0 Address:bf17a376-c094-ea84-3489-52ad4bf33ff0}]" - testing.go:149: server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c: 2025-02-09T06:52:28.997Z [INFO] server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c Address:2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c} {Suffrage:Voter ID:server-fd343c0f-d0c9-813c-8b62-827445bc927e Address:fd343c0f-d0c9-813c-8b62-827445bc927e} {Suffrage:Voter ID:server-bf17a376-c094-ea84-3489-52ad4bf33ff0 Address:bf17a376-c094-ea84-3489-52ad4bf33ff0}]" -testLoggerAdapter verbose: server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c: 2025-02-09T06:52:28.997Z [INFO] server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c: entering follower state: follower="Node at 2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c [Follower]" leader= - testing.go:149: server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c: 2025-02-09T06:52:28.997Z [INFO] server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c: entering follower state: follower="Node at 2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c [Follower]" leader= -testLoggerAdapter verbose: server-fd343c0f-d0c9-813c-8b62-827445bc927e: 2025-02-09T06:52:28.997Z [INFO] server-fd343c0f-d0c9-813c-8b62-827445bc927e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c Address:2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c} {Suffrage:Voter ID:server-fd343c0f-d0c9-813c-8b62-827445bc927e Address:fd343c0f-d0c9-813c-8b62-827445bc927e} {Suffrage:Voter ID:server-bf17a376-c094-ea84-3489-52ad4bf33ff0 Address:bf17a376-c094-ea84-3489-52ad4bf33ff0}]" - testing.go:149: server-fd343c0f-d0c9-813c-8b62-827445bc927e: 2025-02-09T06:52:28.997Z [INFO] server-fd343c0f-d0c9-813c-8b62-827445bc927e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c Address:2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c} {Suffrage:Voter ID:server-fd343c0f-d0c9-813c-8b62-827445bc927e Address:fd343c0f-d0c9-813c-8b62-827445bc927e} {Suffrage:Voter ID:server-bf17a376-c094-ea84-3489-52ad4bf33ff0 Address:bf17a376-c094-ea84-3489-52ad4bf33ff0}]" -testLoggerAdapter verbose: server-fd343c0f-d0c9-813c-8b62-827445bc927e: 2025-02-09T06:52:28.997Z [INFO] server-fd343c0f-d0c9-813c-8b62-827445bc927e: entering follower state: follower="Node at fd343c0f-d0c9-813c-8b62-827445bc927e [Follower]" leader= - testing.go:149: server-fd343c0f-d0c9-813c-8b62-827445bc927e: 2025-02-09T06:52:28.997Z [INFO] server-fd343c0f-d0c9-813c-8b62-827445bc927e: entering follower state: follower="Node at fd343c0f-d0c9-813c-8b62-827445bc927e [Follower]" leader= -testLoggerAdapter verbose: server-bf17a376-c094-ea84-3489-52ad4bf33ff0: 2025-02-09T06:52:28.998Z [INFO] server-bf17a376-c094-ea84-3489-52ad4bf33ff0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c Address:2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c} {Suffrage:Voter ID:server-fd343c0f-d0c9-813c-8b62-827445bc927e Address:fd343c0f-d0c9-813c-8b62-827445bc927e} {Suffrage:Voter ID:server-bf17a376-c094-ea84-3489-52ad4bf33ff0 Address:bf17a376-c094-ea84-3489-52ad4bf33ff0}]" - testing.go:149: server-bf17a376-c094-ea84-3489-52ad4bf33ff0: 2025-02-09T06:52:28.998Z [INFO] server-bf17a376-c094-ea84-3489-52ad4bf33ff0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c Address:2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c} {Suffrage:Voter ID:server-fd343c0f-d0c9-813c-8b62-827445bc927e Address:fd343c0f-d0c9-813c-8b62-827445bc927e} {Suffrage:Voter ID:server-bf17a376-c094-ea84-3489-52ad4bf33ff0 Address:bf17a376-c094-ea84-3489-52ad4bf33ff0}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:28.998Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:28.998Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-bf17a376-c094-ea84-3489-52ad4bf33ff0: 2025-02-09T06:52:28.998Z [INFO] server-bf17a376-c094-ea84-3489-52ad4bf33ff0: entering follower state: follower="Node at bf17a376-c094-ea84-3489-52ad4bf33ff0 [Follower]" leader= - testing.go:149: server-bf17a376-c094-ea84-3489-52ad4bf33ff0: 2025-02-09T06:52:28.998Z [INFO] server-bf17a376-c094-ea84-3489-52ad4bf33ff0: entering follower state: follower="Node at bf17a376-c094-ea84-3489-52ad4bf33ff0 [Follower]" leader= -testLoggerAdapter verbose: server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c: 2025-02-09T06:52:29.058Z [WARN] server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-bf17a376-c094-ea84-3489-52ad4bf33ff0: 2025-02-09T06:52:29.058Z [WARN] server-bf17a376-c094-ea84-3489-52ad4bf33ff0: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c: 2025-02-09T06:52:29.058Z [WARN] server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c: 2025-02-09T06:52:29.058Z [INFO] server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c: entering candidate state: node="Node at 2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c [Candidate]" term=2 - testing.go:149: server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c: 2025-02-09T06:52:29.058Z [INFO] server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c: entering candidate state: node="Node at 2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c [Candidate]" term=2 - testing.go:149: server-bf17a376-c094-ea84-3489-52ad4bf33ff0: 2025-02-09T06:52:29.058Z [WARN] server-bf17a376-c094-ea84-3489-52ad4bf33ff0: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-bf17a376-c094-ea84-3489-52ad4bf33ff0: 2025-02-09T06:52:29.058Z [INFO] server-bf17a376-c094-ea84-3489-52ad4bf33ff0: entering candidate state: node="Node at bf17a376-c094-ea84-3489-52ad4bf33ff0 [Candidate]" term=2 - testing.go:149: server-bf17a376-c094-ea84-3489-52ad4bf33ff0: 2025-02-09T06:52:29.058Z [INFO] server-bf17a376-c094-ea84-3489-52ad4bf33ff0: entering candidate state: node="Node at bf17a376-c094-ea84-3489-52ad4bf33ff0 [Candidate]" term=2 -testLoggerAdapter verbose: server-bf17a376-c094-ea84-3489-52ad4bf33ff0: 2025-02-09T06:52:29.058Z [INFO] server-bf17a376-c094-ea84-3489-52ad4bf33ff0: duplicate requestVote for same term: term=2 -testLoggerAdapter verbose: server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c: 2025-02-09T06:52:29.058Z [INFO] server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c: duplicate requestVote for same term: term=2 - testing.go:149: server-bf17a376-c094-ea84-3489-52ad4bf33ff0: 2025-02-09T06:52:29.058Z [INFO] server-bf17a376-c094-ea84-3489-52ad4bf33ff0: duplicate requestVote for same term: term=2 - testing.go:149: server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c: 2025-02-09T06:52:29.058Z [INFO] server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c: duplicate requestVote for same term: term=2 -testLoggerAdapter verbose: server-bf17a376-c094-ea84-3489-52ad4bf33ff0: 2025-02-09T06:52:29.058Z [INFO] server-bf17a376-c094-ea84-3489-52ad4bf33ff0: election won: tally=2 -testLoggerAdapter verbose: server-fd343c0f-d0c9-813c-8b62-827445bc927e: 2025-02-09T06:52:29.058Z [INFO] server-fd343c0f-d0c9-813c-8b62-827445bc927e: duplicate requestVote for same term: term=2 - testing.go:149: server-bf17a376-c094-ea84-3489-52ad4bf33ff0: 2025-02-09T06:52:29.058Z [INFO] server-bf17a376-c094-ea84-3489-52ad4bf33ff0: election won: tally=2 -testLoggerAdapter verbose: server-bf17a376-c094-ea84-3489-52ad4bf33ff0: 2025-02-09T06:52:29.058Z [INFO] server-bf17a376-c094-ea84-3489-52ad4bf33ff0: entering leader state: leader="Node at bf17a376-c094-ea84-3489-52ad4bf33ff0 [Leader]" - testing.go:149: server-bf17a376-c094-ea84-3489-52ad4bf33ff0: 2025-02-09T06:52:29.058Z [INFO] server-bf17a376-c094-ea84-3489-52ad4bf33ff0: entering leader state: leader="Node at bf17a376-c094-ea84-3489-52ad4bf33ff0 [Leader]" -testLoggerAdapter verbose: server-bf17a376-c094-ea84-3489-52ad4bf33ff0: 2025-02-09T06:52:29.058Z [INFO] server-bf17a376-c094-ea84-3489-52ad4bf33ff0: added peer, starting replication: peer=server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c - testing.go:149: server-bf17a376-c094-ea84-3489-52ad4bf33ff0: 2025-02-09T06:52:29.058Z [INFO] server-bf17a376-c094-ea84-3489-52ad4bf33ff0: added peer, starting replication: peer=server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c -testLoggerAdapter verbose: server-bf17a376-c094-ea84-3489-52ad4bf33ff0: 2025-02-09T06:52:29.059Z [INFO] server-bf17a376-c094-ea84-3489-52ad4bf33ff0: added peer, starting replication: peer=server-fd343c0f-d0c9-813c-8b62-827445bc927e - testing.go:149: server-fd343c0f-d0c9-813c-8b62-827445bc927e: 2025-02-09T06:52:29.058Z [INFO] server-fd343c0f-d0c9-813c-8b62-827445bc927e: duplicate requestVote for same term: term=2 - testing.go:149: server-bf17a376-c094-ea84-3489-52ad4bf33ff0: 2025-02-09T06:52:29.059Z [INFO] server-bf17a376-c094-ea84-3489-52ad4bf33ff0: added peer, starting replication: peer=server-fd343c0f-d0c9-813c-8b62-827445bc927e -testLoggerAdapter verbose: server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c: 2025-02-09T06:52:29.059Z [INFO] server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c: entering follower state: follower="Node at 2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c [Follower]" leader=bf17a376-c094-ea84-3489-52ad4bf33ff0 - testing.go:149: server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c: 2025-02-09T06:52:29.059Z [INFO] server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c: entering follower state: follower="Node at 2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c [Follower]" leader=bf17a376-c094-ea84-3489-52ad4bf33ff0 -testLoggerAdapter verbose: server-bf17a376-c094-ea84-3489-52ad4bf33ff0: 2025-02-09T06:52:29.059Z [INFO] server-bf17a376-c094-ea84-3489-52ad4bf33ff0: pipelining replication: peer="{Voter server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c 2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c}" - testing.go:149: server-bf17a376-c094-ea84-3489-52ad4bf33ff0: 2025-02-09T06:52:29.059Z [INFO] server-bf17a376-c094-ea84-3489-52ad4bf33ff0: pipelining replication: peer="{Voter server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c 2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c}" -testLoggerAdapter verbose: server-bf17a376-c094-ea84-3489-52ad4bf33ff0: 2025-02-09T06:52:29.059Z [INFO] server-bf17a376-c094-ea84-3489-52ad4bf33ff0: pipelining replication: peer="{Voter server-fd343c0f-d0c9-813c-8b62-827445bc927e fd343c0f-d0c9-813c-8b62-827445bc927e}" - testing.go:149: server-bf17a376-c094-ea84-3489-52ad4bf33ff0: 2025-02-09T06:52:29.059Z [INFO] server-bf17a376-c094-ea84-3489-52ad4bf33ff0: pipelining replication: peer="{Voter server-fd343c0f-d0c9-813c-8b62-827445bc927e fd343c0f-d0c9-813c-8b62-827445bc927e}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:29.164Z [INFO] stable state for Leader reached at 2025-02-09 06:52:29.059073297 +0000 UTC m=+10.037882148 (1 nodes), 60.980653ms from start of poll, 61.357755ms from cluster start. Timeout at 2025-02-09 06:52:29.164080528 +0000 UTC m=+10.142889295, 105.007147ms after stability - testing.go:149: cluster: 2025-02-09T06:52:29.164Z [INFO] stable state for Leader reached at 2025-02-09 06:52:29.059073297 +0000 UTC m=+10.037882148 (1 nodes), 60.980653ms from start of poll, 61.357755ms from cluster start. Timeout at 2025-02-09 06:52:29.164080528 +0000 UTC m=+10.142889295, 105.007147ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:29.164Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T06:52:29.164Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:29.269Z [INFO] stable state for Follower reached at 2025-02-09 06:52:29.164164019 +0000 UTC m=+10.142972799 (2 nodes), 707ns from start of poll, 166.448406ms from cluster start. Timeout at 2025-02-09 06:52:29.269947495 +0000 UTC m=+10.248756262, 105.783463ms after stability - testing.go:149: cluster: 2025-02-09T06:52:29.269Z [INFO] stable state for Follower reached at 2025-02-09 06:52:29.164164019 +0000 UTC m=+10.142972799 (2 nodes), 707ns from start of poll, 166.448406ms from cluster start. Timeout at 2025-02-09 06:52:29.269947495 +0000 UTC m=+10.248756262, 105.783463ms after stability -testLoggerAdapter verbose: server-bf17a376-c094-ea84-3489-52ad4bf33ff0: 2025-02-09T06:52:29.270Z [INFO] server-bf17a376-c094-ea84-3489-52ad4bf33ff0: updating configuration: command=AddStaging server-id=server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c server-addr=2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c servers="[{Suffrage:Voter ID:server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c Address:2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c} {Suffrage:Voter ID:server-fd343c0f-d0c9-813c-8b62-827445bc927e Address:fd343c0f-d0c9-813c-8b62-827445bc927e} {Suffrage:Voter ID:server-bf17a376-c094-ea84-3489-52ad4bf33ff0 Address:bf17a376-c094-ea84-3489-52ad4bf33ff0}]" - testing.go:149: server-bf17a376-c094-ea84-3489-52ad4bf33ff0: 2025-02-09T06:52:29.270Z [INFO] server-bf17a376-c094-ea84-3489-52ad4bf33ff0: updating configuration: command=AddStaging server-id=server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c server-addr=2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c servers="[{Suffrage:Voter ID:server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c Address:2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c} {Suffrage:Voter ID:server-fd343c0f-d0c9-813c-8b62-827445bc927e Address:fd343c0f-d0c9-813c-8b62-827445bc927e} {Suffrage:Voter ID:server-bf17a376-c094-ea84-3489-52ad4bf33ff0 Address:bf17a376-c094-ea84-3489-52ad4bf33ff0}]" -testLoggerAdapter verbose: server-bf17a376-c094-ea84-3489-52ad4bf33ff0: 2025-02-09T06:52:29.270Z [INFO] server-bf17a376-c094-ea84-3489-52ad4bf33ff0: aborting pipeline replication: peer="{Voter server-fd343c0f-d0c9-813c-8b62-827445bc927e fd343c0f-d0c9-813c-8b62-827445bc927e}" - testing.go:149: server-bf17a376-c094-ea84-3489-52ad4bf33ff0: 2025-02-09T06:52:29.270Z [INFO] server-bf17a376-c094-ea84-3489-52ad4bf33ff0: aborting pipeline replication: peer="{Voter server-fd343c0f-d0c9-813c-8b62-827445bc927e fd343c0f-d0c9-813c-8b62-827445bc927e}" -testLoggerAdapter verbose: server-bf17a376-c094-ea84-3489-52ad4bf33ff0: 2025-02-09T06:52:29.270Z [INFO] server-bf17a376-c094-ea84-3489-52ad4bf33ff0: aborting pipeline replication: peer="{Voter server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c 2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c}" - testing.go:149: server-bf17a376-c094-ea84-3489-52ad4bf33ff0: 2025-02-09T06:52:29.270Z [INFO] server-bf17a376-c094-ea84-3489-52ad4bf33ff0: aborting pipeline replication: peer="{Voter server-2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c 2f6e6abf-f2b1-4fb9-b4d6-1bc6cbb7315c}" +testLoggerAdapter verbose: server-99fa282e-be8b-8862-f95a-b2ea3590ec5a: 2024-01-08T00:31:03.974Z [INFO] server-99fa282e-be8b-8862-f95a-b2ea3590ec5a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-99fa282e-be8b-8862-f95a-b2ea3590ec5a Address:99fa282e-be8b-8862-f95a-b2ea3590ec5a} {Suffrage:Voter ID:server-a89018d7-5e6b-df40-8a5f-33cd3d806e91 Address:a89018d7-5e6b-df40-8a5f-33cd3d806e91} {Suffrage:Voter ID:server-5cb4b95b-05a4-f504-2037-19709c616ee1 Address:5cb4b95b-05a4-f504-2037-19709c616ee1}]" + testing.go:149: server-99fa282e-be8b-8862-f95a-b2ea3590ec5a: 2024-01-08T00:31:03.974Z [INFO] server-99fa282e-be8b-8862-f95a-b2ea3590ec5a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-99fa282e-be8b-8862-f95a-b2ea3590ec5a Address:99fa282e-be8b-8862-f95a-b2ea3590ec5a} {Suffrage:Voter ID:server-a89018d7-5e6b-df40-8a5f-33cd3d806e91 Address:a89018d7-5e6b-df40-8a5f-33cd3d806e91} {Suffrage:Voter ID:server-5cb4b95b-05a4-f504-2037-19709c616ee1 Address:5cb4b95b-05a4-f504-2037-19709c616ee1}]" +testLoggerAdapter verbose: server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: 2024-01-08T00:31:03.975Z [INFO] server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-99fa282e-be8b-8862-f95a-b2ea3590ec5a Address:99fa282e-be8b-8862-f95a-b2ea3590ec5a} {Suffrage:Voter ID:server-a89018d7-5e6b-df40-8a5f-33cd3d806e91 Address:a89018d7-5e6b-df40-8a5f-33cd3d806e91} {Suffrage:Voter ID:server-5cb4b95b-05a4-f504-2037-19709c616ee1 Address:5cb4b95b-05a4-f504-2037-19709c616ee1}]" + testing.go:149: server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: 2024-01-08T00:31:03.975Z [INFO] server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-99fa282e-be8b-8862-f95a-b2ea3590ec5a Address:99fa282e-be8b-8862-f95a-b2ea3590ec5a} {Suffrage:Voter ID:server-a89018d7-5e6b-df40-8a5f-33cd3d806e91 Address:a89018d7-5e6b-df40-8a5f-33cd3d806e91} {Suffrage:Voter ID:server-5cb4b95b-05a4-f504-2037-19709c616ee1 Address:5cb4b95b-05a4-f504-2037-19709c616ee1}]" +testLoggerAdapter verbose: server-5cb4b95b-05a4-f504-2037-19709c616ee1: 2024-01-08T00:31:03.975Z [INFO] server-5cb4b95b-05a4-f504-2037-19709c616ee1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-99fa282e-be8b-8862-f95a-b2ea3590ec5a Address:99fa282e-be8b-8862-f95a-b2ea3590ec5a} {Suffrage:Voter ID:server-a89018d7-5e6b-df40-8a5f-33cd3d806e91 Address:a89018d7-5e6b-df40-8a5f-33cd3d806e91} {Suffrage:Voter ID:server-5cb4b95b-05a4-f504-2037-19709c616ee1 Address:5cb4b95b-05a4-f504-2037-19709c616ee1}]" + testing.go:149: server-5cb4b95b-05a4-f504-2037-19709c616ee1: 2024-01-08T00:31:03.975Z [INFO] server-5cb4b95b-05a4-f504-2037-19709c616ee1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-99fa282e-be8b-8862-f95a-b2ea3590ec5a Address:99fa282e-be8b-8862-f95a-b2ea3590ec5a} {Suffrage:Voter ID:server-a89018d7-5e6b-df40-8a5f-33cd3d806e91 Address:a89018d7-5e6b-df40-8a5f-33cd3d806e91} {Suffrage:Voter ID:server-5cb4b95b-05a4-f504-2037-19709c616ee1 Address:5cb4b95b-05a4-f504-2037-19709c616ee1}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:03.975Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:03.975Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-99fa282e-be8b-8862-f95a-b2ea3590ec5a: 2024-01-08T00:31:03.975Z [INFO] server-99fa282e-be8b-8862-f95a-b2ea3590ec5a: entering follower state: follower="Node at 99fa282e-be8b-8862-f95a-b2ea3590ec5a [Follower]" leader= + testing.go:149: server-99fa282e-be8b-8862-f95a-b2ea3590ec5a: 2024-01-08T00:31:03.975Z [INFO] server-99fa282e-be8b-8862-f95a-b2ea3590ec5a: entering follower state: follower="Node at 99fa282e-be8b-8862-f95a-b2ea3590ec5a [Follower]" leader= +testLoggerAdapter verbose: server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: 2024-01-08T00:31:03.975Z [INFO] server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: entering follower state: follower="Node at a89018d7-5e6b-df40-8a5f-33cd3d806e91 [Follower]" leader= + testing.go:149: server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: 2024-01-08T00:31:03.975Z [INFO] server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: entering follower state: follower="Node at a89018d7-5e6b-df40-8a5f-33cd3d806e91 [Follower]" leader= +testLoggerAdapter verbose: server-5cb4b95b-05a4-f504-2037-19709c616ee1: 2024-01-08T00:31:03.975Z [INFO] server-5cb4b95b-05a4-f504-2037-19709c616ee1: entering follower state: follower="Node at 5cb4b95b-05a4-f504-2037-19709c616ee1 [Follower]" leader= + testing.go:149: server-5cb4b95b-05a4-f504-2037-19709c616ee1: 2024-01-08T00:31:03.975Z [INFO] server-5cb4b95b-05a4-f504-2037-19709c616ee1: entering follower state: follower="Node at 5cb4b95b-05a4-f504-2037-19709c616ee1 [Follower]" leader= +testLoggerAdapter verbose: server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: 2024-01-08T00:31:04.029Z [WARN] server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: 2024-01-08T00:31:04.029Z [WARN] server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: 2024-01-08T00:31:04.029Z [INFO] server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: entering candidate state: node="Node at a89018d7-5e6b-df40-8a5f-33cd3d806e91 [Candidate]" term=2 + testing.go:149: server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: 2024-01-08T00:31:04.029Z [INFO] server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: entering candidate state: node="Node at a89018d7-5e6b-df40-8a5f-33cd3d806e91 [Candidate]" term=2 +testLoggerAdapter verbose: server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: 2024-01-08T00:31:04.029Z [INFO] server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: election won: tally=2 + testing.go:149: server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: 2024-01-08T00:31:04.029Z [INFO] server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: election won: tally=2 +testLoggerAdapter verbose: server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: 2024-01-08T00:31:04.029Z [INFO] server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: entering leader state: leader="Node at a89018d7-5e6b-df40-8a5f-33cd3d806e91 [Leader]" + testing.go:149: server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: 2024-01-08T00:31:04.029Z [INFO] server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: entering leader state: leader="Node at a89018d7-5e6b-df40-8a5f-33cd3d806e91 [Leader]" +testLoggerAdapter verbose: server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: 2024-01-08T00:31:04.029Z [INFO] server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: added peer, starting replication: peer=server-99fa282e-be8b-8862-f95a-b2ea3590ec5a + testing.go:149: server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: 2024-01-08T00:31:04.029Z [INFO] server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: added peer, starting replication: peer=server-99fa282e-be8b-8862-f95a-b2ea3590ec5a +testLoggerAdapter verbose: server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: 2024-01-08T00:31:04.029Z [INFO] server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: added peer, starting replication: peer=server-5cb4b95b-05a4-f504-2037-19709c616ee1 + testing.go:149: server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: 2024-01-08T00:31:04.029Z [INFO] server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: added peer, starting replication: peer=server-5cb4b95b-05a4-f504-2037-19709c616ee1 +testLoggerAdapter verbose: server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: 2024-01-08T00:31:04.030Z [INFO] server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: pipelining replication: peer="{Voter server-5cb4b95b-05a4-f504-2037-19709c616ee1 5cb4b95b-05a4-f504-2037-19709c616ee1}" + testing.go:149: server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: 2024-01-08T00:31:04.030Z [INFO] server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: pipelining replication: peer="{Voter server-5cb4b95b-05a4-f504-2037-19709c616ee1 5cb4b95b-05a4-f504-2037-19709c616ee1}" +testLoggerAdapter verbose: server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: 2024-01-08T00:31:04.030Z [INFO] server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: pipelining replication: peer="{Voter server-99fa282e-be8b-8862-f95a-b2ea3590ec5a 99fa282e-be8b-8862-f95a-b2ea3590ec5a}" + testing.go:149: server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: 2024-01-08T00:31:04.030Z [INFO] server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: pipelining replication: peer="{Voter server-99fa282e-be8b-8862-f95a-b2ea3590ec5a 99fa282e-be8b-8862-f95a-b2ea3590ec5a}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:04.135Z [INFO] stable state for Leader reached at 2024-01-08 00:31:04.030255164 +0000 UTC m=+10.077144288 (1 nodes), 54.98204ms from start of poll, 55.413551ms from cluster start. Timeout at 2024-01-08 00:31:04.135434641 +0000 UTC m=+10.182323731, 105.179443ms after stability + testing.go:149: cluster: 2024-01-08T00:31:04.135Z [INFO] stable state for Leader reached at 2024-01-08 00:31:04.030255164 +0000 UTC m=+10.077144288 (1 nodes), 54.98204ms from start of poll, 55.413551ms from cluster start. Timeout at 2024-01-08 00:31:04.135434641 +0000 UTC m=+10.182323731, 105.179443ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:04.135Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T00:31:04.135Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:04.240Z [INFO] stable state for Follower reached at 2024-01-08 00:31:04.135590954 +0000 UTC m=+10.182480089 (2 nodes), 2.439µs from start of poll, 160.749352ms from cluster start. Timeout at 2024-01-08 00:31:04.24059557 +0000 UTC m=+10.287484669, 105.00458ms after stability + testing.go:149: cluster: 2024-01-08T00:31:04.240Z [INFO] stable state for Follower reached at 2024-01-08 00:31:04.135590954 +0000 UTC m=+10.182480089 (2 nodes), 2.439µs from start of poll, 160.749352ms from cluster start. Timeout at 2024-01-08 00:31:04.24059557 +0000 UTC m=+10.287484669, 105.00458ms after stability +testLoggerAdapter verbose: server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: 2024-01-08T00:31:04.240Z [INFO] server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: updating configuration: command=AddStaging server-id=server-99fa282e-be8b-8862-f95a-b2ea3590ec5a server-addr=99fa282e-be8b-8862-f95a-b2ea3590ec5a servers="[{Suffrage:Voter ID:server-99fa282e-be8b-8862-f95a-b2ea3590ec5a Address:99fa282e-be8b-8862-f95a-b2ea3590ec5a} {Suffrage:Voter ID:server-a89018d7-5e6b-df40-8a5f-33cd3d806e91 Address:a89018d7-5e6b-df40-8a5f-33cd3d806e91} {Suffrage:Voter ID:server-5cb4b95b-05a4-f504-2037-19709c616ee1 Address:5cb4b95b-05a4-f504-2037-19709c616ee1}]" + testing.go:149: server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: 2024-01-08T00:31:04.240Z [INFO] server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: updating configuration: command=AddStaging server-id=server-99fa282e-be8b-8862-f95a-b2ea3590ec5a server-addr=99fa282e-be8b-8862-f95a-b2ea3590ec5a servers="[{Suffrage:Voter ID:server-99fa282e-be8b-8862-f95a-b2ea3590ec5a Address:99fa282e-be8b-8862-f95a-b2ea3590ec5a} {Suffrage:Voter ID:server-a89018d7-5e6b-df40-8a5f-33cd3d806e91 Address:a89018d7-5e6b-df40-8a5f-33cd3d806e91} {Suffrage:Voter ID:server-5cb4b95b-05a4-f504-2037-19709c616ee1 Address:5cb4b95b-05a4-f504-2037-19709c616ee1}]" +testLoggerAdapter verbose: server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: 2024-01-08T00:31:04.241Z [INFO] server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: aborting pipeline replication: peer="{Voter server-5cb4b95b-05a4-f504-2037-19709c616ee1 5cb4b95b-05a4-f504-2037-19709c616ee1}" + testing.go:149: server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: 2024-01-08T00:31:04.241Z [INFO] server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: aborting pipeline replication: peer="{Voter server-5cb4b95b-05a4-f504-2037-19709c616ee1 5cb4b95b-05a4-f504-2037-19709c616ee1}" +testLoggerAdapter verbose: server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: 2024-01-08T00:31:04.241Z [INFO] server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: aborting pipeline replication: peer="{Voter server-99fa282e-be8b-8862-f95a-b2ea3590ec5a 99fa282e-be8b-8862-f95a-b2ea3590ec5a}" + testing.go:149: server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: 2024-01-08T00:31:04.241Z [INFO] server-a89018d7-5e6b-df40-8a5f-33cd3d806e91: aborting pipeline replication: peer="{Voter server-99fa282e-be8b-8862-f95a-b2ea3590ec5a 99fa282e-be8b-8862-f95a-b2ea3590ec5a}" --- PASS: TestRaft_AddKnownPeer (0.27s) === RUN TestRaft_RemoveUnknownPeer -testLoggerAdapter verbose: server-f73e93b4-e099-1e0b-8a78-0822efda467b: 2025-02-09T06:52:29.271Z [INFO] server-f73e93b4-e099-1e0b-8a78-0822efda467b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f73e93b4-e099-1e0b-8a78-0822efda467b Address:f73e93b4-e099-1e0b-8a78-0822efda467b} {Suffrage:Voter ID:server-dcbaa381-966b-94be-6265-3e90ce55e628 Address:dcbaa381-966b-94be-6265-3e90ce55e628} {Suffrage:Voter ID:server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76 Address:da29ab7d-22f8-62ba-ec4c-32a6ac3cec76}]" - testing.go:149: server-f73e93b4-e099-1e0b-8a78-0822efda467b: 2025-02-09T06:52:29.271Z [INFO] server-f73e93b4-e099-1e0b-8a78-0822efda467b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f73e93b4-e099-1e0b-8a78-0822efda467b Address:f73e93b4-e099-1e0b-8a78-0822efda467b} {Suffrage:Voter ID:server-dcbaa381-966b-94be-6265-3e90ce55e628 Address:dcbaa381-966b-94be-6265-3e90ce55e628} {Suffrage:Voter ID:server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76 Address:da29ab7d-22f8-62ba-ec4c-32a6ac3cec76}]" -testLoggerAdapter verbose: server-f73e93b4-e099-1e0b-8a78-0822efda467b: 2025-02-09T06:52:29.271Z [INFO] server-f73e93b4-e099-1e0b-8a78-0822efda467b: entering follower state: follower="Node at f73e93b4-e099-1e0b-8a78-0822efda467b [Follower]" leader= - testing.go:149: server-f73e93b4-e099-1e0b-8a78-0822efda467b: 2025-02-09T06:52:29.271Z [INFO] server-f73e93b4-e099-1e0b-8a78-0822efda467b: entering follower state: follower="Node at f73e93b4-e099-1e0b-8a78-0822efda467b [Follower]" leader= -testLoggerAdapter verbose: server-dcbaa381-966b-94be-6265-3e90ce55e628: 2025-02-09T06:52:29.271Z [INFO] server-dcbaa381-966b-94be-6265-3e90ce55e628: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f73e93b4-e099-1e0b-8a78-0822efda467b Address:f73e93b4-e099-1e0b-8a78-0822efda467b} {Suffrage:Voter ID:server-dcbaa381-966b-94be-6265-3e90ce55e628 Address:dcbaa381-966b-94be-6265-3e90ce55e628} {Suffrage:Voter ID:server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76 Address:da29ab7d-22f8-62ba-ec4c-32a6ac3cec76}]" - testing.go:149: server-dcbaa381-966b-94be-6265-3e90ce55e628: 2025-02-09T06:52:29.271Z [INFO] server-dcbaa381-966b-94be-6265-3e90ce55e628: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f73e93b4-e099-1e0b-8a78-0822efda467b Address:f73e93b4-e099-1e0b-8a78-0822efda467b} {Suffrage:Voter ID:server-dcbaa381-966b-94be-6265-3e90ce55e628 Address:dcbaa381-966b-94be-6265-3e90ce55e628} {Suffrage:Voter ID:server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76 Address:da29ab7d-22f8-62ba-ec4c-32a6ac3cec76}]" -testLoggerAdapter verbose: server-dcbaa381-966b-94be-6265-3e90ce55e628: 2025-02-09T06:52:29.271Z [INFO] server-dcbaa381-966b-94be-6265-3e90ce55e628: entering follower state: follower="Node at dcbaa381-966b-94be-6265-3e90ce55e628 [Follower]" leader= - testing.go:149: server-dcbaa381-966b-94be-6265-3e90ce55e628: 2025-02-09T06:52:29.271Z [INFO] server-dcbaa381-966b-94be-6265-3e90ce55e628: entering follower state: follower="Node at dcbaa381-966b-94be-6265-3e90ce55e628 [Follower]" leader= -testLoggerAdapter verbose: server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: 2025-02-09T06:52:29.271Z [INFO] server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f73e93b4-e099-1e0b-8a78-0822efda467b Address:f73e93b4-e099-1e0b-8a78-0822efda467b} {Suffrage:Voter ID:server-dcbaa381-966b-94be-6265-3e90ce55e628 Address:dcbaa381-966b-94be-6265-3e90ce55e628} {Suffrage:Voter ID:server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76 Address:da29ab7d-22f8-62ba-ec4c-32a6ac3cec76}]" - testing.go:149: server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: 2025-02-09T06:52:29.271Z [INFO] server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f73e93b4-e099-1e0b-8a78-0822efda467b Address:f73e93b4-e099-1e0b-8a78-0822efda467b} {Suffrage:Voter ID:server-dcbaa381-966b-94be-6265-3e90ce55e628 Address:dcbaa381-966b-94be-6265-3e90ce55e628} {Suffrage:Voter ID:server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76 Address:da29ab7d-22f8-62ba-ec4c-32a6ac3cec76}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:29.271Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:29.271Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: 2025-02-09T06:52:29.271Z [INFO] server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: entering follower state: follower="Node at da29ab7d-22f8-62ba-ec4c-32a6ac3cec76 [Follower]" leader= - testing.go:149: server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: 2025-02-09T06:52:29.271Z [INFO] server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: entering follower state: follower="Node at da29ab7d-22f8-62ba-ec4c-32a6ac3cec76 [Follower]" leader= -testLoggerAdapter verbose: server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: 2025-02-09T06:52:29.356Z [WARN] server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: 2025-02-09T06:52:29.356Z [WARN] server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: 2025-02-09T06:52:29.356Z [INFO] server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: entering candidate state: node="Node at da29ab7d-22f8-62ba-ec4c-32a6ac3cec76 [Candidate]" term=2 - testing.go:149: server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: 2025-02-09T06:52:29.356Z [INFO] server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: entering candidate state: node="Node at da29ab7d-22f8-62ba-ec4c-32a6ac3cec76 [Candidate]" term=2 -testLoggerAdapter verbose: server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: 2025-02-09T06:52:29.356Z [INFO] server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: election won: tally=2 - testing.go:149: server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: 2025-02-09T06:52:29.356Z [INFO] server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: election won: tally=2 -testLoggerAdapter verbose: server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: 2025-02-09T06:52:29.356Z [INFO] server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: entering leader state: leader="Node at da29ab7d-22f8-62ba-ec4c-32a6ac3cec76 [Leader]" - testing.go:149: server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: 2025-02-09T06:52:29.356Z [INFO] server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: entering leader state: leader="Node at da29ab7d-22f8-62ba-ec4c-32a6ac3cec76 [Leader]" -testLoggerAdapter verbose: server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: 2025-02-09T06:52:29.356Z [INFO] server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: added peer, starting replication: peer=server-f73e93b4-e099-1e0b-8a78-0822efda467b - testing.go:149: server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: 2025-02-09T06:52:29.356Z [INFO] server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: added peer, starting replication: peer=server-f73e93b4-e099-1e0b-8a78-0822efda467b -testLoggerAdapter verbose: server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: 2025-02-09T06:52:29.356Z [INFO] server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: added peer, starting replication: peer=server-dcbaa381-966b-94be-6265-3e90ce55e628 - testing.go:149: server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: 2025-02-09T06:52:29.356Z [INFO] server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: added peer, starting replication: peer=server-dcbaa381-966b-94be-6265-3e90ce55e628 -testLoggerAdapter verbose: server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: 2025-02-09T06:52:29.356Z [INFO] server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: pipelining replication: peer="{Voter server-f73e93b4-e099-1e0b-8a78-0822efda467b f73e93b4-e099-1e0b-8a78-0822efda467b}" - testing.go:149: server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: 2025-02-09T06:52:29.356Z [INFO] server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: pipelining replication: peer="{Voter server-f73e93b4-e099-1e0b-8a78-0822efda467b f73e93b4-e099-1e0b-8a78-0822efda467b}" -testLoggerAdapter verbose: server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: 2025-02-09T06:52:29.356Z [INFO] server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: pipelining replication: peer="{Voter server-dcbaa381-966b-94be-6265-3e90ce55e628 dcbaa381-966b-94be-6265-3e90ce55e628}" - testing.go:149: server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: 2025-02-09T06:52:29.356Z [INFO] server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: pipelining replication: peer="{Voter server-dcbaa381-966b-94be-6265-3e90ce55e628 dcbaa381-966b-94be-6265-3e90ce55e628}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:29.462Z [INFO] stable state for Leader reached at 2025-02-09 06:52:29.356946146 +0000 UTC m=+10.335754914 (1 nodes), 85.581788ms from start of poll, 85.925401ms from cluster start. Timeout at 2025-02-09 06:52:29.462875108 +0000 UTC m=+10.441683868, 105.928954ms after stability - testing.go:149: cluster: 2025-02-09T06:52:29.462Z [INFO] stable state for Leader reached at 2025-02-09 06:52:29.356946146 +0000 UTC m=+10.335754914 (1 nodes), 85.581788ms from start of poll, 85.925401ms from cluster start. Timeout at 2025-02-09 06:52:29.462875108 +0000 UTC m=+10.441683868, 105.928954ms after stability -testLoggerAdapter verbose: server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: 2025-02-09T06:52:29.462Z [INFO] server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: updating configuration: command=RemoveServer server-id=a3ad6b08-52dd-0964-96b0-003eb423b83a server-addr= servers="[{Suffrage:Voter ID:server-f73e93b4-e099-1e0b-8a78-0822efda467b Address:f73e93b4-e099-1e0b-8a78-0822efda467b} {Suffrage:Voter ID:server-dcbaa381-966b-94be-6265-3e90ce55e628 Address:dcbaa381-966b-94be-6265-3e90ce55e628} {Suffrage:Voter ID:server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76 Address:da29ab7d-22f8-62ba-ec4c-32a6ac3cec76}]" - testing.go:149: server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: 2025-02-09T06:52:29.462Z [INFO] server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: updating configuration: command=RemoveServer server-id=a3ad6b08-52dd-0964-96b0-003eb423b83a server-addr= servers="[{Suffrage:Voter ID:server-f73e93b4-e099-1e0b-8a78-0822efda467b Address:f73e93b4-e099-1e0b-8a78-0822efda467b} {Suffrage:Voter ID:server-dcbaa381-966b-94be-6265-3e90ce55e628 Address:dcbaa381-966b-94be-6265-3e90ce55e628} {Suffrage:Voter ID:server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76 Address:da29ab7d-22f8-62ba-ec4c-32a6ac3cec76}]" -testLoggerAdapter verbose: server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: 2025-02-09T06:52:29.463Z [INFO] server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: aborting pipeline replication: peer="{Voter server-dcbaa381-966b-94be-6265-3e90ce55e628 dcbaa381-966b-94be-6265-3e90ce55e628}" - testing.go:149: server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: 2025-02-09T06:52:29.463Z [INFO] server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: aborting pipeline replication: peer="{Voter server-dcbaa381-966b-94be-6265-3e90ce55e628 dcbaa381-966b-94be-6265-3e90ce55e628}" -testLoggerAdapter verbose: server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: 2025-02-09T06:52:29.463Z [INFO] server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: aborting pipeline replication: peer="{Voter server-f73e93b4-e099-1e0b-8a78-0822efda467b f73e93b4-e099-1e0b-8a78-0822efda467b}" - testing.go:149: server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: 2025-02-09T06:52:29.463Z [INFO] server-da29ab7d-22f8-62ba-ec4c-32a6ac3cec76: aborting pipeline replication: peer="{Voter server-f73e93b4-e099-1e0b-8a78-0822efda467b f73e93b4-e099-1e0b-8a78-0822efda467b}" ---- PASS: TestRaft_RemoveUnknownPeer (0.19s) +testLoggerAdapter verbose: server-bef9a972-9d46-e167-f0ac-cbc19d7339d5: 2024-01-08T00:31:04.242Z [INFO] server-bef9a972-9d46-e167-f0ac-cbc19d7339d5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bef9a972-9d46-e167-f0ac-cbc19d7339d5 Address:bef9a972-9d46-e167-f0ac-cbc19d7339d5} {Suffrage:Voter ID:server-62fd3dc2-c7f3-be3a-03fe-86bea0637862 Address:62fd3dc2-c7f3-be3a-03fe-86bea0637862} {Suffrage:Voter ID:server-b3355308-2239-2be5-e7fc-63f633d00e06 Address:b3355308-2239-2be5-e7fc-63f633d00e06}]" + testing.go:149: server-bef9a972-9d46-e167-f0ac-cbc19d7339d5: 2024-01-08T00:31:04.242Z [INFO] server-bef9a972-9d46-e167-f0ac-cbc19d7339d5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bef9a972-9d46-e167-f0ac-cbc19d7339d5 Address:bef9a972-9d46-e167-f0ac-cbc19d7339d5} {Suffrage:Voter ID:server-62fd3dc2-c7f3-be3a-03fe-86bea0637862 Address:62fd3dc2-c7f3-be3a-03fe-86bea0637862} {Suffrage:Voter ID:server-b3355308-2239-2be5-e7fc-63f633d00e06 Address:b3355308-2239-2be5-e7fc-63f633d00e06}]" +testLoggerAdapter verbose: server-62fd3dc2-c7f3-be3a-03fe-86bea0637862: 2024-01-08T00:31:04.242Z [INFO] server-62fd3dc2-c7f3-be3a-03fe-86bea0637862: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bef9a972-9d46-e167-f0ac-cbc19d7339d5 Address:bef9a972-9d46-e167-f0ac-cbc19d7339d5} {Suffrage:Voter ID:server-62fd3dc2-c7f3-be3a-03fe-86bea0637862 Address:62fd3dc2-c7f3-be3a-03fe-86bea0637862} {Suffrage:Voter ID:server-b3355308-2239-2be5-e7fc-63f633d00e06 Address:b3355308-2239-2be5-e7fc-63f633d00e06}]" + testing.go:149: server-62fd3dc2-c7f3-be3a-03fe-86bea0637862: 2024-01-08T00:31:04.242Z [INFO] server-62fd3dc2-c7f3-be3a-03fe-86bea0637862: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bef9a972-9d46-e167-f0ac-cbc19d7339d5 Address:bef9a972-9d46-e167-f0ac-cbc19d7339d5} {Suffrage:Voter ID:server-62fd3dc2-c7f3-be3a-03fe-86bea0637862 Address:62fd3dc2-c7f3-be3a-03fe-86bea0637862} {Suffrage:Voter ID:server-b3355308-2239-2be5-e7fc-63f633d00e06 Address:b3355308-2239-2be5-e7fc-63f633d00e06}]" +testLoggerAdapter verbose: server-b3355308-2239-2be5-e7fc-63f633d00e06: 2024-01-08T00:31:04.242Z [INFO] server-b3355308-2239-2be5-e7fc-63f633d00e06: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bef9a972-9d46-e167-f0ac-cbc19d7339d5 Address:bef9a972-9d46-e167-f0ac-cbc19d7339d5} {Suffrage:Voter ID:server-62fd3dc2-c7f3-be3a-03fe-86bea0637862 Address:62fd3dc2-c7f3-be3a-03fe-86bea0637862} {Suffrage:Voter ID:server-b3355308-2239-2be5-e7fc-63f633d00e06 Address:b3355308-2239-2be5-e7fc-63f633d00e06}]" + testing.go:149: server-b3355308-2239-2be5-e7fc-63f633d00e06: 2024-01-08T00:31:04.242Z [INFO] server-b3355308-2239-2be5-e7fc-63f633d00e06: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bef9a972-9d46-e167-f0ac-cbc19d7339d5 Address:bef9a972-9d46-e167-f0ac-cbc19d7339d5} {Suffrage:Voter ID:server-62fd3dc2-c7f3-be3a-03fe-86bea0637862 Address:62fd3dc2-c7f3-be3a-03fe-86bea0637862} {Suffrage:Voter ID:server-b3355308-2239-2be5-e7fc-63f633d00e06 Address:b3355308-2239-2be5-e7fc-63f633d00e06}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:04.242Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:04.242Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-bef9a972-9d46-e167-f0ac-cbc19d7339d5: 2024-01-08T00:31:04.242Z [INFO] server-bef9a972-9d46-e167-f0ac-cbc19d7339d5: entering follower state: follower="Node at bef9a972-9d46-e167-f0ac-cbc19d7339d5 [Follower]" leader= + testing.go:149: server-bef9a972-9d46-e167-f0ac-cbc19d7339d5: 2024-01-08T00:31:04.242Z [INFO] server-bef9a972-9d46-e167-f0ac-cbc19d7339d5: entering follower state: follower="Node at bef9a972-9d46-e167-f0ac-cbc19d7339d5 [Follower]" leader= +testLoggerAdapter verbose: server-62fd3dc2-c7f3-be3a-03fe-86bea0637862: 2024-01-08T00:31:04.242Z [INFO] server-62fd3dc2-c7f3-be3a-03fe-86bea0637862: entering follower state: follower="Node at 62fd3dc2-c7f3-be3a-03fe-86bea0637862 [Follower]" leader= + testing.go:149: server-62fd3dc2-c7f3-be3a-03fe-86bea0637862: 2024-01-08T00:31:04.242Z [INFO] server-62fd3dc2-c7f3-be3a-03fe-86bea0637862: entering follower state: follower="Node at 62fd3dc2-c7f3-be3a-03fe-86bea0637862 [Follower]" leader= +testLoggerAdapter verbose: server-b3355308-2239-2be5-e7fc-63f633d00e06: 2024-01-08T00:31:04.242Z [INFO] server-b3355308-2239-2be5-e7fc-63f633d00e06: entering follower state: follower="Node at b3355308-2239-2be5-e7fc-63f633d00e06 [Follower]" leader= + testing.go:149: server-b3355308-2239-2be5-e7fc-63f633d00e06: 2024-01-08T00:31:04.242Z [INFO] server-b3355308-2239-2be5-e7fc-63f633d00e06: entering follower state: follower="Node at b3355308-2239-2be5-e7fc-63f633d00e06 [Follower]" leader= +testLoggerAdapter verbose: server-b3355308-2239-2be5-e7fc-63f633d00e06: 2024-01-08T00:31:04.318Z [WARN] server-b3355308-2239-2be5-e7fc-63f633d00e06: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-b3355308-2239-2be5-e7fc-63f633d00e06: 2024-01-08T00:31:04.318Z [WARN] server-b3355308-2239-2be5-e7fc-63f633d00e06: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-b3355308-2239-2be5-e7fc-63f633d00e06: 2024-01-08T00:31:04.318Z [INFO] server-b3355308-2239-2be5-e7fc-63f633d00e06: entering candidate state: node="Node at b3355308-2239-2be5-e7fc-63f633d00e06 [Candidate]" term=2 + testing.go:149: server-b3355308-2239-2be5-e7fc-63f633d00e06: 2024-01-08T00:31:04.318Z [INFO] server-b3355308-2239-2be5-e7fc-63f633d00e06: entering candidate state: node="Node at b3355308-2239-2be5-e7fc-63f633d00e06 [Candidate]" term=2 +testLoggerAdapter verbose: server-b3355308-2239-2be5-e7fc-63f633d00e06: 2024-01-08T00:31:04.318Z [INFO] server-b3355308-2239-2be5-e7fc-63f633d00e06: election won: tally=2 + testing.go:149: server-b3355308-2239-2be5-e7fc-63f633d00e06: 2024-01-08T00:31:04.318Z [INFO] server-b3355308-2239-2be5-e7fc-63f633d00e06: election won: tally=2 +testLoggerAdapter verbose: server-b3355308-2239-2be5-e7fc-63f633d00e06: 2024-01-08T00:31:04.318Z [INFO] server-b3355308-2239-2be5-e7fc-63f633d00e06: entering leader state: leader="Node at b3355308-2239-2be5-e7fc-63f633d00e06 [Leader]" + testing.go:149: server-b3355308-2239-2be5-e7fc-63f633d00e06: 2024-01-08T00:31:04.318Z [INFO] server-b3355308-2239-2be5-e7fc-63f633d00e06: entering leader state: leader="Node at b3355308-2239-2be5-e7fc-63f633d00e06 [Leader]" +testLoggerAdapter verbose: server-b3355308-2239-2be5-e7fc-63f633d00e06: 2024-01-08T00:31:04.318Z [INFO] server-b3355308-2239-2be5-e7fc-63f633d00e06: added peer, starting replication: peer=server-bef9a972-9d46-e167-f0ac-cbc19d7339d5 + testing.go:149: server-b3355308-2239-2be5-e7fc-63f633d00e06: 2024-01-08T00:31:04.318Z [INFO] server-b3355308-2239-2be5-e7fc-63f633d00e06: added peer, starting replication: peer=server-bef9a972-9d46-e167-f0ac-cbc19d7339d5 +testLoggerAdapter verbose: server-b3355308-2239-2be5-e7fc-63f633d00e06: 2024-01-08T00:31:04.318Z [INFO] server-b3355308-2239-2be5-e7fc-63f633d00e06: added peer, starting replication: peer=server-62fd3dc2-c7f3-be3a-03fe-86bea0637862 + testing.go:149: server-b3355308-2239-2be5-e7fc-63f633d00e06: 2024-01-08T00:31:04.318Z [INFO] server-b3355308-2239-2be5-e7fc-63f633d00e06: added peer, starting replication: peer=server-62fd3dc2-c7f3-be3a-03fe-86bea0637862 +testLoggerAdapter verbose: server-b3355308-2239-2be5-e7fc-63f633d00e06: 2024-01-08T00:31:04.318Z [INFO] server-b3355308-2239-2be5-e7fc-63f633d00e06: pipelining replication: peer="{Voter server-62fd3dc2-c7f3-be3a-03fe-86bea0637862 62fd3dc2-c7f3-be3a-03fe-86bea0637862}" + testing.go:149: server-b3355308-2239-2be5-e7fc-63f633d00e06: 2024-01-08T00:31:04.318Z [INFO] server-b3355308-2239-2be5-e7fc-63f633d00e06: pipelining replication: peer="{Voter server-62fd3dc2-c7f3-be3a-03fe-86bea0637862 62fd3dc2-c7f3-be3a-03fe-86bea0637862}" +testLoggerAdapter verbose: server-b3355308-2239-2be5-e7fc-63f633d00e06: 2024-01-08T00:31:04.318Z [INFO] server-b3355308-2239-2be5-e7fc-63f633d00e06: pipelining replication: peer="{Voter server-bef9a972-9d46-e167-f0ac-cbc19d7339d5 bef9a972-9d46-e167-f0ac-cbc19d7339d5}" + testing.go:149: server-b3355308-2239-2be5-e7fc-63f633d00e06: 2024-01-08T00:31:04.318Z [INFO] server-b3355308-2239-2be5-e7fc-63f633d00e06: pipelining replication: peer="{Voter server-bef9a972-9d46-e167-f0ac-cbc19d7339d5 bef9a972-9d46-e167-f0ac-cbc19d7339d5}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:04.424Z [INFO] stable state for Leader reached at 2024-01-08 00:31:04.318848665 +0000 UTC m=+10.365737779 (1 nodes), 76.134566ms from start of poll, 76.587108ms from cluster start. Timeout at 2024-01-08 00:31:04.424290039 +0000 UTC m=+10.471179086, 105.441307ms after stability + testing.go:149: cluster: 2024-01-08T00:31:04.424Z [INFO] stable state for Leader reached at 2024-01-08 00:31:04.318848665 +0000 UTC m=+10.365737779 (1 nodes), 76.134566ms from start of poll, 76.587108ms from cluster start. Timeout at 2024-01-08 00:31:04.424290039 +0000 UTC m=+10.471179086, 105.441307ms after stability +testLoggerAdapter verbose: server-b3355308-2239-2be5-e7fc-63f633d00e06: 2024-01-08T00:31:04.424Z [INFO] server-b3355308-2239-2be5-e7fc-63f633d00e06: updating configuration: command=RemoveServer server-id=99ae7ab9-15e9-f264-01bf-7a61134dc8c0 server-addr= servers="[{Suffrage:Voter ID:server-bef9a972-9d46-e167-f0ac-cbc19d7339d5 Address:bef9a972-9d46-e167-f0ac-cbc19d7339d5} {Suffrage:Voter ID:server-62fd3dc2-c7f3-be3a-03fe-86bea0637862 Address:62fd3dc2-c7f3-be3a-03fe-86bea0637862} {Suffrage:Voter ID:server-b3355308-2239-2be5-e7fc-63f633d00e06 Address:b3355308-2239-2be5-e7fc-63f633d00e06}]" + testing.go:149: server-b3355308-2239-2be5-e7fc-63f633d00e06: 2024-01-08T00:31:04.424Z [INFO] server-b3355308-2239-2be5-e7fc-63f633d00e06: updating configuration: command=RemoveServer server-id=99ae7ab9-15e9-f264-01bf-7a61134dc8c0 server-addr= servers="[{Suffrage:Voter ID:server-bef9a972-9d46-e167-f0ac-cbc19d7339d5 Address:bef9a972-9d46-e167-f0ac-cbc19d7339d5} {Suffrage:Voter ID:server-62fd3dc2-c7f3-be3a-03fe-86bea0637862 Address:62fd3dc2-c7f3-be3a-03fe-86bea0637862} {Suffrage:Voter ID:server-b3355308-2239-2be5-e7fc-63f633d00e06 Address:b3355308-2239-2be5-e7fc-63f633d00e06}]" +testLoggerAdapter verbose: server-b3355308-2239-2be5-e7fc-63f633d00e06: 2024-01-08T00:31:04.424Z [INFO] server-b3355308-2239-2be5-e7fc-63f633d00e06: aborting pipeline replication: peer="{Voter server-62fd3dc2-c7f3-be3a-03fe-86bea0637862 62fd3dc2-c7f3-be3a-03fe-86bea0637862}" + testing.go:149: server-b3355308-2239-2be5-e7fc-63f633d00e06: 2024-01-08T00:31:04.424Z [INFO] server-b3355308-2239-2be5-e7fc-63f633d00e06: aborting pipeline replication: peer="{Voter server-62fd3dc2-c7f3-be3a-03fe-86bea0637862 62fd3dc2-c7f3-be3a-03fe-86bea0637862}" +testLoggerAdapter verbose: server-b3355308-2239-2be5-e7fc-63f633d00e06: 2024-01-08T00:31:04.424Z [INFO] server-b3355308-2239-2be5-e7fc-63f633d00e06: aborting pipeline replication: peer="{Voter server-bef9a972-9d46-e167-f0ac-cbc19d7339d5 bef9a972-9d46-e167-f0ac-cbc19d7339d5}" + testing.go:149: server-b3355308-2239-2be5-e7fc-63f633d00e06: 2024-01-08T00:31:04.424Z [INFO] server-b3355308-2239-2be5-e7fc-63f633d00e06: aborting pipeline replication: peer="{Voter server-bef9a972-9d46-e167-f0ac-cbc19d7339d5 bef9a972-9d46-e167-f0ac-cbc19d7339d5}" +--- PASS: TestRaft_RemoveUnknownPeer (0.18s) === RUN TestRaft_SnapshotRestore -testLoggerAdapter verbose: server-85476a8a-6632-bf20-7fe8-c1a95944264a: 2025-02-09T06:52:29.463Z [INFO] server-85476a8a-6632-bf20-7fe8-c1a95944264a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-85476a8a-6632-bf20-7fe8-c1a95944264a Address:85476a8a-6632-bf20-7fe8-c1a95944264a}]" - testing.go:149: server-85476a8a-6632-bf20-7fe8-c1a95944264a: 2025-02-09T06:52:29.463Z [INFO] server-85476a8a-6632-bf20-7fe8-c1a95944264a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-85476a8a-6632-bf20-7fe8-c1a95944264a Address:85476a8a-6632-bf20-7fe8-c1a95944264a}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:29.463Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:29.463Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-85476a8a-6632-bf20-7fe8-c1a95944264a: 2025-02-09T06:52:29.463Z [INFO] server-85476a8a-6632-bf20-7fe8-c1a95944264a: entering follower state: follower="Node at 85476a8a-6632-bf20-7fe8-c1a95944264a [Follower]" leader= - testing.go:149: server-85476a8a-6632-bf20-7fe8-c1a95944264a: 2025-02-09T06:52:29.463Z [INFO] server-85476a8a-6632-bf20-7fe8-c1a95944264a: entering follower state: follower="Node at 85476a8a-6632-bf20-7fe8-c1a95944264a [Follower]" leader= -testLoggerAdapter verbose: server-85476a8a-6632-bf20-7fe8-c1a95944264a: 2025-02-09T06:52:29.552Z [WARN] server-85476a8a-6632-bf20-7fe8-c1a95944264a: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-85476a8a-6632-bf20-7fe8-c1a95944264a: 2025-02-09T06:52:29.552Z [WARN] server-85476a8a-6632-bf20-7fe8-c1a95944264a: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-85476a8a-6632-bf20-7fe8-c1a95944264a: 2025-02-09T06:52:29.552Z [INFO] server-85476a8a-6632-bf20-7fe8-c1a95944264a: entering candidate state: node="Node at 85476a8a-6632-bf20-7fe8-c1a95944264a [Candidate]" term=2 - testing.go:149: server-85476a8a-6632-bf20-7fe8-c1a95944264a: 2025-02-09T06:52:29.552Z [INFO] server-85476a8a-6632-bf20-7fe8-c1a95944264a: entering candidate state: node="Node at 85476a8a-6632-bf20-7fe8-c1a95944264a [Candidate]" term=2 -testLoggerAdapter verbose: server-85476a8a-6632-bf20-7fe8-c1a95944264a: 2025-02-09T06:52:29.552Z [INFO] server-85476a8a-6632-bf20-7fe8-c1a95944264a: election won: tally=1 - testing.go:149: server-85476a8a-6632-bf20-7fe8-c1a95944264a: 2025-02-09T06:52:29.552Z [INFO] server-85476a8a-6632-bf20-7fe8-c1a95944264a: election won: tally=1 -testLoggerAdapter verbose: server-85476a8a-6632-bf20-7fe8-c1a95944264a: 2025-02-09T06:52:29.552Z [INFO] server-85476a8a-6632-bf20-7fe8-c1a95944264a: entering leader state: leader="Node at 85476a8a-6632-bf20-7fe8-c1a95944264a [Leader]" - testing.go:149: server-85476a8a-6632-bf20-7fe8-c1a95944264a: 2025-02-09T06:52:29.552Z [INFO] server-85476a8a-6632-bf20-7fe8-c1a95944264a: entering leader state: leader="Node at 85476a8a-6632-bf20-7fe8-c1a95944264a [Leader]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:29.658Z [INFO] stable state for Leader reached at 2025-02-09 06:52:29.552957648 +0000 UTC m=+10.531766414 (1 nodes), 89.071099ms from start of poll, 89.201011ms from cluster start. Timeout at 2025-02-09 06:52:29.658842003 +0000 UTC m=+10.637650771, 105.884357ms after stability - testing.go:149: cluster: 2025-02-09T06:52:29.658Z [INFO] stable state for Leader reached at 2025-02-09 06:52:29.552957648 +0000 UTC m=+10.531766414 (1 nodes), 89.071099ms from start of poll, 89.201011ms from cluster start. Timeout at 2025-02-09 06:52:29.658842003 +0000 UTC m=+10.637650771, 105.884357ms after stability -testLoggerAdapter verbose: server-85476a8a-6632-bf20-7fe8-c1a95944264a: 2025-02-09T06:52:29.659Z [INFO] server-85476a8a-6632-bf20-7fe8-c1a95944264a: starting snapshot up to: index=102 - testing.go:149: server-85476a8a-6632-bf20-7fe8-c1a95944264a: 2025-02-09T06:52:29.659Z [INFO] server-85476a8a-6632-bf20-7fe8-c1a95944264a: starting snapshot up to: index=102 - testing.go:153: 2025-02-09T06:52:29.659Z [INFO] creating new snapshot: path=/tmp/raft1788383974/snapshots/2-102-1739083949659.tmp -testLoggerAdapter verbose: server-85476a8a-6632-bf20-7fe8-c1a95944264a: 2025-02-09T06:52:29.743Z [INFO] server-85476a8a-6632-bf20-7fe8-c1a95944264a: compacting logs: from=1 to=92 - testing.go:149: server-85476a8a-6632-bf20-7fe8-c1a95944264a: 2025-02-09T06:52:29.743Z [INFO] server-85476a8a-6632-bf20-7fe8-c1a95944264a: compacting logs: from=1 to=92 -testLoggerAdapter verbose: server-85476a8a-6632-bf20-7fe8-c1a95944264a: 2025-02-09T06:52:29.743Z [INFO] server-85476a8a-6632-bf20-7fe8-c1a95944264a: snapshot complete up to: index=102 - testing.go:149: server-85476a8a-6632-bf20-7fe8-c1a95944264a: 2025-02-09T06:52:29.743Z [INFO] server-85476a8a-6632-bf20-7fe8-c1a95944264a: snapshot complete up to: index=102 -testLoggerAdapter verbose: server-85476a8a-6632-bf20-7fe8-c1a95944264a: 2025-02-09T06:52:29.745Z [INFO] server-85476a8a-6632-bf20-7fe8-c1a95944264a: restored from snapshot: id=2-102-1739083949659 - testing.go:149: server-85476a8a-6632-bf20-7fe8-c1a95944264a: 2025-02-09T06:52:29.745Z [INFO] server-85476a8a-6632-bf20-7fe8-c1a95944264a: restored from snapshot: id=2-102-1739083949659 -testLoggerAdapter verbose: server-85476a8a-6632-bf20-7fe8-c1a95944264a: 2025-02-09T06:52:29.745Z [INFO] server-85476a8a-6632-bf20-7fe8-c1a95944264a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-85476a8a-6632-bf20-7fe8-c1a95944264a Address:85476a8a-6632-bf20-7fe8-c1a95944264a}]" - testing.go:149: server-85476a8a-6632-bf20-7fe8-c1a95944264a: 2025-02-09T06:52:29.745Z [INFO] server-85476a8a-6632-bf20-7fe8-c1a95944264a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-85476a8a-6632-bf20-7fe8-c1a95944264a Address:85476a8a-6632-bf20-7fe8-c1a95944264a}]" ---- PASS: TestRaft_SnapshotRestore (0.28s) +testLoggerAdapter verbose: server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: 2024-01-08T00:31:04.425Z [INFO] server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e3d918a9-adc4-4e25-5a00-a928c10fd24c Address:e3d918a9-adc4-4e25-5a00-a928c10fd24c}]" + testing.go:149: server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: 2024-01-08T00:31:04.425Z [INFO] server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e3d918a9-adc4-4e25-5a00-a928c10fd24c Address:e3d918a9-adc4-4e25-5a00-a928c10fd24c}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:04.425Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:04.425Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: 2024-01-08T00:31:04.425Z [INFO] server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: entering follower state: follower="Node at e3d918a9-adc4-4e25-5a00-a928c10fd24c [Follower]" leader= + testing.go:149: server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: 2024-01-08T00:31:04.425Z [INFO] server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: entering follower state: follower="Node at e3d918a9-adc4-4e25-5a00-a928c10fd24c [Follower]" leader= +testLoggerAdapter verbose: server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: 2024-01-08T00:31:04.498Z [WARN] server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: 2024-01-08T00:31:04.498Z [WARN] server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: 2024-01-08T00:31:04.498Z [INFO] server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: entering candidate state: node="Node at e3d918a9-adc4-4e25-5a00-a928c10fd24c [Candidate]" term=2 + testing.go:149: server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: 2024-01-08T00:31:04.498Z [INFO] server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: entering candidate state: node="Node at e3d918a9-adc4-4e25-5a00-a928c10fd24c [Candidate]" term=2 +testLoggerAdapter verbose: server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: 2024-01-08T00:31:04.498Z [INFO] server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: election won: tally=1 + testing.go:149: server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: 2024-01-08T00:31:04.498Z [INFO] server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: election won: tally=1 +testLoggerAdapter verbose: server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: 2024-01-08T00:31:04.498Z [INFO] server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: entering leader state: leader="Node at e3d918a9-adc4-4e25-5a00-a928c10fd24c [Leader]" + testing.go:149: server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: 2024-01-08T00:31:04.498Z [INFO] server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: entering leader state: leader="Node at e3d918a9-adc4-4e25-5a00-a928c10fd24c [Leader]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:04.604Z [INFO] stable state for Leader reached at 2024-01-08 00:31:04.498633221 +0000 UTC m=+10.545522260 (1 nodes), 73.069697ms from start of poll, 73.2095ms from cluster start. Timeout at 2024-01-08 00:31:04.604377403 +0000 UTC m=+10.651266477, 105.744217ms after stability + testing.go:149: cluster: 2024-01-08T00:31:04.604Z [INFO] stable state for Leader reached at 2024-01-08 00:31:04.498633221 +0000 UTC m=+10.545522260 (1 nodes), 73.069697ms from start of poll, 73.2095ms from cluster start. Timeout at 2024-01-08 00:31:04.604377403 +0000 UTC m=+10.651266477, 105.744217ms after stability +testLoggerAdapter verbose: server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: 2024-01-08T00:31:04.605Z [INFO] server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: starting snapshot up to: index=102 + testing.go:149: server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: 2024-01-08T00:31:04.605Z [INFO] server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: starting snapshot up to: index=102 + testing.go:153: 2024-01-08T00:31:04.605Z [INFO] creating new snapshot: path=/tmp/raft2694771695/snapshots/2-102-1704673864605.tmp +testLoggerAdapter verbose: server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: 2024-01-08T00:31:04.615Z [INFO] server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: compacting logs: from=1 to=92 + testing.go:149: server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: 2024-01-08T00:31:04.615Z [INFO] server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: compacting logs: from=1 to=92 +testLoggerAdapter verbose: server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: 2024-01-08T00:31:04.615Z [INFO] server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: snapshot complete up to: index=102 + testing.go:149: server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: 2024-01-08T00:31:04.615Z [INFO] server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: snapshot complete up to: index=102 +testLoggerAdapter verbose: server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: 2024-01-08T00:31:04.615Z [INFO] server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: restored from snapshot: id=2-102-1704673864605 + testing.go:149: server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: 2024-01-08T00:31:04.615Z [INFO] server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: restored from snapshot: id=2-102-1704673864605 +testLoggerAdapter verbose: server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: 2024-01-08T00:31:04.615Z [INFO] server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e3d918a9-adc4-4e25-5a00-a928c10fd24c Address:e3d918a9-adc4-4e25-5a00-a928c10fd24c}]" + testing.go:149: server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: 2024-01-08T00:31:04.615Z [INFO] server-e3d918a9-adc4-4e25-5a00-a928c10fd24c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e3d918a9-adc4-4e25-5a00-a928c10fd24c Address:e3d918a9-adc4-4e25-5a00-a928c10fd24c}]" +--- PASS: TestRaft_SnapshotRestore (0.19s) === RUN TestRaft_NoRestoreOnStart -testLoggerAdapter verbose: server-98400f3d-bf44-68b0-12a6-9a833d849c4b: 2025-02-09T06:52:29.746Z [INFO] server-98400f3d-bf44-68b0-12a6-9a833d849c4b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-98400f3d-bf44-68b0-12a6-9a833d849c4b Address:98400f3d-bf44-68b0-12a6-9a833d849c4b}]" - testing.go:149: server-98400f3d-bf44-68b0-12a6-9a833d849c4b: 2025-02-09T06:52:29.746Z [INFO] server-98400f3d-bf44-68b0-12a6-9a833d849c4b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-98400f3d-bf44-68b0-12a6-9a833d849c4b Address:98400f3d-bf44-68b0-12a6-9a833d849c4b}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:29.746Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:29.746Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-98400f3d-bf44-68b0-12a6-9a833d849c4b: 2025-02-09T06:52:29.746Z [INFO] server-98400f3d-bf44-68b0-12a6-9a833d849c4b: entering follower state: follower="Node at 98400f3d-bf44-68b0-12a6-9a833d849c4b [Follower]" leader= - testing.go:149: server-98400f3d-bf44-68b0-12a6-9a833d849c4b: 2025-02-09T06:52:29.746Z [INFO] server-98400f3d-bf44-68b0-12a6-9a833d849c4b: entering follower state: follower="Node at 98400f3d-bf44-68b0-12a6-9a833d849c4b [Follower]" leader= -testLoggerAdapter verbose: server-98400f3d-bf44-68b0-12a6-9a833d849c4b: 2025-02-09T06:52:29.812Z [WARN] server-98400f3d-bf44-68b0-12a6-9a833d849c4b: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-98400f3d-bf44-68b0-12a6-9a833d849c4b: 2025-02-09T06:52:29.812Z [WARN] server-98400f3d-bf44-68b0-12a6-9a833d849c4b: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-98400f3d-bf44-68b0-12a6-9a833d849c4b: 2025-02-09T06:52:29.812Z [INFO] server-98400f3d-bf44-68b0-12a6-9a833d849c4b: entering candidate state: node="Node at 98400f3d-bf44-68b0-12a6-9a833d849c4b [Candidate]" term=2 - testing.go:149: server-98400f3d-bf44-68b0-12a6-9a833d849c4b: 2025-02-09T06:52:29.812Z [INFO] server-98400f3d-bf44-68b0-12a6-9a833d849c4b: entering candidate state: node="Node at 98400f3d-bf44-68b0-12a6-9a833d849c4b [Candidate]" term=2 -testLoggerAdapter verbose: server-98400f3d-bf44-68b0-12a6-9a833d849c4b: 2025-02-09T06:52:29.813Z [INFO] server-98400f3d-bf44-68b0-12a6-9a833d849c4b: election won: tally=1 - testing.go:149: server-98400f3d-bf44-68b0-12a6-9a833d849c4b: 2025-02-09T06:52:29.813Z [INFO] server-98400f3d-bf44-68b0-12a6-9a833d849c4b: election won: tally=1 -testLoggerAdapter verbose: server-98400f3d-bf44-68b0-12a6-9a833d849c4b: 2025-02-09T06:52:29.813Z [INFO] server-98400f3d-bf44-68b0-12a6-9a833d849c4b: entering leader state: leader="Node at 98400f3d-bf44-68b0-12a6-9a833d849c4b [Leader]" - testing.go:149: server-98400f3d-bf44-68b0-12a6-9a833d849c4b: 2025-02-09T06:52:29.813Z [INFO] server-98400f3d-bf44-68b0-12a6-9a833d849c4b: entering leader state: leader="Node at 98400f3d-bf44-68b0-12a6-9a833d849c4b [Leader]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:29.919Z [INFO] stable state for Leader reached at 2025-02-09 06:52:29.813087994 +0000 UTC m=+10.791896756 (1 nodes), 66.917737ms from start of poll, 67.114562ms from cluster start. Timeout at 2025-02-09 06:52:29.919018331 +0000 UTC m=+10.897827100, 105.930344ms after stability - testing.go:149: cluster: 2025-02-09T06:52:29.919Z [INFO] stable state for Leader reached at 2025-02-09 06:52:29.813087994 +0000 UTC m=+10.791896756 (1 nodes), 66.917737ms from start of poll, 67.114562ms from cluster start. Timeout at 2025-02-09 06:52:29.919018331 +0000 UTC m=+10.897827100, 105.930344ms after stability -testLoggerAdapter verbose: server-98400f3d-bf44-68b0-12a6-9a833d849c4b: 2025-02-09T06:52:29.919Z [INFO] server-98400f3d-bf44-68b0-12a6-9a833d849c4b: starting snapshot up to: index=102 - testing.go:149: server-98400f3d-bf44-68b0-12a6-9a833d849c4b: 2025-02-09T06:52:29.919Z [INFO] server-98400f3d-bf44-68b0-12a6-9a833d849c4b: starting snapshot up to: index=102 - testing.go:153: 2025-02-09T06:52:29.919Z [INFO] creating new snapshot: path=/tmp/raft930944203/snapshots/2-102-1739083949919.tmp -testLoggerAdapter verbose: server-98400f3d-bf44-68b0-12a6-9a833d849c4b: 2025-02-09T06:52:29.923Z [INFO] server-98400f3d-bf44-68b0-12a6-9a833d849c4b: compacting logs: from=1 to=92 - testing.go:149: server-98400f3d-bf44-68b0-12a6-9a833d849c4b: 2025-02-09T06:52:29.923Z [INFO] server-98400f3d-bf44-68b0-12a6-9a833d849c4b: compacting logs: from=1 to=92 -testLoggerAdapter verbose: server-98400f3d-bf44-68b0-12a6-9a833d849c4b: 2025-02-09T06:52:29.923Z [INFO] server-98400f3d-bf44-68b0-12a6-9a833d849c4b: snapshot complete up to: index=102 - testing.go:149: server-98400f3d-bf44-68b0-12a6-9a833d849c4b: 2025-02-09T06:52:29.923Z [INFO] server-98400f3d-bf44-68b0-12a6-9a833d849c4b: snapshot complete up to: index=102 -testLoggerAdapter verbose: server-98400f3d-bf44-68b0-12a6-9a833d849c4b: 2025-02-09T06:52:29.923Z [INFO] server-98400f3d-bf44-68b0-12a6-9a833d849c4b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-98400f3d-bf44-68b0-12a6-9a833d849c4b Address:98400f3d-bf44-68b0-12a6-9a833d849c4b}]" - testing.go:149: server-98400f3d-bf44-68b0-12a6-9a833d849c4b: 2025-02-09T06:52:29.923Z [INFO] server-98400f3d-bf44-68b0-12a6-9a833d849c4b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-98400f3d-bf44-68b0-12a6-9a833d849c4b Address:98400f3d-bf44-68b0-12a6-9a833d849c4b}]" ---- PASS: TestRaft_NoRestoreOnStart (0.18s) +testLoggerAdapter verbose: server-46420970-2970-2c8c-2ac8-09bd60779bd4: 2024-01-08T00:31:04.616Z [INFO] server-46420970-2970-2c8c-2ac8-09bd60779bd4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-46420970-2970-2c8c-2ac8-09bd60779bd4 Address:46420970-2970-2c8c-2ac8-09bd60779bd4}]" + testing.go:149: server-46420970-2970-2c8c-2ac8-09bd60779bd4: 2024-01-08T00:31:04.616Z [INFO] server-46420970-2970-2c8c-2ac8-09bd60779bd4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-46420970-2970-2c8c-2ac8-09bd60779bd4 Address:46420970-2970-2c8c-2ac8-09bd60779bd4}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:04.616Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:04.616Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-46420970-2970-2c8c-2ac8-09bd60779bd4: 2024-01-08T00:31:04.616Z [INFO] server-46420970-2970-2c8c-2ac8-09bd60779bd4: entering follower state: follower="Node at 46420970-2970-2c8c-2ac8-09bd60779bd4 [Follower]" leader= + testing.go:149: server-46420970-2970-2c8c-2ac8-09bd60779bd4: 2024-01-08T00:31:04.616Z [INFO] server-46420970-2970-2c8c-2ac8-09bd60779bd4: entering follower state: follower="Node at 46420970-2970-2c8c-2ac8-09bd60779bd4 [Follower]" leader= +testLoggerAdapter verbose: server-46420970-2970-2c8c-2ac8-09bd60779bd4: 2024-01-08T00:31:04.692Z [WARN] server-46420970-2970-2c8c-2ac8-09bd60779bd4: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-46420970-2970-2c8c-2ac8-09bd60779bd4: 2024-01-08T00:31:04.692Z [WARN] server-46420970-2970-2c8c-2ac8-09bd60779bd4: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-46420970-2970-2c8c-2ac8-09bd60779bd4: 2024-01-08T00:31:04.692Z [INFO] server-46420970-2970-2c8c-2ac8-09bd60779bd4: entering candidate state: node="Node at 46420970-2970-2c8c-2ac8-09bd60779bd4 [Candidate]" term=2 + testing.go:149: server-46420970-2970-2c8c-2ac8-09bd60779bd4: 2024-01-08T00:31:04.692Z [INFO] server-46420970-2970-2c8c-2ac8-09bd60779bd4: entering candidate state: node="Node at 46420970-2970-2c8c-2ac8-09bd60779bd4 [Candidate]" term=2 +testLoggerAdapter verbose: server-46420970-2970-2c8c-2ac8-09bd60779bd4: 2024-01-08T00:31:04.692Z [INFO] server-46420970-2970-2c8c-2ac8-09bd60779bd4: election won: tally=1 + testing.go:149: server-46420970-2970-2c8c-2ac8-09bd60779bd4: 2024-01-08T00:31:04.692Z [INFO] server-46420970-2970-2c8c-2ac8-09bd60779bd4: election won: tally=1 +testLoggerAdapter verbose: server-46420970-2970-2c8c-2ac8-09bd60779bd4: 2024-01-08T00:31:04.692Z [INFO] server-46420970-2970-2c8c-2ac8-09bd60779bd4: entering leader state: leader="Node at 46420970-2970-2c8c-2ac8-09bd60779bd4 [Leader]" + testing.go:149: server-46420970-2970-2c8c-2ac8-09bd60779bd4: 2024-01-08T00:31:04.692Z [INFO] server-46420970-2970-2c8c-2ac8-09bd60779bd4: entering leader state: leader="Node at 46420970-2970-2c8c-2ac8-09bd60779bd4 [Leader]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:04.798Z [INFO] stable state for Leader reached at 2024-01-08 00:31:04.692822191 +0000 UTC m=+10.739711250 (1 nodes), 76.679261ms from start of poll, 76.80049ms from cluster start. Timeout at 2024-01-08 00:31:04.798355828 +0000 UTC m=+10.845244931, 105.533681ms after stability + testing.go:149: cluster: 2024-01-08T00:31:04.798Z [INFO] stable state for Leader reached at 2024-01-08 00:31:04.692822191 +0000 UTC m=+10.739711250 (1 nodes), 76.679261ms from start of poll, 76.80049ms from cluster start. Timeout at 2024-01-08 00:31:04.798355828 +0000 UTC m=+10.845244931, 105.533681ms after stability +testLoggerAdapter verbose: server-46420970-2970-2c8c-2ac8-09bd60779bd4: 2024-01-08T00:31:04.799Z [INFO] server-46420970-2970-2c8c-2ac8-09bd60779bd4: starting snapshot up to: index=102 + testing.go:149: server-46420970-2970-2c8c-2ac8-09bd60779bd4: 2024-01-08T00:31:04.799Z [INFO] server-46420970-2970-2c8c-2ac8-09bd60779bd4: starting snapshot up to: index=102 + testing.go:153: 2024-01-08T00:31:04.799Z [INFO] creating new snapshot: path=/tmp/raft763335452/snapshots/2-102-1704673864799.tmp +testLoggerAdapter verbose: server-46420970-2970-2c8c-2ac8-09bd60779bd4: 2024-01-08T00:31:04.808Z [INFO] server-46420970-2970-2c8c-2ac8-09bd60779bd4: compacting logs: from=1 to=92 + testing.go:149: server-46420970-2970-2c8c-2ac8-09bd60779bd4: 2024-01-08T00:31:04.808Z [INFO] server-46420970-2970-2c8c-2ac8-09bd60779bd4: compacting logs: from=1 to=92 +testLoggerAdapter verbose: server-46420970-2970-2c8c-2ac8-09bd60779bd4: 2024-01-08T00:31:04.808Z [INFO] server-46420970-2970-2c8c-2ac8-09bd60779bd4: snapshot complete up to: index=102 + testing.go:149: server-46420970-2970-2c8c-2ac8-09bd60779bd4: 2024-01-08T00:31:04.808Z [INFO] server-46420970-2970-2c8c-2ac8-09bd60779bd4: snapshot complete up to: index=102 +testLoggerAdapter verbose: server-46420970-2970-2c8c-2ac8-09bd60779bd4: 2024-01-08T00:31:04.808Z [INFO] server-46420970-2970-2c8c-2ac8-09bd60779bd4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-46420970-2970-2c8c-2ac8-09bd60779bd4 Address:46420970-2970-2c8c-2ac8-09bd60779bd4}]" + testing.go:149: server-46420970-2970-2c8c-2ac8-09bd60779bd4: 2024-01-08T00:31:04.808Z [INFO] server-46420970-2970-2c8c-2ac8-09bd60779bd4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-46420970-2970-2c8c-2ac8-09bd60779bd4 Address:46420970-2970-2c8c-2ac8-09bd60779bd4}]" +--- PASS: TestRaft_NoRestoreOnStart (0.19s) === RUN TestRaft_SnapshotRestore_PeerChange raft_test.go:1026: Disabled unreliable test --- SKIP: TestRaft_SnapshotRestore_PeerChange (0.00s) @@ -2154,1751 +2208,1863 @@ raft_test.go:1132: Disabled unreliable test --- SKIP: TestRaft_AutoSnapshot (0.00s) === RUN TestRaft_UserSnapshot -testLoggerAdapter verbose: server-98400f3d-bf44-68b0-12a6-9a833d849c4b: 2025-02-09T06:52:29.923Z [INFO] server-98400f3d-bf44-68b0-12a6-9a833d849c4b: entering follower state: follower="Node at 98400f3d-bf44-68b0-12a6-9a833d849c4b [Follower]" leader= -testLoggerAdapter verbose: server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: 2025-02-09T06:52:29.924Z [INFO] server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a Address:a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a}]" - testing.go:149: server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: 2025-02-09T06:52:29.924Z [INFO] server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a Address:a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:29.924Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:29.924Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: 2025-02-09T06:52:29.924Z [INFO] server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: entering follower state: follower="Node at a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a [Follower]" leader= - testing.go:149: server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: 2025-02-09T06:52:29.924Z [INFO] server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: entering follower state: follower="Node at a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a [Follower]" leader= -testLoggerAdapter verbose: server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: 2025-02-09T06:52:29.981Z [WARN] server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: 2025-02-09T06:52:29.981Z [WARN] server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: 2025-02-09T06:52:29.981Z [INFO] server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: entering candidate state: node="Node at a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a [Candidate]" term=2 - testing.go:149: server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: 2025-02-09T06:52:29.981Z [INFO] server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: entering candidate state: node="Node at a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a [Candidate]" term=2 -testLoggerAdapter verbose: server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: 2025-02-09T06:52:29.981Z [INFO] server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: election won: tally=1 - testing.go:149: server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: 2025-02-09T06:52:29.981Z [INFO] server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: election won: tally=1 -testLoggerAdapter verbose: server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: 2025-02-09T06:52:29.981Z [INFO] server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: entering leader state: leader="Node at a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a [Leader]" - testing.go:149: server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: 2025-02-09T06:52:29.981Z [INFO] server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: entering leader state: leader="Node at a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a [Leader]" -testLoggerAdapter verbose: server-98400f3d-bf44-68b0-12a6-9a833d849c4b: 2025-02-09T06:52:30.002Z [WARN] server-98400f3d-bf44-68b0-12a6-9a833d849c4b: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-98400f3d-bf44-68b0-12a6-9a833d849c4b: 2025-02-09T06:52:30.002Z [INFO] server-98400f3d-bf44-68b0-12a6-9a833d849c4b: entering candidate state: node="Node at 98400f3d-bf44-68b0-12a6-9a833d849c4b [Candidate]" term=3 -testLoggerAdapter verbose: server-98400f3d-bf44-68b0-12a6-9a833d849c4b: 2025-02-09T06:52:30.002Z [INFO] server-98400f3d-bf44-68b0-12a6-9a833d849c4b: election won: tally=1 -testLoggerAdapter verbose: server-98400f3d-bf44-68b0-12a6-9a833d849c4b: 2025-02-09T06:52:30.002Z [INFO] server-98400f3d-bf44-68b0-12a6-9a833d849c4b: entering leader state: leader="Node at 98400f3d-bf44-68b0-12a6-9a833d849c4b [Leader]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:30.087Z [INFO] stable state for Leader reached at 2025-02-09 06:52:29.981581866 +0000 UTC m=+10.960390645 (1 nodes), 57.239174ms from start of poll, 57.376655ms from cluster start. Timeout at 2025-02-09 06:52:30.087036947 +0000 UTC m=+11.065845715, 105.45507ms after stability - testing.go:149: cluster: 2025-02-09T06:52:30.087Z [INFO] stable state for Leader reached at 2025-02-09 06:52:29.981581866 +0000 UTC m=+10.960390645 (1 nodes), 57.239174ms from start of poll, 57.376655ms from cluster start. Timeout at 2025-02-09 06:52:30.087036947 +0000 UTC m=+11.065845715, 105.45507ms after stability -testLoggerAdapter verbose: server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: 2025-02-09T06:52:30.087Z [ERROR] server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: failed to take snapshot: error="nothing new to snapshot" - testing.go:149: server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: 2025-02-09T06:52:30.087Z [ERROR] server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: failed to take snapshot: error="nothing new to snapshot" -testLoggerAdapter verbose: server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: 2025-02-09T06:52:30.087Z [INFO] server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: starting snapshot up to: index=12 - testing.go:149: server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: 2025-02-09T06:52:30.087Z [INFO] server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: starting snapshot up to: index=12 - testing.go:153: 2025-02-09T06:52:30.087Z [INFO] creating new snapshot: path=/tmp/raft276410526/snapshots/2-12-1739083950087.tmp -testLoggerAdapter verbose: server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: 2025-02-09T06:52:30.091Z [INFO] server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: compacting logs: from=1 to=2 - testing.go:149: server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: 2025-02-09T06:52:30.091Z [INFO] server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: compacting logs: from=1 to=2 -testLoggerAdapter verbose: server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: 2025-02-09T06:52:30.091Z [INFO] server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: snapshot complete up to: index=12 - testing.go:149: server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: 2025-02-09T06:52:30.091Z [INFO] server-a9d432a2-42a5-0fb4-e0ca-5ff591e6cc7a: snapshot complete up to: index=12 ---- PASS: TestRaft_UserSnapshot (0.17s) +testLoggerAdapter verbose: server-399e17c6-176c-5779-808a-568b0affb669: 2024-01-08T00:31:04.808Z [INFO] server-399e17c6-176c-5779-808a-568b0affb669: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-399e17c6-176c-5779-808a-568b0affb669 Address:399e17c6-176c-5779-808a-568b0affb669}]" + testing.go:149: server-399e17c6-176c-5779-808a-568b0affb669: 2024-01-08T00:31:04.808Z [INFO] server-399e17c6-176c-5779-808a-568b0affb669: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-399e17c6-176c-5779-808a-568b0affb669 Address:399e17c6-176c-5779-808a-568b0affb669}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:04.808Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:04.808Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-46420970-2970-2c8c-2ac8-09bd60779bd4: 2024-01-08T00:31:04.808Z [INFO] server-46420970-2970-2c8c-2ac8-09bd60779bd4: entering follower state: follower="Node at 46420970-2970-2c8c-2ac8-09bd60779bd4 [Follower]" leader= +testLoggerAdapter verbose: server-399e17c6-176c-5779-808a-568b0affb669: 2024-01-08T00:31:04.808Z [INFO] server-399e17c6-176c-5779-808a-568b0affb669: entering follower state: follower="Node at 399e17c6-176c-5779-808a-568b0affb669 [Follower]" leader= + testing.go:149: server-399e17c6-176c-5779-808a-568b0affb669: 2024-01-08T00:31:04.808Z [INFO] server-399e17c6-176c-5779-808a-568b0affb669: entering follower state: follower="Node at 399e17c6-176c-5779-808a-568b0affb669 [Follower]" leader= +testLoggerAdapter verbose: server-46420970-2970-2c8c-2ac8-09bd60779bd4: 2024-01-08T00:31:04.860Z [WARN] server-46420970-2970-2c8c-2ac8-09bd60779bd4: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-46420970-2970-2c8c-2ac8-09bd60779bd4: 2024-01-08T00:31:04.860Z [INFO] server-46420970-2970-2c8c-2ac8-09bd60779bd4: entering candidate state: node="Node at 46420970-2970-2c8c-2ac8-09bd60779bd4 [Candidate]" term=3 +testLoggerAdapter verbose: server-46420970-2970-2c8c-2ac8-09bd60779bd4: 2024-01-08T00:31:04.860Z [INFO] server-46420970-2970-2c8c-2ac8-09bd60779bd4: election won: tally=1 +testLoggerAdapter verbose: server-46420970-2970-2c8c-2ac8-09bd60779bd4: 2024-01-08T00:31:04.860Z [INFO] server-46420970-2970-2c8c-2ac8-09bd60779bd4: entering leader state: leader="Node at 46420970-2970-2c8c-2ac8-09bd60779bd4 [Leader]" +testLoggerAdapter verbose: server-399e17c6-176c-5779-808a-568b0affb669: 2024-01-08T00:31:04.902Z [WARN] server-399e17c6-176c-5779-808a-568b0affb669: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-399e17c6-176c-5779-808a-568b0affb669: 2024-01-08T00:31:04.902Z [WARN] server-399e17c6-176c-5779-808a-568b0affb669: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-399e17c6-176c-5779-808a-568b0affb669: 2024-01-08T00:31:04.902Z [INFO] server-399e17c6-176c-5779-808a-568b0affb669: entering candidate state: node="Node at 399e17c6-176c-5779-808a-568b0affb669 [Candidate]" term=2 + testing.go:149: server-399e17c6-176c-5779-808a-568b0affb669: 2024-01-08T00:31:04.902Z [INFO] server-399e17c6-176c-5779-808a-568b0affb669: entering candidate state: node="Node at 399e17c6-176c-5779-808a-568b0affb669 [Candidate]" term=2 +testLoggerAdapter verbose: server-399e17c6-176c-5779-808a-568b0affb669: 2024-01-08T00:31:04.902Z [INFO] server-399e17c6-176c-5779-808a-568b0affb669: election won: tally=1 + testing.go:149: server-399e17c6-176c-5779-808a-568b0affb669: 2024-01-08T00:31:04.902Z [INFO] server-399e17c6-176c-5779-808a-568b0affb669: election won: tally=1 +testLoggerAdapter verbose: server-399e17c6-176c-5779-808a-568b0affb669: 2024-01-08T00:31:04.902Z [INFO] server-399e17c6-176c-5779-808a-568b0affb669: entering leader state: leader="Node at 399e17c6-176c-5779-808a-568b0affb669 [Leader]" + testing.go:149: server-399e17c6-176c-5779-808a-568b0affb669: 2024-01-08T00:31:04.902Z [INFO] server-399e17c6-176c-5779-808a-568b0affb669: entering leader state: leader="Node at 399e17c6-176c-5779-808a-568b0affb669 [Leader]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:05.007Z [INFO] stable state for Leader reached at 2024-01-08 00:31:04.902449631 +0000 UTC m=+10.949338685 (1 nodes), 93.726439ms from start of poll, 93.8615ms from cluster start. Timeout at 2024-01-08 00:31:05.007491179 +0000 UTC m=+11.054380275, 105.04159ms after stability + testing.go:149: cluster: 2024-01-08T00:31:05.007Z [INFO] stable state for Leader reached at 2024-01-08 00:31:04.902449631 +0000 UTC m=+10.949338685 (1 nodes), 93.726439ms from start of poll, 93.8615ms from cluster start. Timeout at 2024-01-08 00:31:05.007491179 +0000 UTC m=+11.054380275, 105.04159ms after stability +testLoggerAdapter verbose: server-399e17c6-176c-5779-808a-568b0affb669: 2024-01-08T00:31:05.007Z [ERROR] server-399e17c6-176c-5779-808a-568b0affb669: failed to take snapshot: error="nothing new to snapshot" + testing.go:149: server-399e17c6-176c-5779-808a-568b0affb669: 2024-01-08T00:31:05.007Z [ERROR] server-399e17c6-176c-5779-808a-568b0affb669: failed to take snapshot: error="nothing new to snapshot" +testLoggerAdapter verbose: server-399e17c6-176c-5779-808a-568b0affb669: 2024-01-08T00:31:05.007Z [INFO] server-399e17c6-176c-5779-808a-568b0affb669: starting snapshot up to: index=12 + testing.go:149: server-399e17c6-176c-5779-808a-568b0affb669: 2024-01-08T00:31:05.007Z [INFO] server-399e17c6-176c-5779-808a-568b0affb669: starting snapshot up to: index=12 + testing.go:153: 2024-01-08T00:31:05.007Z [INFO] creating new snapshot: path=/tmp/raft3724544202/snapshots/2-12-1704673865007.tmp +testLoggerAdapter verbose: server-399e17c6-176c-5779-808a-568b0affb669: 2024-01-08T00:31:05.017Z [INFO] server-399e17c6-176c-5779-808a-568b0affb669: compacting logs: from=1 to=2 + testing.go:149: server-399e17c6-176c-5779-808a-568b0affb669: 2024-01-08T00:31:05.017Z [INFO] server-399e17c6-176c-5779-808a-568b0affb669: compacting logs: from=1 to=2 +testLoggerAdapter verbose: server-399e17c6-176c-5779-808a-568b0affb669: 2024-01-08T00:31:05.017Z [INFO] server-399e17c6-176c-5779-808a-568b0affb669: snapshot complete up to: index=12 + testing.go:149: server-399e17c6-176c-5779-808a-568b0affb669: 2024-01-08T00:31:05.017Z [INFO] server-399e17c6-176c-5779-808a-568b0affb669: snapshot complete up to: index=12 +--- PASS: TestRaft_UserSnapshot (0.21s) === RUN TestRaft_UserRestore -testLoggerAdapter verbose: server-90870093-0964-9a2e-1b19-e08e62de00f7: 2025-02-09T06:52:30.092Z [INFO] server-90870093-0964-9a2e-1b19-e08e62de00f7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-90870093-0964-9a2e-1b19-e08e62de00f7 Address:90870093-0964-9a2e-1b19-e08e62de00f7} {Suffrage:Voter ID:server-776ce372-c6a9-bb52-cddd-a76589382d7b Address:776ce372-c6a9-bb52-cddd-a76589382d7b} {Suffrage:Voter ID:server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920 Address:c7f5cb9b-efca-1d3f-e539-e4b29ae2e920}]" - testing.go:149: server-90870093-0964-9a2e-1b19-e08e62de00f7: 2025-02-09T06:52:30.092Z [INFO] server-90870093-0964-9a2e-1b19-e08e62de00f7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-90870093-0964-9a2e-1b19-e08e62de00f7 Address:90870093-0964-9a2e-1b19-e08e62de00f7} {Suffrage:Voter ID:server-776ce372-c6a9-bb52-cddd-a76589382d7b Address:776ce372-c6a9-bb52-cddd-a76589382d7b} {Suffrage:Voter ID:server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920 Address:c7f5cb9b-efca-1d3f-e539-e4b29ae2e920}]" -testLoggerAdapter verbose: server-90870093-0964-9a2e-1b19-e08e62de00f7: 2025-02-09T06:52:30.092Z [INFO] server-90870093-0964-9a2e-1b19-e08e62de00f7: entering follower state: follower="Node at 90870093-0964-9a2e-1b19-e08e62de00f7 [Follower]" leader= - testing.go:149: server-90870093-0964-9a2e-1b19-e08e62de00f7: 2025-02-09T06:52:30.092Z [INFO] server-90870093-0964-9a2e-1b19-e08e62de00f7: entering follower state: follower="Node at 90870093-0964-9a2e-1b19-e08e62de00f7 [Follower]" leader= -testLoggerAdapter verbose: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.092Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-90870093-0964-9a2e-1b19-e08e62de00f7 Address:90870093-0964-9a2e-1b19-e08e62de00f7} {Suffrage:Voter ID:server-776ce372-c6a9-bb52-cddd-a76589382d7b Address:776ce372-c6a9-bb52-cddd-a76589382d7b} {Suffrage:Voter ID:server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920 Address:c7f5cb9b-efca-1d3f-e539-e4b29ae2e920}]" - testing.go:149: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.092Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-90870093-0964-9a2e-1b19-e08e62de00f7 Address:90870093-0964-9a2e-1b19-e08e62de00f7} {Suffrage:Voter ID:server-776ce372-c6a9-bb52-cddd-a76589382d7b Address:776ce372-c6a9-bb52-cddd-a76589382d7b} {Suffrage:Voter ID:server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920 Address:c7f5cb9b-efca-1d3f-e539-e4b29ae2e920}]" -testLoggerAdapter verbose: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.092Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: entering follower state: follower="Node at 776ce372-c6a9-bb52-cddd-a76589382d7b [Follower]" leader= - testing.go:149: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.092Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: entering follower state: follower="Node at 776ce372-c6a9-bb52-cddd-a76589382d7b [Follower]" leader= -testLoggerAdapter verbose: server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920: 2025-02-09T06:52:30.092Z [INFO] server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-90870093-0964-9a2e-1b19-e08e62de00f7 Address:90870093-0964-9a2e-1b19-e08e62de00f7} {Suffrage:Voter ID:server-776ce372-c6a9-bb52-cddd-a76589382d7b Address:776ce372-c6a9-bb52-cddd-a76589382d7b} {Suffrage:Voter ID:server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920 Address:c7f5cb9b-efca-1d3f-e539-e4b29ae2e920}]" - testing.go:149: server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920: 2025-02-09T06:52:30.092Z [INFO] server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-90870093-0964-9a2e-1b19-e08e62de00f7 Address:90870093-0964-9a2e-1b19-e08e62de00f7} {Suffrage:Voter ID:server-776ce372-c6a9-bb52-cddd-a76589382d7b Address:776ce372-c6a9-bb52-cddd-a76589382d7b} {Suffrage:Voter ID:server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920 Address:c7f5cb9b-efca-1d3f-e539-e4b29ae2e920}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:30.092Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:30.092Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920: 2025-02-09T06:52:30.092Z [INFO] server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920: entering follower state: follower="Node at c7f5cb9b-efca-1d3f-e539-e4b29ae2e920 [Follower]" leader= - testing.go:149: server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920: 2025-02-09T06:52:30.092Z [INFO] server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920: entering follower state: follower="Node at c7f5cb9b-efca-1d3f-e539-e4b29ae2e920 [Follower]" leader= -testLoggerAdapter verbose: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.175Z [WARN] server-776ce372-c6a9-bb52-cddd-a76589382d7b: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.175Z [WARN] server-776ce372-c6a9-bb52-cddd-a76589382d7b: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.176Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: entering candidate state: node="Node at 776ce372-c6a9-bb52-cddd-a76589382d7b [Candidate]" term=2 - testing.go:149: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.176Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: entering candidate state: node="Node at 776ce372-c6a9-bb52-cddd-a76589382d7b [Candidate]" term=2 -testLoggerAdapter verbose: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.176Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: election won: tally=2 - testing.go:149: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.176Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: election won: tally=2 -testLoggerAdapter verbose: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.176Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: entering leader state: leader="Node at 776ce372-c6a9-bb52-cddd-a76589382d7b [Leader]" - testing.go:149: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.176Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: entering leader state: leader="Node at 776ce372-c6a9-bb52-cddd-a76589382d7b [Leader]" -testLoggerAdapter verbose: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.176Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: added peer, starting replication: peer=server-90870093-0964-9a2e-1b19-e08e62de00f7 - testing.go:149: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.176Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: added peer, starting replication: peer=server-90870093-0964-9a2e-1b19-e08e62de00f7 -testLoggerAdapter verbose: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.176Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: added peer, starting replication: peer=server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920 - testing.go:149: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.176Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: added peer, starting replication: peer=server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920 -testLoggerAdapter verbose: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.176Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: pipelining replication: peer="{Voter server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920 c7f5cb9b-efca-1d3f-e539-e4b29ae2e920}" - testing.go:149: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.176Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: pipelining replication: peer="{Voter server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920 c7f5cb9b-efca-1d3f-e539-e4b29ae2e920}" -testLoggerAdapter verbose: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.176Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: pipelining replication: peer="{Voter server-90870093-0964-9a2e-1b19-e08e62de00f7 90870093-0964-9a2e-1b19-e08e62de00f7}" - testing.go:149: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.176Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: pipelining replication: peer="{Voter server-90870093-0964-9a2e-1b19-e08e62de00f7 90870093-0964-9a2e-1b19-e08e62de00f7}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:30.281Z [INFO] stable state for Leader reached at 2025-02-09 06:52:30.176121262 +0000 UTC m=+11.154930049 (1 nodes), 83.447919ms from start of poll, 83.794836ms from cluster start. Timeout at 2025-02-09 06:52:30.281967019 +0000 UTC m=+11.260775792, 105.845743ms after stability - testing.go:149: cluster: 2025-02-09T06:52:30.281Z [INFO] stable state for Leader reached at 2025-02-09 06:52:30.176121262 +0000 UTC m=+11.154930049 (1 nodes), 83.447919ms from start of poll, 83.794836ms from cluster start. Timeout at 2025-02-09 06:52:30.281967019 +0000 UTC m=+11.260775792, 105.845743ms after stability -testLoggerAdapter verbose: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.282Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: starting snapshot up to: index=12 - testing.go:149: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.282Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: starting snapshot up to: index=12 - testing.go:153: 2025-02-09T06:52:30.282Z [INFO] creating new snapshot: path=/tmp/raft472498701/snapshots/2-12-1739083950282.tmp -testLoggerAdapter verbose: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.286Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: snapshot complete up to: index=12 - testing.go:149: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.286Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: snapshot complete up to: index=12 - testing.go:153: 2025-02-09T06:52:30.287Z [INFO] creating new snapshot: path=/tmp/raft472498701/snapshots/2-23-1739083950287.tmp -testLoggerAdapter verbose: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.290Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: copied to local snapshot: bytes=71 - testing.go:149: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.290Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.290Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: restored user snapshot: index=1 - testing.go:149: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.290Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: restored user snapshot: index=1 -testLoggerAdapter verbose: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.290Z [ERROR] server-776ce372-c6a9-bb52-cddd-a76589382d7b: failed to get log: index=23 error="log not found" - testing.go:149: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.290Z [ERROR] server-776ce372-c6a9-bb52-cddd-a76589382d7b: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.290Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: aborting pipeline replication: peer="{Voter server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920 c7f5cb9b-efca-1d3f-e539-e4b29ae2e920}" - testing.go:149: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.290Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: aborting pipeline replication: peer="{Voter server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920 c7f5cb9b-efca-1d3f-e539-e4b29ae2e920}" -testLoggerAdapter verbose: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.290Z [ERROR] server-776ce372-c6a9-bb52-cddd-a76589382d7b: failed to get log: index=23 error="log not found" - testing.go:149: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.290Z [ERROR] server-776ce372-c6a9-bb52-cddd-a76589382d7b: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.290Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: aborting pipeline replication: peer="{Voter server-90870093-0964-9a2e-1b19-e08e62de00f7 90870093-0964-9a2e-1b19-e08e62de00f7}" - testing.go:149: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.290Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: aborting pipeline replication: peer="{Voter server-90870093-0964-9a2e-1b19-e08e62de00f7 90870093-0964-9a2e-1b19-e08e62de00f7}" -testLoggerAdapter verbose: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.298Z [ERROR] server-776ce372-c6a9-bb52-cddd-a76589382d7b: failed to get log: index=23 error="log not found" - testing.go:149: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.298Z [ERROR] server-776ce372-c6a9-bb52-cddd-a76589382d7b: failed to get log: index=23 error="log not found" - testing.go:153: 2025-02-09T06:52:30.298Z [INFO] creating new snapshot: path=/tmp/raft1122326152/snapshots/2-23-1739083950298.tmp -testLoggerAdapter verbose: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.300Z [ERROR] server-776ce372-c6a9-bb52-cddd-a76589382d7b: failed to get log: index=23 error="log not found" - testing.go:149: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.300Z [ERROR] server-776ce372-c6a9-bb52-cddd-a76589382d7b: failed to get log: index=23 error="log not found" - testing.go:153: 2025-02-09T06:52:30.300Z [INFO] creating new snapshot: path=/tmp/raft1536740690/snapshots/2-23-1739083950300.tmp -testLoggerAdapter verbose: server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920: 2025-02-09T06:52:30.302Z [INFO] server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920: copied to local snapshot: bytes=71 - testing.go:149: server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920: 2025-02-09T06:52:30.302Z [INFO] server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920: 2025-02-09T06:52:30.302Z [INFO] server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920: Installed remote snapshot - testing.go:149: server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920: 2025-02-09T06:52:30.302Z [INFO] server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920: Installed remote snapshot -testLoggerAdapter verbose: server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920: 2025-02-09T06:52:30.302Z [WARN] server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920: skipping application of old log: index=22 - testing.go:149: server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920: 2025-02-09T06:52:30.302Z [WARN] server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920: skipping application of old log: index=22 -testLoggerAdapter verbose: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.302Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: pipelining replication: peer="{Voter server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920 c7f5cb9b-efca-1d3f-e539-e4b29ae2e920}" - testing.go:149: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.302Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: pipelining replication: peer="{Voter server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920 c7f5cb9b-efca-1d3f-e539-e4b29ae2e920}" -testLoggerAdapter verbose: server-90870093-0964-9a2e-1b19-e08e62de00f7: 2025-02-09T06:52:30.303Z [INFO] server-90870093-0964-9a2e-1b19-e08e62de00f7: copied to local snapshot: bytes=71 - testing.go:149: server-90870093-0964-9a2e-1b19-e08e62de00f7: 2025-02-09T06:52:30.303Z [INFO] server-90870093-0964-9a2e-1b19-e08e62de00f7: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-90870093-0964-9a2e-1b19-e08e62de00f7: 2025-02-09T06:52:30.304Z [INFO] server-90870093-0964-9a2e-1b19-e08e62de00f7: Installed remote snapshot - testing.go:149: server-90870093-0964-9a2e-1b19-e08e62de00f7: 2025-02-09T06:52:30.304Z [INFO] server-90870093-0964-9a2e-1b19-e08e62de00f7: Installed remote snapshot -testLoggerAdapter verbose: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.304Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: pipelining replication: peer="{Voter server-90870093-0964-9a2e-1b19-e08e62de00f7 90870093-0964-9a2e-1b19-e08e62de00f7}" - testing.go:149: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.304Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: pipelining replication: peer="{Voter server-90870093-0964-9a2e-1b19-e08e62de00f7 90870093-0964-9a2e-1b19-e08e62de00f7}" -testLoggerAdapter verbose: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.320Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: aborting pipeline replication: peer="{Voter server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920 c7f5cb9b-efca-1d3f-e539-e4b29ae2e920}" - testing.go:149: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.320Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: aborting pipeline replication: peer="{Voter server-c7f5cb9b-efca-1d3f-e539-e4b29ae2e920 c7f5cb9b-efca-1d3f-e539-e4b29ae2e920}" -testLoggerAdapter verbose: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.320Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: aborting pipeline replication: peer="{Voter server-90870093-0964-9a2e-1b19-e08e62de00f7 90870093-0964-9a2e-1b19-e08e62de00f7}" - testing.go:149: server-776ce372-c6a9-bb52-cddd-a76589382d7b: 2025-02-09T06:52:30.320Z [INFO] server-776ce372-c6a9-bb52-cddd-a76589382d7b: aborting pipeline replication: peer="{Voter server-90870093-0964-9a2e-1b19-e08e62de00f7 90870093-0964-9a2e-1b19-e08e62de00f7}" -testLoggerAdapter verbose: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.321Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-520eb645-077f-ef5e-bceb-d7cdf8ac9494 Address:520eb645-077f-ef5e-bceb-d7cdf8ac9494} {Suffrage:Voter ID:server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b Address:d1334e8b-1f25-e18a-3f36-adaa88c6c26b} {Suffrage:Voter ID:server-087cc8b9-c1b2-c264-9d20-5034fb4069a6 Address:087cc8b9-c1b2-c264-9d20-5034fb4069a6}]" - testing.go:149: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.321Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-520eb645-077f-ef5e-bceb-d7cdf8ac9494 Address:520eb645-077f-ef5e-bceb-d7cdf8ac9494} {Suffrage:Voter ID:server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b Address:d1334e8b-1f25-e18a-3f36-adaa88c6c26b} {Suffrage:Voter ID:server-087cc8b9-c1b2-c264-9d20-5034fb4069a6 Address:087cc8b9-c1b2-c264-9d20-5034fb4069a6}]" -testLoggerAdapter verbose: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.321Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: entering follower state: follower="Node at 520eb645-077f-ef5e-bceb-d7cdf8ac9494 [Follower]" leader= - testing.go:149: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.321Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: entering follower state: follower="Node at 520eb645-077f-ef5e-bceb-d7cdf8ac9494 [Follower]" leader= -testLoggerAdapter verbose: server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b: 2025-02-09T06:52:30.321Z [INFO] server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-520eb645-077f-ef5e-bceb-d7cdf8ac9494 Address:520eb645-077f-ef5e-bceb-d7cdf8ac9494} {Suffrage:Voter ID:server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b Address:d1334e8b-1f25-e18a-3f36-adaa88c6c26b} {Suffrage:Voter ID:server-087cc8b9-c1b2-c264-9d20-5034fb4069a6 Address:087cc8b9-c1b2-c264-9d20-5034fb4069a6}]" - testing.go:149: server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b: 2025-02-09T06:52:30.321Z [INFO] server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-520eb645-077f-ef5e-bceb-d7cdf8ac9494 Address:520eb645-077f-ef5e-bceb-d7cdf8ac9494} {Suffrage:Voter ID:server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b Address:d1334e8b-1f25-e18a-3f36-adaa88c6c26b} {Suffrage:Voter ID:server-087cc8b9-c1b2-c264-9d20-5034fb4069a6 Address:087cc8b9-c1b2-c264-9d20-5034fb4069a6}]" -testLoggerAdapter verbose: server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b: 2025-02-09T06:52:30.321Z [INFO] server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b: entering follower state: follower="Node at d1334e8b-1f25-e18a-3f36-adaa88c6c26b [Follower]" leader= - testing.go:149: server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b: 2025-02-09T06:52:30.321Z [INFO] server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b: entering follower state: follower="Node at d1334e8b-1f25-e18a-3f36-adaa88c6c26b [Follower]" leader= -testLoggerAdapter verbose: server-087cc8b9-c1b2-c264-9d20-5034fb4069a6: 2025-02-09T06:52:30.321Z [INFO] server-087cc8b9-c1b2-c264-9d20-5034fb4069a6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-520eb645-077f-ef5e-bceb-d7cdf8ac9494 Address:520eb645-077f-ef5e-bceb-d7cdf8ac9494} {Suffrage:Voter ID:server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b Address:d1334e8b-1f25-e18a-3f36-adaa88c6c26b} {Suffrage:Voter ID:server-087cc8b9-c1b2-c264-9d20-5034fb4069a6 Address:087cc8b9-c1b2-c264-9d20-5034fb4069a6}]" - testing.go:149: server-087cc8b9-c1b2-c264-9d20-5034fb4069a6: 2025-02-09T06:52:30.321Z [INFO] server-087cc8b9-c1b2-c264-9d20-5034fb4069a6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-520eb645-077f-ef5e-bceb-d7cdf8ac9494 Address:520eb645-077f-ef5e-bceb-d7cdf8ac9494} {Suffrage:Voter ID:server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b Address:d1334e8b-1f25-e18a-3f36-adaa88c6c26b} {Suffrage:Voter ID:server-087cc8b9-c1b2-c264-9d20-5034fb4069a6 Address:087cc8b9-c1b2-c264-9d20-5034fb4069a6}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:30.321Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:30.321Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-087cc8b9-c1b2-c264-9d20-5034fb4069a6: 2025-02-09T06:52:30.321Z [INFO] server-087cc8b9-c1b2-c264-9d20-5034fb4069a6: entering follower state: follower="Node at 087cc8b9-c1b2-c264-9d20-5034fb4069a6 [Follower]" leader= - testing.go:149: server-087cc8b9-c1b2-c264-9d20-5034fb4069a6: 2025-02-09T06:52:30.321Z [INFO] server-087cc8b9-c1b2-c264-9d20-5034fb4069a6: entering follower state: follower="Node at 087cc8b9-c1b2-c264-9d20-5034fb4069a6 [Follower]" leader= -testLoggerAdapter verbose: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.375Z [WARN] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.375Z [WARN] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.375Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: entering candidate state: node="Node at 520eb645-077f-ef5e-bceb-d7cdf8ac9494 [Candidate]" term=2 - testing.go:149: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.375Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: entering candidate state: node="Node at 520eb645-077f-ef5e-bceb-d7cdf8ac9494 [Candidate]" term=2 -testLoggerAdapter verbose: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.375Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: election won: tally=2 - testing.go:149: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.375Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: election won: tally=2 -testLoggerAdapter verbose: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.375Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: entering leader state: leader="Node at 520eb645-077f-ef5e-bceb-d7cdf8ac9494 [Leader]" - testing.go:149: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.375Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: entering leader state: leader="Node at 520eb645-077f-ef5e-bceb-d7cdf8ac9494 [Leader]" -testLoggerAdapter verbose: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.375Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: added peer, starting replication: peer=server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b - testing.go:149: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.375Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: added peer, starting replication: peer=server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b -testLoggerAdapter verbose: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.375Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: added peer, starting replication: peer=server-087cc8b9-c1b2-c264-9d20-5034fb4069a6 - testing.go:149: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.375Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: added peer, starting replication: peer=server-087cc8b9-c1b2-c264-9d20-5034fb4069a6 -testLoggerAdapter verbose: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.375Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: pipelining replication: peer="{Voter server-087cc8b9-c1b2-c264-9d20-5034fb4069a6 087cc8b9-c1b2-c264-9d20-5034fb4069a6}" - testing.go:149: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.375Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: pipelining replication: peer="{Voter server-087cc8b9-c1b2-c264-9d20-5034fb4069a6 087cc8b9-c1b2-c264-9d20-5034fb4069a6}" -testLoggerAdapter verbose: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.376Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: pipelining replication: peer="{Voter server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b d1334e8b-1f25-e18a-3f36-adaa88c6c26b}" - testing.go:149: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.376Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: pipelining replication: peer="{Voter server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b d1334e8b-1f25-e18a-3f36-adaa88c6c26b}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:30.481Z [INFO] stable state for Leader reached at 2025-02-09 06:52:30.375917432 +0000 UTC m=+11.354726214 (1 nodes), 54.405307ms from start of poll, 54.768132ms from cluster start. Timeout at 2025-02-09 06:52:30.48174604 +0000 UTC m=+11.460554809, 105.828595ms after stability - testing.go:149: cluster: 2025-02-09T06:52:30.481Z [INFO] stable state for Leader reached at 2025-02-09 06:52:30.375917432 +0000 UTC m=+11.354726214 (1 nodes), 54.405307ms from start of poll, 54.768132ms from cluster start. Timeout at 2025-02-09 06:52:30.48174604 +0000 UTC m=+11.460554809, 105.828595ms after stability -testLoggerAdapter verbose: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.481Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: starting snapshot up to: index=12 - testing.go:149: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.481Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: starting snapshot up to: index=12 - testing.go:153: 2025-02-09T06:52:30.482Z [INFO] creating new snapshot: path=/tmp/raft3653764569/snapshots/2-12-1739083950482.tmp -testLoggerAdapter verbose: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.485Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: snapshot complete up to: index=12 - testing.go:149: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.485Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: snapshot complete up to: index=12 - testing.go:153: 2025-02-09T06:52:30.485Z [INFO] creating new snapshot: path=/tmp/raft3653764569/snapshots/2-23-1739083950485.tmp -testLoggerAdapter verbose: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.489Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: copied to local snapshot: bytes=71 - testing.go:149: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.489Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.489Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: restored user snapshot: index=1 - testing.go:149: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.489Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: restored user snapshot: index=1 -testLoggerAdapter verbose: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.490Z [ERROR] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: failed to get log: index=23 error="log not found" - testing.go:149: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.490Z [ERROR] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.490Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: aborting pipeline replication: peer="{Voter server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b d1334e8b-1f25-e18a-3f36-adaa88c6c26b}" - testing.go:149: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.490Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: aborting pipeline replication: peer="{Voter server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b d1334e8b-1f25-e18a-3f36-adaa88c6c26b}" -testLoggerAdapter verbose: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.490Z [ERROR] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: failed to get log: index=23 error="log not found" - testing.go:149: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.490Z [ERROR] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.490Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: aborting pipeline replication: peer="{Voter server-087cc8b9-c1b2-c264-9d20-5034fb4069a6 087cc8b9-c1b2-c264-9d20-5034fb4069a6}" - testing.go:149: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.490Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: aborting pipeline replication: peer="{Voter server-087cc8b9-c1b2-c264-9d20-5034fb4069a6 087cc8b9-c1b2-c264-9d20-5034fb4069a6}" -testLoggerAdapter verbose: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.500Z [ERROR] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: failed to get log: index=23 error="log not found" - testing.go:149: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.500Z [ERROR] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.500Z [ERROR] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: failed to get log: index=23 error="log not found" - testing.go:149: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.500Z [ERROR] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: failed to get log: index=23 error="log not found" - testing.go:153: 2025-02-09T06:52:30.500Z [INFO] creating new snapshot: path=/tmp/raft3837035945/snapshots/2-23-1739083950500.tmp - testing.go:153: 2025-02-09T06:52:30.500Z [INFO] creating new snapshot: path=/tmp/raft1414319403/snapshots/2-23-1739083950500.tmp -testLoggerAdapter verbose: server-087cc8b9-c1b2-c264-9d20-5034fb4069a6: 2025-02-09T06:52:30.504Z [INFO] server-087cc8b9-c1b2-c264-9d20-5034fb4069a6: copied to local snapshot: bytes=71 - testing.go:149: server-087cc8b9-c1b2-c264-9d20-5034fb4069a6: 2025-02-09T06:52:30.504Z [INFO] server-087cc8b9-c1b2-c264-9d20-5034fb4069a6: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-087cc8b9-c1b2-c264-9d20-5034fb4069a6: 2025-02-09T06:52:30.504Z [INFO] server-087cc8b9-c1b2-c264-9d20-5034fb4069a6: Installed remote snapshot - testing.go:149: server-087cc8b9-c1b2-c264-9d20-5034fb4069a6: 2025-02-09T06:52:30.504Z [INFO] server-087cc8b9-c1b2-c264-9d20-5034fb4069a6: Installed remote snapshot -testLoggerAdapter verbose: server-087cc8b9-c1b2-c264-9d20-5034fb4069a6: 2025-02-09T06:52:30.504Z [WARN] server-087cc8b9-c1b2-c264-9d20-5034fb4069a6: skipping application of old log: index=22 - testing.go:149: server-087cc8b9-c1b2-c264-9d20-5034fb4069a6: 2025-02-09T06:52:30.504Z [WARN] server-087cc8b9-c1b2-c264-9d20-5034fb4069a6: skipping application of old log: index=22 -testLoggerAdapter verbose: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.504Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: pipelining replication: peer="{Voter server-087cc8b9-c1b2-c264-9d20-5034fb4069a6 087cc8b9-c1b2-c264-9d20-5034fb4069a6}" - testing.go:149: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.504Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: pipelining replication: peer="{Voter server-087cc8b9-c1b2-c264-9d20-5034fb4069a6 087cc8b9-c1b2-c264-9d20-5034fb4069a6}" -testLoggerAdapter verbose: server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b: 2025-02-09T06:52:30.504Z [INFO] server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b: copied to local snapshot: bytes=71 - testing.go:149: server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b: 2025-02-09T06:52:30.504Z [INFO] server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b: 2025-02-09T06:52:30.504Z [INFO] server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b: Installed remote snapshot - testing.go:149: server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b: 2025-02-09T06:52:30.504Z [INFO] server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b: Installed remote snapshot -testLoggerAdapter verbose: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.504Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: pipelining replication: peer="{Voter server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b d1334e8b-1f25-e18a-3f36-adaa88c6c26b}" - testing.go:149: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.504Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: pipelining replication: peer="{Voter server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b d1334e8b-1f25-e18a-3f36-adaa88c6c26b}" -testLoggerAdapter verbose: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.520Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: aborting pipeline replication: peer="{Voter server-087cc8b9-c1b2-c264-9d20-5034fb4069a6 087cc8b9-c1b2-c264-9d20-5034fb4069a6}" - testing.go:149: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.520Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: aborting pipeline replication: peer="{Voter server-087cc8b9-c1b2-c264-9d20-5034fb4069a6 087cc8b9-c1b2-c264-9d20-5034fb4069a6}" -testLoggerAdapter verbose: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.520Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: aborting pipeline replication: peer="{Voter server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b d1334e8b-1f25-e18a-3f36-adaa88c6c26b}" - testing.go:149: server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: 2025-02-09T06:52:30.520Z [INFO] server-520eb645-077f-ef5e-bceb-d7cdf8ac9494: aborting pipeline replication: peer="{Voter server-d1334e8b-1f25-e18a-3f36-adaa88c6c26b d1334e8b-1f25-e18a-3f36-adaa88c6c26b}" -testLoggerAdapter verbose: server-9d77a21b-f17d-6ef4-3c55-355704340786: 2025-02-09T06:52:30.521Z [INFO] server-9d77a21b-f17d-6ef4-3c55-355704340786: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9d77a21b-f17d-6ef4-3c55-355704340786 Address:9d77a21b-f17d-6ef4-3c55-355704340786} {Suffrage:Voter ID:server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0 Address:42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0} {Suffrage:Voter ID:server-97d5b64c-70f6-ee6b-cef7-b89b31861f93 Address:97d5b64c-70f6-ee6b-cef7-b89b31861f93}]" - testing.go:149: server-9d77a21b-f17d-6ef4-3c55-355704340786: 2025-02-09T06:52:30.521Z [INFO] server-9d77a21b-f17d-6ef4-3c55-355704340786: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9d77a21b-f17d-6ef4-3c55-355704340786 Address:9d77a21b-f17d-6ef4-3c55-355704340786} {Suffrage:Voter ID:server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0 Address:42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0} {Suffrage:Voter ID:server-97d5b64c-70f6-ee6b-cef7-b89b31861f93 Address:97d5b64c-70f6-ee6b-cef7-b89b31861f93}]" -testLoggerAdapter verbose: server-9d77a21b-f17d-6ef4-3c55-355704340786: 2025-02-09T06:52:30.522Z [INFO] server-9d77a21b-f17d-6ef4-3c55-355704340786: entering follower state: follower="Node at 9d77a21b-f17d-6ef4-3c55-355704340786 [Follower]" leader= - testing.go:149: server-9d77a21b-f17d-6ef4-3c55-355704340786: 2025-02-09T06:52:30.522Z [INFO] server-9d77a21b-f17d-6ef4-3c55-355704340786: entering follower state: follower="Node at 9d77a21b-f17d-6ef4-3c55-355704340786 [Follower]" leader= -testLoggerAdapter verbose: server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0: 2025-02-09T06:52:30.522Z [INFO] server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9d77a21b-f17d-6ef4-3c55-355704340786 Address:9d77a21b-f17d-6ef4-3c55-355704340786} {Suffrage:Voter ID:server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0 Address:42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0} {Suffrage:Voter ID:server-97d5b64c-70f6-ee6b-cef7-b89b31861f93 Address:97d5b64c-70f6-ee6b-cef7-b89b31861f93}]" - testing.go:149: server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0: 2025-02-09T06:52:30.522Z [INFO] server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9d77a21b-f17d-6ef4-3c55-355704340786 Address:9d77a21b-f17d-6ef4-3c55-355704340786} {Suffrage:Voter ID:server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0 Address:42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0} {Suffrage:Voter ID:server-97d5b64c-70f6-ee6b-cef7-b89b31861f93 Address:97d5b64c-70f6-ee6b-cef7-b89b31861f93}]" -testLoggerAdapter verbose: server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0: 2025-02-09T06:52:30.522Z [INFO] server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0: entering follower state: follower="Node at 42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0 [Follower]" leader= - testing.go:149: server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0: 2025-02-09T06:52:30.522Z [INFO] server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0: entering follower state: follower="Node at 42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0 [Follower]" leader= -testLoggerAdapter verbose: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.522Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9d77a21b-f17d-6ef4-3c55-355704340786 Address:9d77a21b-f17d-6ef4-3c55-355704340786} {Suffrage:Voter ID:server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0 Address:42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0} {Suffrage:Voter ID:server-97d5b64c-70f6-ee6b-cef7-b89b31861f93 Address:97d5b64c-70f6-ee6b-cef7-b89b31861f93}]" - testing.go:149: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.522Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9d77a21b-f17d-6ef4-3c55-355704340786 Address:9d77a21b-f17d-6ef4-3c55-355704340786} {Suffrage:Voter ID:server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0 Address:42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0} {Suffrage:Voter ID:server-97d5b64c-70f6-ee6b-cef7-b89b31861f93 Address:97d5b64c-70f6-ee6b-cef7-b89b31861f93}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:30.522Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:30.522Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.522Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: entering follower state: follower="Node at 97d5b64c-70f6-ee6b-cef7-b89b31861f93 [Follower]" leader= - testing.go:149: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.522Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: entering follower state: follower="Node at 97d5b64c-70f6-ee6b-cef7-b89b31861f93 [Follower]" leader= -testLoggerAdapter verbose: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.577Z [WARN] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.577Z [WARN] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.577Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: entering candidate state: node="Node at 97d5b64c-70f6-ee6b-cef7-b89b31861f93 [Candidate]" term=2 - testing.go:149: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.577Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: entering candidate state: node="Node at 97d5b64c-70f6-ee6b-cef7-b89b31861f93 [Candidate]" term=2 -testLoggerAdapter verbose: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.578Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: election won: tally=2 - testing.go:149: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.578Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: election won: tally=2 -testLoggerAdapter verbose: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.578Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: entering leader state: leader="Node at 97d5b64c-70f6-ee6b-cef7-b89b31861f93 [Leader]" - testing.go:149: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.578Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: entering leader state: leader="Node at 97d5b64c-70f6-ee6b-cef7-b89b31861f93 [Leader]" -testLoggerAdapter verbose: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.578Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: added peer, starting replication: peer=server-9d77a21b-f17d-6ef4-3c55-355704340786 - testing.go:149: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.578Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: added peer, starting replication: peer=server-9d77a21b-f17d-6ef4-3c55-355704340786 -testLoggerAdapter verbose: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.578Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: added peer, starting replication: peer=server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0 - testing.go:149: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.578Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: added peer, starting replication: peer=server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0 -testLoggerAdapter verbose: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.578Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: pipelining replication: peer="{Voter server-9d77a21b-f17d-6ef4-3c55-355704340786 9d77a21b-f17d-6ef4-3c55-355704340786}" - testing.go:149: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.578Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: pipelining replication: peer="{Voter server-9d77a21b-f17d-6ef4-3c55-355704340786 9d77a21b-f17d-6ef4-3c55-355704340786}" -testLoggerAdapter verbose: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.578Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: pipelining replication: peer="{Voter server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0 42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0}" - testing.go:149: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.578Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: pipelining replication: peer="{Voter server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0 42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:30.683Z [INFO] stable state for Leader reached at 2025-02-09 06:52:30.578113266 +0000 UTC m=+11.556922031 (1 nodes), 55.694498ms from start of poll, 56.203714ms from cluster start. Timeout at 2025-02-09 06:52:30.683384756 +0000 UTC m=+11.662193524, 105.271493ms after stability - testing.go:149: cluster: 2025-02-09T06:52:30.683Z [INFO] stable state for Leader reached at 2025-02-09 06:52:30.578113266 +0000 UTC m=+11.556922031 (1 nodes), 55.694498ms from start of poll, 56.203714ms from cluster start. Timeout at 2025-02-09 06:52:30.683384756 +0000 UTC m=+11.662193524, 105.271493ms after stability -testLoggerAdapter verbose: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.683Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: starting snapshot up to: index=12 - testing.go:149: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.683Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: starting snapshot up to: index=12 - testing.go:153: 2025-02-09T06:52:30.683Z [INFO] creating new snapshot: path=/tmp/raft1583009783/snapshots/2-12-1739083950683.tmp -testLoggerAdapter verbose: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.688Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: snapshot complete up to: index=12 - testing.go:149: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.688Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: snapshot complete up to: index=12 - testing.go:153: 2025-02-09T06:52:30.688Z [INFO] creating new snapshot: path=/tmp/raft1583009783/snapshots/2-23-1739083950688.tmp -testLoggerAdapter verbose: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.691Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: copied to local snapshot: bytes=71 - testing.go:149: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.691Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.691Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: restored user snapshot: index=1 - testing.go:149: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.691Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: restored user snapshot: index=1 -testLoggerAdapter verbose: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.691Z [ERROR] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: failed to get log: index=23 error="log not found" - testing.go:149: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.691Z [ERROR] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.691Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: aborting pipeline replication: peer="{Voter server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0 42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0}" - testing.go:149: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.691Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: aborting pipeline replication: peer="{Voter server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0 42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0}" -testLoggerAdapter verbose: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.691Z [ERROR] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: failed to get log: index=23 error="log not found" - testing.go:149: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.691Z [ERROR] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.691Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: aborting pipeline replication: peer="{Voter server-9d77a21b-f17d-6ef4-3c55-355704340786 9d77a21b-f17d-6ef4-3c55-355704340786}" - testing.go:149: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.691Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: aborting pipeline replication: peer="{Voter server-9d77a21b-f17d-6ef4-3c55-355704340786 9d77a21b-f17d-6ef4-3c55-355704340786}" -testLoggerAdapter verbose: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.698Z [ERROR] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: failed to get log: index=23 error="log not found" - testing.go:149: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.698Z [ERROR] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: failed to get log: index=23 error="log not found" - testing.go:153: 2025-02-09T06:52:30.698Z [INFO] creating new snapshot: path=/tmp/raft2258289372/snapshots/2-23-1739083950698.tmp -testLoggerAdapter verbose: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.701Z [ERROR] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: failed to get log: index=23 error="log not found" - testing.go:149: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.701Z [ERROR] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: failed to get log: index=23 error="log not found" - testing.go:153: 2025-02-09T06:52:30.702Z [INFO] creating new snapshot: path=/tmp/raft2603878710/snapshots/2-23-1739083950702.tmp -testLoggerAdapter verbose: server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0: 2025-02-09T06:52:30.702Z [INFO] server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0: copied to local snapshot: bytes=71 - testing.go:149: server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0: 2025-02-09T06:52:30.702Z [INFO] server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0: 2025-02-09T06:52:30.702Z [INFO] server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0: Installed remote snapshot - testing.go:149: server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0: 2025-02-09T06:52:30.702Z [INFO] server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0: Installed remote snapshot -testLoggerAdapter verbose: server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0: 2025-02-09T06:52:30.702Z [WARN] server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0: skipping application of old log: index=22 - testing.go:149: server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0: 2025-02-09T06:52:30.702Z [WARN] server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0: skipping application of old log: index=22 -testLoggerAdapter verbose: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.702Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: pipelining replication: peer="{Voter server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0 42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0}" - testing.go:149: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.702Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: pipelining replication: peer="{Voter server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0 42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0}" -testLoggerAdapter verbose: server-9d77a21b-f17d-6ef4-3c55-355704340786: 2025-02-09T06:52:30.705Z [INFO] server-9d77a21b-f17d-6ef4-3c55-355704340786: copied to local snapshot: bytes=71 - testing.go:149: server-9d77a21b-f17d-6ef4-3c55-355704340786: 2025-02-09T06:52:30.705Z [INFO] server-9d77a21b-f17d-6ef4-3c55-355704340786: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-9d77a21b-f17d-6ef4-3c55-355704340786: 2025-02-09T06:52:30.705Z [INFO] server-9d77a21b-f17d-6ef4-3c55-355704340786: Installed remote snapshot - testing.go:149: server-9d77a21b-f17d-6ef4-3c55-355704340786: 2025-02-09T06:52:30.705Z [INFO] server-9d77a21b-f17d-6ef4-3c55-355704340786: Installed remote snapshot -testLoggerAdapter verbose: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.705Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: pipelining replication: peer="{Voter server-9d77a21b-f17d-6ef4-3c55-355704340786 9d77a21b-f17d-6ef4-3c55-355704340786}" - testing.go:149: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.705Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: pipelining replication: peer="{Voter server-9d77a21b-f17d-6ef4-3c55-355704340786 9d77a21b-f17d-6ef4-3c55-355704340786}" -testLoggerAdapter verbose: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.719Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: aborting pipeline replication: peer="{Voter server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0 42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0}" - testing.go:149: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.719Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: aborting pipeline replication: peer="{Voter server-42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0 42fd7b7b-283c-3ea8-cadf-d9cba3b80ec0}" -testLoggerAdapter verbose: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.719Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: aborting pipeline replication: peer="{Voter server-9d77a21b-f17d-6ef4-3c55-355704340786 9d77a21b-f17d-6ef4-3c55-355704340786}" - testing.go:149: server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: 2025-02-09T06:52:30.719Z [INFO] server-97d5b64c-70f6-ee6b-cef7-b89b31861f93: aborting pipeline replication: peer="{Voter server-9d77a21b-f17d-6ef4-3c55-355704340786 9d77a21b-f17d-6ef4-3c55-355704340786}" -testLoggerAdapter verbose: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.720Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f893d015-09a3-921d-e2f9-eeb74c1e09a1 Address:f893d015-09a3-921d-e2f9-eeb74c1e09a1} {Suffrage:Voter ID:server-7ba6a560-8794-659b-71b8-e26c4a258a13 Address:7ba6a560-8794-659b-71b8-e26c4a258a13} {Suffrage:Voter ID:server-fc863720-0f1f-7639-7491-7b3c2d787261 Address:fc863720-0f1f-7639-7491-7b3c2d787261}]" - testing.go:149: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.720Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f893d015-09a3-921d-e2f9-eeb74c1e09a1 Address:f893d015-09a3-921d-e2f9-eeb74c1e09a1} {Suffrage:Voter ID:server-7ba6a560-8794-659b-71b8-e26c4a258a13 Address:7ba6a560-8794-659b-71b8-e26c4a258a13} {Suffrage:Voter ID:server-fc863720-0f1f-7639-7491-7b3c2d787261 Address:fc863720-0f1f-7639-7491-7b3c2d787261}]" -testLoggerAdapter verbose: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.720Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: entering follower state: follower="Node at f893d015-09a3-921d-e2f9-eeb74c1e09a1 [Follower]" leader= - testing.go:149: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.720Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: entering follower state: follower="Node at f893d015-09a3-921d-e2f9-eeb74c1e09a1 [Follower]" leader= -testLoggerAdapter verbose: server-7ba6a560-8794-659b-71b8-e26c4a258a13: 2025-02-09T06:52:30.720Z [INFO] server-7ba6a560-8794-659b-71b8-e26c4a258a13: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f893d015-09a3-921d-e2f9-eeb74c1e09a1 Address:f893d015-09a3-921d-e2f9-eeb74c1e09a1} {Suffrage:Voter ID:server-7ba6a560-8794-659b-71b8-e26c4a258a13 Address:7ba6a560-8794-659b-71b8-e26c4a258a13} {Suffrage:Voter ID:server-fc863720-0f1f-7639-7491-7b3c2d787261 Address:fc863720-0f1f-7639-7491-7b3c2d787261}]" - testing.go:149: server-7ba6a560-8794-659b-71b8-e26c4a258a13: 2025-02-09T06:52:30.720Z [INFO] server-7ba6a560-8794-659b-71b8-e26c4a258a13: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f893d015-09a3-921d-e2f9-eeb74c1e09a1 Address:f893d015-09a3-921d-e2f9-eeb74c1e09a1} {Suffrage:Voter ID:server-7ba6a560-8794-659b-71b8-e26c4a258a13 Address:7ba6a560-8794-659b-71b8-e26c4a258a13} {Suffrage:Voter ID:server-fc863720-0f1f-7639-7491-7b3c2d787261 Address:fc863720-0f1f-7639-7491-7b3c2d787261}]" -testLoggerAdapter verbose: server-7ba6a560-8794-659b-71b8-e26c4a258a13: 2025-02-09T06:52:30.720Z [INFO] server-7ba6a560-8794-659b-71b8-e26c4a258a13: entering follower state: follower="Node at 7ba6a560-8794-659b-71b8-e26c4a258a13 [Follower]" leader= - testing.go:149: server-7ba6a560-8794-659b-71b8-e26c4a258a13: 2025-02-09T06:52:30.720Z [INFO] server-7ba6a560-8794-659b-71b8-e26c4a258a13: entering follower state: follower="Node at 7ba6a560-8794-659b-71b8-e26c4a258a13 [Follower]" leader= -testLoggerAdapter verbose: server-fc863720-0f1f-7639-7491-7b3c2d787261: 2025-02-09T06:52:30.720Z [INFO] server-fc863720-0f1f-7639-7491-7b3c2d787261: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f893d015-09a3-921d-e2f9-eeb74c1e09a1 Address:f893d015-09a3-921d-e2f9-eeb74c1e09a1} {Suffrage:Voter ID:server-7ba6a560-8794-659b-71b8-e26c4a258a13 Address:7ba6a560-8794-659b-71b8-e26c4a258a13} {Suffrage:Voter ID:server-fc863720-0f1f-7639-7491-7b3c2d787261 Address:fc863720-0f1f-7639-7491-7b3c2d787261}]" - testing.go:149: server-fc863720-0f1f-7639-7491-7b3c2d787261: 2025-02-09T06:52:30.720Z [INFO] server-fc863720-0f1f-7639-7491-7b3c2d787261: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f893d015-09a3-921d-e2f9-eeb74c1e09a1 Address:f893d015-09a3-921d-e2f9-eeb74c1e09a1} {Suffrage:Voter ID:server-7ba6a560-8794-659b-71b8-e26c4a258a13 Address:7ba6a560-8794-659b-71b8-e26c4a258a13} {Suffrage:Voter ID:server-fc863720-0f1f-7639-7491-7b3c2d787261 Address:fc863720-0f1f-7639-7491-7b3c2d787261}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:30.720Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:30.720Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-fc863720-0f1f-7639-7491-7b3c2d787261: 2025-02-09T06:52:30.720Z [INFO] server-fc863720-0f1f-7639-7491-7b3c2d787261: entering follower state: follower="Node at fc863720-0f1f-7639-7491-7b3c2d787261 [Follower]" leader= - testing.go:149: server-fc863720-0f1f-7639-7491-7b3c2d787261: 2025-02-09T06:52:30.720Z [INFO] server-fc863720-0f1f-7639-7491-7b3c2d787261: entering follower state: follower="Node at fc863720-0f1f-7639-7491-7b3c2d787261 [Follower]" leader= -testLoggerAdapter verbose: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.788Z [WARN] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.788Z [WARN] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.788Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: entering candidate state: node="Node at f893d015-09a3-921d-e2f9-eeb74c1e09a1 [Candidate]" term=2 - testing.go:149: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.788Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: entering candidate state: node="Node at f893d015-09a3-921d-e2f9-eeb74c1e09a1 [Candidate]" term=2 -testLoggerAdapter verbose: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.788Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: election won: tally=2 - testing.go:149: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.788Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: election won: tally=2 -testLoggerAdapter verbose: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.788Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: entering leader state: leader="Node at f893d015-09a3-921d-e2f9-eeb74c1e09a1 [Leader]" - testing.go:149: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.788Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: entering leader state: leader="Node at f893d015-09a3-921d-e2f9-eeb74c1e09a1 [Leader]" -testLoggerAdapter verbose: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.788Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: added peer, starting replication: peer=server-7ba6a560-8794-659b-71b8-e26c4a258a13 - testing.go:149: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.788Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: added peer, starting replication: peer=server-7ba6a560-8794-659b-71b8-e26c4a258a13 -testLoggerAdapter verbose: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.788Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: added peer, starting replication: peer=server-fc863720-0f1f-7639-7491-7b3c2d787261 - testing.go:149: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.788Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: added peer, starting replication: peer=server-fc863720-0f1f-7639-7491-7b3c2d787261 -testLoggerAdapter verbose: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.788Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: pipelining replication: peer="{Voter server-fc863720-0f1f-7639-7491-7b3c2d787261 fc863720-0f1f-7639-7491-7b3c2d787261}" - testing.go:149: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.788Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: pipelining replication: peer="{Voter server-fc863720-0f1f-7639-7491-7b3c2d787261 fc863720-0f1f-7639-7491-7b3c2d787261}" -testLoggerAdapter verbose: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.788Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: pipelining replication: peer="{Voter server-7ba6a560-8794-659b-71b8-e26c4a258a13 7ba6a560-8794-659b-71b8-e26c4a258a13}" - testing.go:149: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.788Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: pipelining replication: peer="{Voter server-7ba6a560-8794-659b-71b8-e26c4a258a13 7ba6a560-8794-659b-71b8-e26c4a258a13}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:30.894Z [INFO] stable state for Leader reached at 2025-02-09 06:52:30.788344401 +0000 UTC m=+11.767153179 (1 nodes), 67.737571ms from start of poll, 68.081779ms from cluster start. Timeout at 2025-02-09 06:52:30.894032045 +0000 UTC m=+11.872840804, 105.687625ms after stability - testing.go:149: cluster: 2025-02-09T06:52:30.894Z [INFO] stable state for Leader reached at 2025-02-09 06:52:30.788344401 +0000 UTC m=+11.767153179 (1 nodes), 67.737571ms from start of poll, 68.081779ms from cluster start. Timeout at 2025-02-09 06:52:30.894032045 +0000 UTC m=+11.872840804, 105.687625ms after stability -testLoggerAdapter verbose: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.894Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: starting snapshot up to: index=12 - testing.go:149: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.894Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: starting snapshot up to: index=12 - testing.go:153: 2025-02-09T06:52:30.894Z [INFO] creating new snapshot: path=/tmp/raft2966269005/snapshots/2-12-1739083950894.tmp -testLoggerAdapter verbose: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.898Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: snapshot complete up to: index=12 - testing.go:149: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.898Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: snapshot complete up to: index=12 - testing.go:153: 2025-02-09T06:52:30.898Z [INFO] creating new snapshot: path=/tmp/raft2966269005/snapshots/2-113-1739083950898.tmp -testLoggerAdapter verbose: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.902Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: copied to local snapshot: bytes=71 - testing.go:149: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.902Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.902Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: restored user snapshot: index=1 - testing.go:149: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.902Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: restored user snapshot: index=1 -testLoggerAdapter verbose: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.902Z [ERROR] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: failed to get log: index=23 error="log not found" - testing.go:149: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.902Z [ERROR] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.902Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: aborting pipeline replication: peer="{Voter server-fc863720-0f1f-7639-7491-7b3c2d787261 fc863720-0f1f-7639-7491-7b3c2d787261}" - testing.go:149: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.902Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: aborting pipeline replication: peer="{Voter server-fc863720-0f1f-7639-7491-7b3c2d787261 fc863720-0f1f-7639-7491-7b3c2d787261}" -testLoggerAdapter verbose: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.902Z [ERROR] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: failed to get log: index=23 error="log not found" - testing.go:149: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.902Z [ERROR] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.903Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: aborting pipeline replication: peer="{Voter server-7ba6a560-8794-659b-71b8-e26c4a258a13 7ba6a560-8794-659b-71b8-e26c4a258a13}" - testing.go:149: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.903Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: aborting pipeline replication: peer="{Voter server-7ba6a560-8794-659b-71b8-e26c4a258a13 7ba6a560-8794-659b-71b8-e26c4a258a13}" -testLoggerAdapter verbose: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.909Z [ERROR] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: failed to get log: index=23 error="log not found" - testing.go:149: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.909Z [ERROR] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: failed to get log: index=23 error="log not found" - testing.go:153: 2025-02-09T06:52:30.910Z [INFO] creating new snapshot: path=/tmp/raft517204571/snapshots/2-113-1739083950910.tmp -testLoggerAdapter verbose: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.912Z [ERROR] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: failed to get log: index=23 error="log not found" - testing.go:149: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.912Z [ERROR] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: failed to get log: index=23 error="log not found" - testing.go:153: 2025-02-09T06:52:30.913Z [INFO] creating new snapshot: path=/tmp/raft447112047/snapshots/2-113-1739083950913.tmp -testLoggerAdapter verbose: server-fc863720-0f1f-7639-7491-7b3c2d787261: 2025-02-09T06:52:30.914Z [INFO] server-fc863720-0f1f-7639-7491-7b3c2d787261: copied to local snapshot: bytes=71 - testing.go:149: server-fc863720-0f1f-7639-7491-7b3c2d787261: 2025-02-09T06:52:30.914Z [INFO] server-fc863720-0f1f-7639-7491-7b3c2d787261: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-fc863720-0f1f-7639-7491-7b3c2d787261: 2025-02-09T06:52:30.914Z [INFO] server-fc863720-0f1f-7639-7491-7b3c2d787261: Installed remote snapshot - testing.go:149: server-fc863720-0f1f-7639-7491-7b3c2d787261: 2025-02-09T06:52:30.914Z [INFO] server-fc863720-0f1f-7639-7491-7b3c2d787261: Installed remote snapshot -testLoggerAdapter verbose: server-fc863720-0f1f-7639-7491-7b3c2d787261: 2025-02-09T06:52:30.914Z [WARN] server-fc863720-0f1f-7639-7491-7b3c2d787261: skipping application of old log: index=22 - testing.go:149: server-fc863720-0f1f-7639-7491-7b3c2d787261: 2025-02-09T06:52:30.914Z [WARN] server-fc863720-0f1f-7639-7491-7b3c2d787261: skipping application of old log: index=22 -testLoggerAdapter verbose: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.914Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: pipelining replication: peer="{Voter server-fc863720-0f1f-7639-7491-7b3c2d787261 fc863720-0f1f-7639-7491-7b3c2d787261}" - testing.go:149: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.914Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: pipelining replication: peer="{Voter server-fc863720-0f1f-7639-7491-7b3c2d787261 fc863720-0f1f-7639-7491-7b3c2d787261}" -testLoggerAdapter verbose: server-7ba6a560-8794-659b-71b8-e26c4a258a13: 2025-02-09T06:52:30.916Z [INFO] server-7ba6a560-8794-659b-71b8-e26c4a258a13: copied to local snapshot: bytes=71 - testing.go:149: server-7ba6a560-8794-659b-71b8-e26c4a258a13: 2025-02-09T06:52:30.916Z [INFO] server-7ba6a560-8794-659b-71b8-e26c4a258a13: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-7ba6a560-8794-659b-71b8-e26c4a258a13: 2025-02-09T06:52:30.916Z [INFO] server-7ba6a560-8794-659b-71b8-e26c4a258a13: Installed remote snapshot - testing.go:149: server-7ba6a560-8794-659b-71b8-e26c4a258a13: 2025-02-09T06:52:30.916Z [INFO] server-7ba6a560-8794-659b-71b8-e26c4a258a13: Installed remote snapshot -testLoggerAdapter verbose: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.916Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: pipelining replication: peer="{Voter server-7ba6a560-8794-659b-71b8-e26c4a258a13 7ba6a560-8794-659b-71b8-e26c4a258a13}" - testing.go:149: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.916Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: pipelining replication: peer="{Voter server-7ba6a560-8794-659b-71b8-e26c4a258a13 7ba6a560-8794-659b-71b8-e26c4a258a13}" -testLoggerAdapter verbose: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.936Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: aborting pipeline replication: peer="{Voter server-fc863720-0f1f-7639-7491-7b3c2d787261 fc863720-0f1f-7639-7491-7b3c2d787261}" - testing.go:149: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.936Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: aborting pipeline replication: peer="{Voter server-fc863720-0f1f-7639-7491-7b3c2d787261 fc863720-0f1f-7639-7491-7b3c2d787261}" -testLoggerAdapter verbose: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.936Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: aborting pipeline replication: peer="{Voter server-7ba6a560-8794-659b-71b8-e26c4a258a13 7ba6a560-8794-659b-71b8-e26c4a258a13}" - testing.go:149: server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: 2025-02-09T06:52:30.936Z [INFO] server-f893d015-09a3-921d-e2f9-eeb74c1e09a1: aborting pipeline replication: peer="{Voter server-7ba6a560-8794-659b-71b8-e26c4a258a13 7ba6a560-8794-659b-71b8-e26c4a258a13}" -testLoggerAdapter verbose: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:30.937Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b Address:3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b} {Suffrage:Voter ID:server-993631aa-85a8-0845-55f3-9c16d10b6eca Address:993631aa-85a8-0845-55f3-9c16d10b6eca} {Suffrage:Voter ID:server-abd23e07-12ed-2b70-1e64-334ea4b1055d Address:abd23e07-12ed-2b70-1e64-334ea4b1055d}]" - testing.go:149: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:30.937Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b Address:3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b} {Suffrage:Voter ID:server-993631aa-85a8-0845-55f3-9c16d10b6eca Address:993631aa-85a8-0845-55f3-9c16d10b6eca} {Suffrage:Voter ID:server-abd23e07-12ed-2b70-1e64-334ea4b1055d Address:abd23e07-12ed-2b70-1e64-334ea4b1055d}]" -testLoggerAdapter verbose: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:30.937Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: entering follower state: follower="Node at 3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b [Follower]" leader= - testing.go:149: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:30.937Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: entering follower state: follower="Node at 3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b [Follower]" leader= -testLoggerAdapter verbose: server-993631aa-85a8-0845-55f3-9c16d10b6eca: 2025-02-09T06:52:30.937Z [INFO] server-993631aa-85a8-0845-55f3-9c16d10b6eca: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b Address:3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b} {Suffrage:Voter ID:server-993631aa-85a8-0845-55f3-9c16d10b6eca Address:993631aa-85a8-0845-55f3-9c16d10b6eca} {Suffrage:Voter ID:server-abd23e07-12ed-2b70-1e64-334ea4b1055d Address:abd23e07-12ed-2b70-1e64-334ea4b1055d}]" - testing.go:149: server-993631aa-85a8-0845-55f3-9c16d10b6eca: 2025-02-09T06:52:30.937Z [INFO] server-993631aa-85a8-0845-55f3-9c16d10b6eca: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b Address:3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b} {Suffrage:Voter ID:server-993631aa-85a8-0845-55f3-9c16d10b6eca Address:993631aa-85a8-0845-55f3-9c16d10b6eca} {Suffrage:Voter ID:server-abd23e07-12ed-2b70-1e64-334ea4b1055d Address:abd23e07-12ed-2b70-1e64-334ea4b1055d}]" -testLoggerAdapter verbose: server-993631aa-85a8-0845-55f3-9c16d10b6eca: 2025-02-09T06:52:30.938Z [INFO] server-993631aa-85a8-0845-55f3-9c16d10b6eca: entering follower state: follower="Node at 993631aa-85a8-0845-55f3-9c16d10b6eca [Follower]" leader= - testing.go:149: server-993631aa-85a8-0845-55f3-9c16d10b6eca: 2025-02-09T06:52:30.938Z [INFO] server-993631aa-85a8-0845-55f3-9c16d10b6eca: entering follower state: follower="Node at 993631aa-85a8-0845-55f3-9c16d10b6eca [Follower]" leader= -testLoggerAdapter verbose: server-abd23e07-12ed-2b70-1e64-334ea4b1055d: 2025-02-09T06:52:30.938Z [INFO] server-abd23e07-12ed-2b70-1e64-334ea4b1055d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b Address:3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b} {Suffrage:Voter ID:server-993631aa-85a8-0845-55f3-9c16d10b6eca Address:993631aa-85a8-0845-55f3-9c16d10b6eca} {Suffrage:Voter ID:server-abd23e07-12ed-2b70-1e64-334ea4b1055d Address:abd23e07-12ed-2b70-1e64-334ea4b1055d}]" - testing.go:149: server-abd23e07-12ed-2b70-1e64-334ea4b1055d: 2025-02-09T06:52:30.938Z [INFO] server-abd23e07-12ed-2b70-1e64-334ea4b1055d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b Address:3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b} {Suffrage:Voter ID:server-993631aa-85a8-0845-55f3-9c16d10b6eca Address:993631aa-85a8-0845-55f3-9c16d10b6eca} {Suffrage:Voter ID:server-abd23e07-12ed-2b70-1e64-334ea4b1055d Address:abd23e07-12ed-2b70-1e64-334ea4b1055d}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:30.938Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:30.938Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-abd23e07-12ed-2b70-1e64-334ea4b1055d: 2025-02-09T06:52:30.938Z [INFO] server-abd23e07-12ed-2b70-1e64-334ea4b1055d: entering follower state: follower="Node at abd23e07-12ed-2b70-1e64-334ea4b1055d [Follower]" leader= - testing.go:149: server-abd23e07-12ed-2b70-1e64-334ea4b1055d: 2025-02-09T06:52:30.938Z [INFO] server-abd23e07-12ed-2b70-1e64-334ea4b1055d: entering follower state: follower="Node at abd23e07-12ed-2b70-1e64-334ea4b1055d [Follower]" leader= -testLoggerAdapter verbose: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:30.999Z [WARN] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:30.999Z [WARN] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:30.999Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: entering candidate state: node="Node at 3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b [Candidate]" term=2 - testing.go:149: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:30.999Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: entering candidate state: node="Node at 3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b [Candidate]" term=2 -testLoggerAdapter verbose: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.000Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: election won: tally=2 - testing.go:149: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.000Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: election won: tally=2 -testLoggerAdapter verbose: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.000Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: entering leader state: leader="Node at 3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b [Leader]" - testing.go:149: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.000Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: entering leader state: leader="Node at 3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b [Leader]" -testLoggerAdapter verbose: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.000Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: added peer, starting replication: peer=server-993631aa-85a8-0845-55f3-9c16d10b6eca - testing.go:149: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.000Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: added peer, starting replication: peer=server-993631aa-85a8-0845-55f3-9c16d10b6eca -testLoggerAdapter verbose: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.000Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: added peer, starting replication: peer=server-abd23e07-12ed-2b70-1e64-334ea4b1055d - testing.go:149: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.000Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: added peer, starting replication: peer=server-abd23e07-12ed-2b70-1e64-334ea4b1055d -testLoggerAdapter verbose: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.000Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: pipelining replication: peer="{Voter server-abd23e07-12ed-2b70-1e64-334ea4b1055d abd23e07-12ed-2b70-1e64-334ea4b1055d}" - testing.go:149: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.000Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: pipelining replication: peer="{Voter server-abd23e07-12ed-2b70-1e64-334ea4b1055d abd23e07-12ed-2b70-1e64-334ea4b1055d}" -testLoggerAdapter verbose: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.000Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: pipelining replication: peer="{Voter server-993631aa-85a8-0845-55f3-9c16d10b6eca 993631aa-85a8-0845-55f3-9c16d10b6eca}" - testing.go:149: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.000Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: pipelining replication: peer="{Voter server-993631aa-85a8-0845-55f3-9c16d10b6eca 993631aa-85a8-0845-55f3-9c16d10b6eca}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:31.105Z [INFO] stable state for Leader reached at 2025-02-09 06:52:31.000116399 +0000 UTC m=+11.978925213 (1 nodes), 61.993958ms from start of poll, 62.355225ms from cluster start. Timeout at 2025-02-09 06:52:31.105498884 +0000 UTC m=+12.084307643, 105.38243ms after stability - testing.go:149: cluster: 2025-02-09T06:52:31.105Z [INFO] stable state for Leader reached at 2025-02-09 06:52:31.000116399 +0000 UTC m=+11.978925213 (1 nodes), 61.993958ms from start of poll, 62.355225ms from cluster start. Timeout at 2025-02-09 06:52:31.105498884 +0000 UTC m=+12.084307643, 105.38243ms after stability -testLoggerAdapter verbose: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.105Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: starting snapshot up to: index=12 - testing.go:149: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.105Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: starting snapshot up to: index=12 - testing.go:153: 2025-02-09T06:52:31.105Z [INFO] creating new snapshot: path=/tmp/raft3164157557/snapshots/2-12-1739083951105.tmp -testLoggerAdapter verbose: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.111Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: snapshot complete up to: index=12 - testing.go:149: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.111Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: snapshot complete up to: index=12 - testing.go:153: 2025-02-09T06:52:31.111Z [INFO] creating new snapshot: path=/tmp/raft3164157557/snapshots/2-1013-1739083951111.tmp -testLoggerAdapter verbose: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.114Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: copied to local snapshot: bytes=71 - testing.go:149: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.114Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.114Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: restored user snapshot: index=1 - testing.go:149: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.114Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: restored user snapshot: index=1 -testLoggerAdapter verbose: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.114Z [ERROR] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: failed to get log: index=23 error="log not found" - testing.go:149: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.114Z [ERROR] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.114Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: aborting pipeline replication: peer="{Voter server-abd23e07-12ed-2b70-1e64-334ea4b1055d abd23e07-12ed-2b70-1e64-334ea4b1055d}" - testing.go:149: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.114Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: aborting pipeline replication: peer="{Voter server-abd23e07-12ed-2b70-1e64-334ea4b1055d abd23e07-12ed-2b70-1e64-334ea4b1055d}" -testLoggerAdapter verbose: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.114Z [ERROR] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: failed to get log: index=23 error="log not found" - testing.go:149: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.114Z [ERROR] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.114Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: aborting pipeline replication: peer="{Voter server-993631aa-85a8-0845-55f3-9c16d10b6eca 993631aa-85a8-0845-55f3-9c16d10b6eca}" - testing.go:149: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.114Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: aborting pipeline replication: peer="{Voter server-993631aa-85a8-0845-55f3-9c16d10b6eca 993631aa-85a8-0845-55f3-9c16d10b6eca}" -testLoggerAdapter verbose: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.121Z [ERROR] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: failed to get log: index=23 error="log not found" - testing.go:149: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.121Z [ERROR] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: failed to get log: index=23 error="log not found" - testing.go:153: 2025-02-09T06:52:31.122Z [INFO] creating new snapshot: path=/tmp/raft3180562438/snapshots/2-1013-1739083951122.tmp -testLoggerAdapter verbose: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.122Z [ERROR] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: failed to get log: index=23 error="log not found" - testing.go:149: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.122Z [ERROR] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: failed to get log: index=23 error="log not found" - testing.go:153: 2025-02-09T06:52:31.123Z [INFO] creating new snapshot: path=/tmp/raft124009344/snapshots/2-1013-1739083951123.tmp -testLoggerAdapter verbose: server-993631aa-85a8-0845-55f3-9c16d10b6eca: 2025-02-09T06:52:31.125Z [INFO] server-993631aa-85a8-0845-55f3-9c16d10b6eca: copied to local snapshot: bytes=71 - testing.go:149: server-993631aa-85a8-0845-55f3-9c16d10b6eca: 2025-02-09T06:52:31.125Z [INFO] server-993631aa-85a8-0845-55f3-9c16d10b6eca: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-993631aa-85a8-0845-55f3-9c16d10b6eca: 2025-02-09T06:52:31.125Z [INFO] server-993631aa-85a8-0845-55f3-9c16d10b6eca: Installed remote snapshot - testing.go:149: server-993631aa-85a8-0845-55f3-9c16d10b6eca: 2025-02-09T06:52:31.125Z [INFO] server-993631aa-85a8-0845-55f3-9c16d10b6eca: Installed remote snapshot -testLoggerAdapter verbose: server-993631aa-85a8-0845-55f3-9c16d10b6eca: 2025-02-09T06:52:31.125Z [WARN] server-993631aa-85a8-0845-55f3-9c16d10b6eca: skipping application of old log: index=22 - testing.go:149: server-993631aa-85a8-0845-55f3-9c16d10b6eca: 2025-02-09T06:52:31.125Z [WARN] server-993631aa-85a8-0845-55f3-9c16d10b6eca: skipping application of old log: index=22 -testLoggerAdapter verbose: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.125Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: pipelining replication: peer="{Voter server-993631aa-85a8-0845-55f3-9c16d10b6eca 993631aa-85a8-0845-55f3-9c16d10b6eca}" - testing.go:149: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.125Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: pipelining replication: peer="{Voter server-993631aa-85a8-0845-55f3-9c16d10b6eca 993631aa-85a8-0845-55f3-9c16d10b6eca}" -testLoggerAdapter verbose: server-abd23e07-12ed-2b70-1e64-334ea4b1055d: 2025-02-09T06:52:31.126Z [INFO] server-abd23e07-12ed-2b70-1e64-334ea4b1055d: copied to local snapshot: bytes=71 - testing.go:149: server-abd23e07-12ed-2b70-1e64-334ea4b1055d: 2025-02-09T06:52:31.126Z [INFO] server-abd23e07-12ed-2b70-1e64-334ea4b1055d: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-abd23e07-12ed-2b70-1e64-334ea4b1055d: 2025-02-09T06:52:31.126Z [INFO] server-abd23e07-12ed-2b70-1e64-334ea4b1055d: Installed remote snapshot - testing.go:149: server-abd23e07-12ed-2b70-1e64-334ea4b1055d: 2025-02-09T06:52:31.126Z [INFO] server-abd23e07-12ed-2b70-1e64-334ea4b1055d: Installed remote snapshot -testLoggerAdapter verbose: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.127Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: pipelining replication: peer="{Voter server-abd23e07-12ed-2b70-1e64-334ea4b1055d abd23e07-12ed-2b70-1e64-334ea4b1055d}" - testing.go:149: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.127Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: pipelining replication: peer="{Voter server-abd23e07-12ed-2b70-1e64-334ea4b1055d abd23e07-12ed-2b70-1e64-334ea4b1055d}" -testLoggerAdapter verbose: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.142Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: aborting pipeline replication: peer="{Voter server-abd23e07-12ed-2b70-1e64-334ea4b1055d abd23e07-12ed-2b70-1e64-334ea4b1055d}" - testing.go:149: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.142Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: aborting pipeline replication: peer="{Voter server-abd23e07-12ed-2b70-1e64-334ea4b1055d abd23e07-12ed-2b70-1e64-334ea4b1055d}" -testLoggerAdapter verbose: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.142Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: aborting pipeline replication: peer="{Voter server-993631aa-85a8-0845-55f3-9c16d10b6eca 993631aa-85a8-0845-55f3-9c16d10b6eca}" - testing.go:149: server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: 2025-02-09T06:52:31.142Z [INFO] server-3224f6d6-ff0a-ab2c-f705-fd0479ee6a7b: aborting pipeline replication: peer="{Voter server-993631aa-85a8-0845-55f3-9c16d10b6eca 993631aa-85a8-0845-55f3-9c16d10b6eca}" -testLoggerAdapter verbose: server-30a429f1-aec4-9684-97eb-cebe43f05f2c: 2025-02-09T06:52:31.143Z [INFO] server-30a429f1-aec4-9684-97eb-cebe43f05f2c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-30a429f1-aec4-9684-97eb-cebe43f05f2c Address:30a429f1-aec4-9684-97eb-cebe43f05f2c} {Suffrage:Voter ID:server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e Address:f1c4879e-7dc6-3a38-864b-453bc6cabe5e} {Suffrage:Voter ID:server-a7056529-d904-7f8e-13b7-db02fda90a80 Address:a7056529-d904-7f8e-13b7-db02fda90a80}]" - testing.go:149: server-30a429f1-aec4-9684-97eb-cebe43f05f2c: 2025-02-09T06:52:31.143Z [INFO] server-30a429f1-aec4-9684-97eb-cebe43f05f2c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-30a429f1-aec4-9684-97eb-cebe43f05f2c Address:30a429f1-aec4-9684-97eb-cebe43f05f2c} {Suffrage:Voter ID:server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e Address:f1c4879e-7dc6-3a38-864b-453bc6cabe5e} {Suffrage:Voter ID:server-a7056529-d904-7f8e-13b7-db02fda90a80 Address:a7056529-d904-7f8e-13b7-db02fda90a80}]" -testLoggerAdapter verbose: server-30a429f1-aec4-9684-97eb-cebe43f05f2c: 2025-02-09T06:52:31.143Z [INFO] server-30a429f1-aec4-9684-97eb-cebe43f05f2c: entering follower state: follower="Node at 30a429f1-aec4-9684-97eb-cebe43f05f2c [Follower]" leader= - testing.go:149: server-30a429f1-aec4-9684-97eb-cebe43f05f2c: 2025-02-09T06:52:31.143Z [INFO] server-30a429f1-aec4-9684-97eb-cebe43f05f2c: entering follower state: follower="Node at 30a429f1-aec4-9684-97eb-cebe43f05f2c [Follower]" leader= -testLoggerAdapter verbose: server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e: 2025-02-09T06:52:31.143Z [INFO] server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-30a429f1-aec4-9684-97eb-cebe43f05f2c Address:30a429f1-aec4-9684-97eb-cebe43f05f2c} {Suffrage:Voter ID:server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e Address:f1c4879e-7dc6-3a38-864b-453bc6cabe5e} {Suffrage:Voter ID:server-a7056529-d904-7f8e-13b7-db02fda90a80 Address:a7056529-d904-7f8e-13b7-db02fda90a80}]" - testing.go:149: server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e: 2025-02-09T06:52:31.143Z [INFO] server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-30a429f1-aec4-9684-97eb-cebe43f05f2c Address:30a429f1-aec4-9684-97eb-cebe43f05f2c} {Suffrage:Voter ID:server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e Address:f1c4879e-7dc6-3a38-864b-453bc6cabe5e} {Suffrage:Voter ID:server-a7056529-d904-7f8e-13b7-db02fda90a80 Address:a7056529-d904-7f8e-13b7-db02fda90a80}]" -testLoggerAdapter verbose: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.143Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-30a429f1-aec4-9684-97eb-cebe43f05f2c Address:30a429f1-aec4-9684-97eb-cebe43f05f2c} {Suffrage:Voter ID:server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e Address:f1c4879e-7dc6-3a38-864b-453bc6cabe5e} {Suffrage:Voter ID:server-a7056529-d904-7f8e-13b7-db02fda90a80 Address:a7056529-d904-7f8e-13b7-db02fda90a80}]" -testLoggerAdapter verbose: server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e: 2025-02-09T06:52:31.143Z [INFO] server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e: entering follower state: follower="Node at f1c4879e-7dc6-3a38-864b-453bc6cabe5e [Follower]" leader= - testing.go:149: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.143Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-30a429f1-aec4-9684-97eb-cebe43f05f2c Address:30a429f1-aec4-9684-97eb-cebe43f05f2c} {Suffrage:Voter ID:server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e Address:f1c4879e-7dc6-3a38-864b-453bc6cabe5e} {Suffrage:Voter ID:server-a7056529-d904-7f8e-13b7-db02fda90a80 Address:a7056529-d904-7f8e-13b7-db02fda90a80}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:31.143Z [INFO] starting stability test: raft-state=Leader - testing.go:149: server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e: 2025-02-09T06:52:31.143Z [INFO] server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e: entering follower state: follower="Node at f1c4879e-7dc6-3a38-864b-453bc6cabe5e [Follower]" leader= - testing.go:149: cluster: 2025-02-09T06:52:31.143Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.143Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: entering follower state: follower="Node at a7056529-d904-7f8e-13b7-db02fda90a80 [Follower]" leader= - testing.go:149: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.143Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: entering follower state: follower="Node at a7056529-d904-7f8e-13b7-db02fda90a80 [Follower]" leader= -testLoggerAdapter verbose: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.198Z [WARN] server-a7056529-d904-7f8e-13b7-db02fda90a80: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.198Z [WARN] server-a7056529-d904-7f8e-13b7-db02fda90a80: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.198Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: entering candidate state: node="Node at a7056529-d904-7f8e-13b7-db02fda90a80 [Candidate]" term=2 - testing.go:149: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.198Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: entering candidate state: node="Node at a7056529-d904-7f8e-13b7-db02fda90a80 [Candidate]" term=2 -testLoggerAdapter verbose: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.198Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: election won: tally=2 - testing.go:149: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.198Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: election won: tally=2 -testLoggerAdapter verbose: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.198Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: entering leader state: leader="Node at a7056529-d904-7f8e-13b7-db02fda90a80 [Leader]" - testing.go:149: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.198Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: entering leader state: leader="Node at a7056529-d904-7f8e-13b7-db02fda90a80 [Leader]" -testLoggerAdapter verbose: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.198Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: added peer, starting replication: peer=server-30a429f1-aec4-9684-97eb-cebe43f05f2c - testing.go:149: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.198Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: added peer, starting replication: peer=server-30a429f1-aec4-9684-97eb-cebe43f05f2c -testLoggerAdapter verbose: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.198Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: added peer, starting replication: peer=server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e - testing.go:149: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.198Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: added peer, starting replication: peer=server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e -testLoggerAdapter verbose: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.198Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: pipelining replication: peer="{Voter server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e f1c4879e-7dc6-3a38-864b-453bc6cabe5e}" - testing.go:149: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.198Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: pipelining replication: peer="{Voter server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e f1c4879e-7dc6-3a38-864b-453bc6cabe5e}" -testLoggerAdapter verbose: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.198Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: pipelining replication: peer="{Voter server-30a429f1-aec4-9684-97eb-cebe43f05f2c 30a429f1-aec4-9684-97eb-cebe43f05f2c}" - testing.go:149: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.198Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: pipelining replication: peer="{Voter server-30a429f1-aec4-9684-97eb-cebe43f05f2c 30a429f1-aec4-9684-97eb-cebe43f05f2c}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:31.304Z [INFO] stable state for Leader reached at 2025-02-09 06:52:31.198513698 +0000 UTC m=+12.177322462 (1 nodes), 54.621058ms from start of poll, 54.956851ms from cluster start. Timeout at 2025-02-09 06:52:31.304065433 +0000 UTC m=+12.282874228, 105.551766ms after stability - testing.go:149: cluster: 2025-02-09T06:52:31.304Z [INFO] stable state for Leader reached at 2025-02-09 06:52:31.198513698 +0000 UTC m=+12.177322462 (1 nodes), 54.621058ms from start of poll, 54.956851ms from cluster start. Timeout at 2025-02-09 06:52:31.304065433 +0000 UTC m=+12.282874228, 105.551766ms after stability -testLoggerAdapter verbose: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.304Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: starting snapshot up to: index=12 - testing.go:149: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.304Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: starting snapshot up to: index=12 - testing.go:153: 2025-02-09T06:52:31.304Z [INFO] creating new snapshot: path=/tmp/raft3427325543/snapshots/2-12-1739083951304.tmp -testLoggerAdapter verbose: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.307Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: snapshot complete up to: index=12 - testing.go:149: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.307Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: snapshot complete up to: index=12 - testing.go:153: 2025-02-09T06:52:31.308Z [INFO] creating new snapshot: path=/tmp/raft3427325543/snapshots/2-10013-1739083951308.tmp -testLoggerAdapter verbose: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.311Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: copied to local snapshot: bytes=71 - testing.go:149: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.311Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.311Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: restored user snapshot: index=1 - testing.go:149: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.311Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: restored user snapshot: index=1 -testLoggerAdapter verbose: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.311Z [ERROR] server-a7056529-d904-7f8e-13b7-db02fda90a80: failed to get log: index=23 error="log not found" - testing.go:149: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.311Z [ERROR] server-a7056529-d904-7f8e-13b7-db02fda90a80: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.311Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: aborting pipeline replication: peer="{Voter server-30a429f1-aec4-9684-97eb-cebe43f05f2c 30a429f1-aec4-9684-97eb-cebe43f05f2c}" - testing.go:149: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.311Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: aborting pipeline replication: peer="{Voter server-30a429f1-aec4-9684-97eb-cebe43f05f2c 30a429f1-aec4-9684-97eb-cebe43f05f2c}" -testLoggerAdapter verbose: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.311Z [ERROR] server-a7056529-d904-7f8e-13b7-db02fda90a80: failed to get log: index=23 error="log not found" - testing.go:149: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.311Z [ERROR] server-a7056529-d904-7f8e-13b7-db02fda90a80: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.311Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: aborting pipeline replication: peer="{Voter server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e f1c4879e-7dc6-3a38-864b-453bc6cabe5e}" - testing.go:149: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.311Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: aborting pipeline replication: peer="{Voter server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e f1c4879e-7dc6-3a38-864b-453bc6cabe5e}" -testLoggerAdapter verbose: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.319Z [ERROR] server-a7056529-d904-7f8e-13b7-db02fda90a80: failed to get log: index=23 error="log not found" - testing.go:149: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.319Z [ERROR] server-a7056529-d904-7f8e-13b7-db02fda90a80: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.319Z [ERROR] server-a7056529-d904-7f8e-13b7-db02fda90a80: failed to get log: index=23 error="log not found" - testing.go:149: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.319Z [ERROR] server-a7056529-d904-7f8e-13b7-db02fda90a80: failed to get log: index=23 error="log not found" - testing.go:153: 2025-02-09T06:52:31.319Z [INFO] creating new snapshot: path=/tmp/raft4195242853/snapshots/2-10013-1739083951319.tmp - testing.go:153: 2025-02-09T06:52:31.319Z [INFO] creating new snapshot: path=/tmp/raft2597715895/snapshots/2-10013-1739083951319.tmp -testLoggerAdapter verbose: server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e: 2025-02-09T06:52:31.323Z [INFO] server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e: copied to local snapshot: bytes=71 - testing.go:149: server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e: 2025-02-09T06:52:31.323Z [INFO] server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e: 2025-02-09T06:52:31.323Z [INFO] server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e: Installed remote snapshot - testing.go:149: server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e: 2025-02-09T06:52:31.323Z [INFO] server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e: Installed remote snapshot -testLoggerAdapter verbose: server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e: 2025-02-09T06:52:31.323Z [WARN] server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e: skipping application of old log: index=22 - testing.go:149: server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e: 2025-02-09T06:52:31.323Z [WARN] server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e: skipping application of old log: index=22 -testLoggerAdapter verbose: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.323Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: pipelining replication: peer="{Voter server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e f1c4879e-7dc6-3a38-864b-453bc6cabe5e}" - testing.go:149: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.323Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: pipelining replication: peer="{Voter server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e f1c4879e-7dc6-3a38-864b-453bc6cabe5e}" -testLoggerAdapter verbose: server-30a429f1-aec4-9684-97eb-cebe43f05f2c: 2025-02-09T06:52:31.323Z [INFO] server-30a429f1-aec4-9684-97eb-cebe43f05f2c: copied to local snapshot: bytes=71 - testing.go:149: server-30a429f1-aec4-9684-97eb-cebe43f05f2c: 2025-02-09T06:52:31.323Z [INFO] server-30a429f1-aec4-9684-97eb-cebe43f05f2c: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-30a429f1-aec4-9684-97eb-cebe43f05f2c: 2025-02-09T06:52:31.323Z [INFO] server-30a429f1-aec4-9684-97eb-cebe43f05f2c: Installed remote snapshot - testing.go:149: server-30a429f1-aec4-9684-97eb-cebe43f05f2c: 2025-02-09T06:52:31.323Z [INFO] server-30a429f1-aec4-9684-97eb-cebe43f05f2c: Installed remote snapshot -testLoggerAdapter verbose: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.323Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: pipelining replication: peer="{Voter server-30a429f1-aec4-9684-97eb-cebe43f05f2c 30a429f1-aec4-9684-97eb-cebe43f05f2c}" - testing.go:149: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.323Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: pipelining replication: peer="{Voter server-30a429f1-aec4-9684-97eb-cebe43f05f2c 30a429f1-aec4-9684-97eb-cebe43f05f2c}" -testLoggerAdapter verbose: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.340Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: aborting pipeline replication: peer="{Voter server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e f1c4879e-7dc6-3a38-864b-453bc6cabe5e}" - testing.go:149: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.340Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: aborting pipeline replication: peer="{Voter server-f1c4879e-7dc6-3a38-864b-453bc6cabe5e f1c4879e-7dc6-3a38-864b-453bc6cabe5e}" -testLoggerAdapter verbose: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.340Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: aborting pipeline replication: peer="{Voter server-30a429f1-aec4-9684-97eb-cebe43f05f2c 30a429f1-aec4-9684-97eb-cebe43f05f2c}" - testing.go:149: server-a7056529-d904-7f8e-13b7-db02fda90a80: 2025-02-09T06:52:31.340Z [INFO] server-a7056529-d904-7f8e-13b7-db02fda90a80: aborting pipeline replication: peer="{Voter server-30a429f1-aec4-9684-97eb-cebe43f05f2c 30a429f1-aec4-9684-97eb-cebe43f05f2c}" ---- PASS: TestRaft_UserRestore (1.25s) +testLoggerAdapter verbose: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.018Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7050e622-91b0-d318-c4fb-e32d1e4573e9 Address:7050e622-91b0-d318-c4fb-e32d1e4573e9} {Suffrage:Voter ID:server-00eb74b9-fc78-e96d-b62f-9811c7956233 Address:00eb74b9-fc78-e96d-b62f-9811c7956233} {Suffrage:Voter ID:server-73c1b83f-4a4c-e160-fb5a-091047822f4b Address:73c1b83f-4a4c-e160-fb5a-091047822f4b}]" + testing.go:149: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.018Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7050e622-91b0-d318-c4fb-e32d1e4573e9 Address:7050e622-91b0-d318-c4fb-e32d1e4573e9} {Suffrage:Voter ID:server-00eb74b9-fc78-e96d-b62f-9811c7956233 Address:00eb74b9-fc78-e96d-b62f-9811c7956233} {Suffrage:Voter ID:server-73c1b83f-4a4c-e160-fb5a-091047822f4b Address:73c1b83f-4a4c-e160-fb5a-091047822f4b}]" +testLoggerAdapter verbose: server-00eb74b9-fc78-e96d-b62f-9811c7956233: 2024-01-08T00:31:05.018Z [INFO] server-00eb74b9-fc78-e96d-b62f-9811c7956233: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7050e622-91b0-d318-c4fb-e32d1e4573e9 Address:7050e622-91b0-d318-c4fb-e32d1e4573e9} {Suffrage:Voter ID:server-00eb74b9-fc78-e96d-b62f-9811c7956233 Address:00eb74b9-fc78-e96d-b62f-9811c7956233} {Suffrage:Voter ID:server-73c1b83f-4a4c-e160-fb5a-091047822f4b Address:73c1b83f-4a4c-e160-fb5a-091047822f4b}]" + testing.go:149: server-00eb74b9-fc78-e96d-b62f-9811c7956233: 2024-01-08T00:31:05.018Z [INFO] server-00eb74b9-fc78-e96d-b62f-9811c7956233: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7050e622-91b0-d318-c4fb-e32d1e4573e9 Address:7050e622-91b0-d318-c4fb-e32d1e4573e9} {Suffrage:Voter ID:server-00eb74b9-fc78-e96d-b62f-9811c7956233 Address:00eb74b9-fc78-e96d-b62f-9811c7956233} {Suffrage:Voter ID:server-73c1b83f-4a4c-e160-fb5a-091047822f4b Address:73c1b83f-4a4c-e160-fb5a-091047822f4b}]" +testLoggerAdapter verbose: server-73c1b83f-4a4c-e160-fb5a-091047822f4b: 2024-01-08T00:31:05.018Z [INFO] server-73c1b83f-4a4c-e160-fb5a-091047822f4b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7050e622-91b0-d318-c4fb-e32d1e4573e9 Address:7050e622-91b0-d318-c4fb-e32d1e4573e9} {Suffrage:Voter ID:server-00eb74b9-fc78-e96d-b62f-9811c7956233 Address:00eb74b9-fc78-e96d-b62f-9811c7956233} {Suffrage:Voter ID:server-73c1b83f-4a4c-e160-fb5a-091047822f4b Address:73c1b83f-4a4c-e160-fb5a-091047822f4b}]" + testing.go:149: server-73c1b83f-4a4c-e160-fb5a-091047822f4b: 2024-01-08T00:31:05.018Z [INFO] server-73c1b83f-4a4c-e160-fb5a-091047822f4b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7050e622-91b0-d318-c4fb-e32d1e4573e9 Address:7050e622-91b0-d318-c4fb-e32d1e4573e9} {Suffrage:Voter ID:server-00eb74b9-fc78-e96d-b62f-9811c7956233 Address:00eb74b9-fc78-e96d-b62f-9811c7956233} {Suffrage:Voter ID:server-73c1b83f-4a4c-e160-fb5a-091047822f4b Address:73c1b83f-4a4c-e160-fb5a-091047822f4b}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:05.018Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:05.018Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.018Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: entering follower state: follower="Node at 7050e622-91b0-d318-c4fb-e32d1e4573e9 [Follower]" leader= + testing.go:149: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.018Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: entering follower state: follower="Node at 7050e622-91b0-d318-c4fb-e32d1e4573e9 [Follower]" leader= +testLoggerAdapter verbose: server-00eb74b9-fc78-e96d-b62f-9811c7956233: 2024-01-08T00:31:05.018Z [INFO] server-00eb74b9-fc78-e96d-b62f-9811c7956233: entering follower state: follower="Node at 00eb74b9-fc78-e96d-b62f-9811c7956233 [Follower]" leader= + testing.go:149: server-00eb74b9-fc78-e96d-b62f-9811c7956233: 2024-01-08T00:31:05.018Z [INFO] server-00eb74b9-fc78-e96d-b62f-9811c7956233: entering follower state: follower="Node at 00eb74b9-fc78-e96d-b62f-9811c7956233 [Follower]" leader= +testLoggerAdapter verbose: server-73c1b83f-4a4c-e160-fb5a-091047822f4b: 2024-01-08T00:31:05.019Z [INFO] server-73c1b83f-4a4c-e160-fb5a-091047822f4b: entering follower state: follower="Node at 73c1b83f-4a4c-e160-fb5a-091047822f4b [Follower]" leader= + testing.go:149: server-73c1b83f-4a4c-e160-fb5a-091047822f4b: 2024-01-08T00:31:05.019Z [INFO] server-73c1b83f-4a4c-e160-fb5a-091047822f4b: entering follower state: follower="Node at 73c1b83f-4a4c-e160-fb5a-091047822f4b [Follower]" leader= +testLoggerAdapter verbose: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.083Z [WARN] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.083Z [WARN] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.083Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: entering candidate state: node="Node at 7050e622-91b0-d318-c4fb-e32d1e4573e9 [Candidate]" term=2 + testing.go:149: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.083Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: entering candidate state: node="Node at 7050e622-91b0-d318-c4fb-e32d1e4573e9 [Candidate]" term=2 +testLoggerAdapter verbose: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.083Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: election won: tally=2 + testing.go:149: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.083Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: election won: tally=2 +testLoggerAdapter verbose: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.083Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: entering leader state: leader="Node at 7050e622-91b0-d318-c4fb-e32d1e4573e9 [Leader]" + testing.go:149: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.083Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: entering leader state: leader="Node at 7050e622-91b0-d318-c4fb-e32d1e4573e9 [Leader]" +testLoggerAdapter verbose: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.083Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: added peer, starting replication: peer=server-00eb74b9-fc78-e96d-b62f-9811c7956233 + testing.go:149: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.083Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: added peer, starting replication: peer=server-00eb74b9-fc78-e96d-b62f-9811c7956233 +testLoggerAdapter verbose: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.083Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: added peer, starting replication: peer=server-73c1b83f-4a4c-e160-fb5a-091047822f4b + testing.go:149: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.083Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: added peer, starting replication: peer=server-73c1b83f-4a4c-e160-fb5a-091047822f4b +testLoggerAdapter verbose: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.083Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: pipelining replication: peer="{Voter server-73c1b83f-4a4c-e160-fb5a-091047822f4b 73c1b83f-4a4c-e160-fb5a-091047822f4b}" + testing.go:149: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.083Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: pipelining replication: peer="{Voter server-73c1b83f-4a4c-e160-fb5a-091047822f4b 73c1b83f-4a4c-e160-fb5a-091047822f4b}" +testLoggerAdapter verbose: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.083Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: pipelining replication: peer="{Voter server-00eb74b9-fc78-e96d-b62f-9811c7956233 00eb74b9-fc78-e96d-b62f-9811c7956233}" + testing.go:149: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.083Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: pipelining replication: peer="{Voter server-00eb74b9-fc78-e96d-b62f-9811c7956233 00eb74b9-fc78-e96d-b62f-9811c7956233}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:05.189Z [INFO] stable state for Leader reached at 2024-01-08 00:31:05.083807014 +0000 UTC m=+11.130696055 (1 nodes), 64.881517ms from start of poll, 65.237087ms from cluster start. Timeout at 2024-01-08 00:31:05.189486328 +0000 UTC m=+11.236375413, 105.679358ms after stability + testing.go:149: cluster: 2024-01-08T00:31:05.189Z [INFO] stable state for Leader reached at 2024-01-08 00:31:05.083807014 +0000 UTC m=+11.130696055 (1 nodes), 64.881517ms from start of poll, 65.237087ms from cluster start. Timeout at 2024-01-08 00:31:05.189486328 +0000 UTC m=+11.236375413, 105.679358ms after stability +testLoggerAdapter verbose: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.190Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: starting snapshot up to: index=12 + testing.go:149: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.190Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: starting snapshot up to: index=12 + testing.go:153: 2024-01-08T00:31:05.190Z [INFO] creating new snapshot: path=/tmp/raft3821454391/snapshots/2-12-1704673865190.tmp +testLoggerAdapter verbose: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.198Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: snapshot complete up to: index=12 + testing.go:149: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.198Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: snapshot complete up to: index=12 + testing.go:153: 2024-01-08T00:31:05.198Z [INFO] creating new snapshot: path=/tmp/raft3821454391/snapshots/2-23-1704673865198.tmp +testLoggerAdapter verbose: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.206Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: copied to local snapshot: bytes=71 + testing.go:149: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.206Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.207Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: restored user snapshot: index=1 + testing.go:149: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.207Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: restored user snapshot: index=1 +testLoggerAdapter verbose: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.207Z [ERROR] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: failed to get log: index=23 error="log not found" + testing.go:149: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.207Z [ERROR] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.207Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: aborting pipeline replication: peer="{Voter server-73c1b83f-4a4c-e160-fb5a-091047822f4b 73c1b83f-4a4c-e160-fb5a-091047822f4b}" + testing.go:149: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.207Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: aborting pipeline replication: peer="{Voter server-73c1b83f-4a4c-e160-fb5a-091047822f4b 73c1b83f-4a4c-e160-fb5a-091047822f4b}" +testLoggerAdapter verbose: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.207Z [ERROR] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: failed to get log: index=23 error="log not found" + testing.go:149: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.207Z [ERROR] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.207Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: aborting pipeline replication: peer="{Voter server-00eb74b9-fc78-e96d-b62f-9811c7956233 00eb74b9-fc78-e96d-b62f-9811c7956233}" + testing.go:149: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.207Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: aborting pipeline replication: peer="{Voter server-00eb74b9-fc78-e96d-b62f-9811c7956233 00eb74b9-fc78-e96d-b62f-9811c7956233}" +testLoggerAdapter verbose: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.213Z [ERROR] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: failed to get log: index=23 error="log not found" + testing.go:149: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.213Z [ERROR] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: failed to get log: index=23 error="log not found" + testing.go:153: 2024-01-08T00:31:05.214Z [INFO] creating new snapshot: path=/tmp/raft2450759202/snapshots/2-23-1704673865214.tmp +testLoggerAdapter verbose: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.215Z [ERROR] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: failed to get log: index=23 error="log not found" + testing.go:149: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.215Z [ERROR] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: failed to get log: index=23 error="log not found" + testing.go:153: 2024-01-08T00:31:05.216Z [INFO] creating new snapshot: path=/tmp/raft2422308845/snapshots/2-23-1704673865216.tmp +testLoggerAdapter verbose: server-00eb74b9-fc78-e96d-b62f-9811c7956233: 2024-01-08T00:31:05.226Z [INFO] server-00eb74b9-fc78-e96d-b62f-9811c7956233: copied to local snapshot: bytes=71 + testing.go:149: server-00eb74b9-fc78-e96d-b62f-9811c7956233: 2024-01-08T00:31:05.226Z [INFO] server-00eb74b9-fc78-e96d-b62f-9811c7956233: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-00eb74b9-fc78-e96d-b62f-9811c7956233: 2024-01-08T00:31:05.226Z [INFO] server-00eb74b9-fc78-e96d-b62f-9811c7956233: Installed remote snapshot + testing.go:149: server-00eb74b9-fc78-e96d-b62f-9811c7956233: 2024-01-08T00:31:05.226Z [INFO] server-00eb74b9-fc78-e96d-b62f-9811c7956233: Installed remote snapshot +testLoggerAdapter verbose: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.226Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: pipelining replication: peer="{Voter server-00eb74b9-fc78-e96d-b62f-9811c7956233 00eb74b9-fc78-e96d-b62f-9811c7956233}" + testing.go:149: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.226Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: pipelining replication: peer="{Voter server-00eb74b9-fc78-e96d-b62f-9811c7956233 00eb74b9-fc78-e96d-b62f-9811c7956233}" +testLoggerAdapter verbose: server-73c1b83f-4a4c-e160-fb5a-091047822f4b: 2024-01-08T00:31:05.226Z [INFO] server-73c1b83f-4a4c-e160-fb5a-091047822f4b: copied to local snapshot: bytes=71 + testing.go:149: server-73c1b83f-4a4c-e160-fb5a-091047822f4b: 2024-01-08T00:31:05.226Z [INFO] server-73c1b83f-4a4c-e160-fb5a-091047822f4b: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-73c1b83f-4a4c-e160-fb5a-091047822f4b: 2024-01-08T00:31:05.226Z [INFO] server-73c1b83f-4a4c-e160-fb5a-091047822f4b: Installed remote snapshot + testing.go:149: server-73c1b83f-4a4c-e160-fb5a-091047822f4b: 2024-01-08T00:31:05.226Z [INFO] server-73c1b83f-4a4c-e160-fb5a-091047822f4b: Installed remote snapshot +testLoggerAdapter verbose: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.227Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: pipelining replication: peer="{Voter server-73c1b83f-4a4c-e160-fb5a-091047822f4b 73c1b83f-4a4c-e160-fb5a-091047822f4b}" + testing.go:149: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.227Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: pipelining replication: peer="{Voter server-73c1b83f-4a4c-e160-fb5a-091047822f4b 73c1b83f-4a4c-e160-fb5a-091047822f4b}" +testLoggerAdapter verbose: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.244Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: aborting pipeline replication: peer="{Voter server-73c1b83f-4a4c-e160-fb5a-091047822f4b 73c1b83f-4a4c-e160-fb5a-091047822f4b}" + testing.go:149: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.244Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: aborting pipeline replication: peer="{Voter server-73c1b83f-4a4c-e160-fb5a-091047822f4b 73c1b83f-4a4c-e160-fb5a-091047822f4b}" +testLoggerAdapter verbose: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.244Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: aborting pipeline replication: peer="{Voter server-00eb74b9-fc78-e96d-b62f-9811c7956233 00eb74b9-fc78-e96d-b62f-9811c7956233}" + testing.go:149: server-7050e622-91b0-d318-c4fb-e32d1e4573e9: 2024-01-08T00:31:05.244Z [INFO] server-7050e622-91b0-d318-c4fb-e32d1e4573e9: aborting pipeline replication: peer="{Voter server-00eb74b9-fc78-e96d-b62f-9811c7956233 00eb74b9-fc78-e96d-b62f-9811c7956233}" +testLoggerAdapter verbose: server-2af0776a-5a1b-5a54-d2d5-5626395e9402: 2024-01-08T00:31:05.246Z [INFO] server-2af0776a-5a1b-5a54-d2d5-5626395e9402: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2af0776a-5a1b-5a54-d2d5-5626395e9402 Address:2af0776a-5a1b-5a54-d2d5-5626395e9402} {Suffrage:Voter ID:server-37991d03-32aa-cfcf-1adb-12eaa3b69498 Address:37991d03-32aa-cfcf-1adb-12eaa3b69498} {Suffrage:Voter ID:server-792e1d30-30af-6ad5-b688-5c14ba5d38c9 Address:792e1d30-30af-6ad5-b688-5c14ba5d38c9}]" + testing.go:149: server-2af0776a-5a1b-5a54-d2d5-5626395e9402: 2024-01-08T00:31:05.246Z [INFO] server-2af0776a-5a1b-5a54-d2d5-5626395e9402: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2af0776a-5a1b-5a54-d2d5-5626395e9402 Address:2af0776a-5a1b-5a54-d2d5-5626395e9402} {Suffrage:Voter ID:server-37991d03-32aa-cfcf-1adb-12eaa3b69498 Address:37991d03-32aa-cfcf-1adb-12eaa3b69498} {Suffrage:Voter ID:server-792e1d30-30af-6ad5-b688-5c14ba5d38c9 Address:792e1d30-30af-6ad5-b688-5c14ba5d38c9}]" +testLoggerAdapter verbose: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.246Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2af0776a-5a1b-5a54-d2d5-5626395e9402 Address:2af0776a-5a1b-5a54-d2d5-5626395e9402} {Suffrage:Voter ID:server-37991d03-32aa-cfcf-1adb-12eaa3b69498 Address:37991d03-32aa-cfcf-1adb-12eaa3b69498} {Suffrage:Voter ID:server-792e1d30-30af-6ad5-b688-5c14ba5d38c9 Address:792e1d30-30af-6ad5-b688-5c14ba5d38c9}]" + testing.go:149: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.246Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2af0776a-5a1b-5a54-d2d5-5626395e9402 Address:2af0776a-5a1b-5a54-d2d5-5626395e9402} {Suffrage:Voter ID:server-37991d03-32aa-cfcf-1adb-12eaa3b69498 Address:37991d03-32aa-cfcf-1adb-12eaa3b69498} {Suffrage:Voter ID:server-792e1d30-30af-6ad5-b688-5c14ba5d38c9 Address:792e1d30-30af-6ad5-b688-5c14ba5d38c9}]" +testLoggerAdapter verbose: server-792e1d30-30af-6ad5-b688-5c14ba5d38c9: 2024-01-08T00:31:05.246Z [INFO] server-792e1d30-30af-6ad5-b688-5c14ba5d38c9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2af0776a-5a1b-5a54-d2d5-5626395e9402 Address:2af0776a-5a1b-5a54-d2d5-5626395e9402} {Suffrage:Voter ID:server-37991d03-32aa-cfcf-1adb-12eaa3b69498 Address:37991d03-32aa-cfcf-1adb-12eaa3b69498} {Suffrage:Voter ID:server-792e1d30-30af-6ad5-b688-5c14ba5d38c9 Address:792e1d30-30af-6ad5-b688-5c14ba5d38c9}]" + testing.go:149: server-792e1d30-30af-6ad5-b688-5c14ba5d38c9: 2024-01-08T00:31:05.246Z [INFO] server-792e1d30-30af-6ad5-b688-5c14ba5d38c9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2af0776a-5a1b-5a54-d2d5-5626395e9402 Address:2af0776a-5a1b-5a54-d2d5-5626395e9402} {Suffrage:Voter ID:server-37991d03-32aa-cfcf-1adb-12eaa3b69498 Address:37991d03-32aa-cfcf-1adb-12eaa3b69498} {Suffrage:Voter ID:server-792e1d30-30af-6ad5-b688-5c14ba5d38c9 Address:792e1d30-30af-6ad5-b688-5c14ba5d38c9}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:05.247Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:05.247Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.247Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: entering follower state: follower="Node at 37991d03-32aa-cfcf-1adb-12eaa3b69498 [Follower]" leader= + testing.go:149: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.247Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: entering follower state: follower="Node at 37991d03-32aa-cfcf-1adb-12eaa3b69498 [Follower]" leader= +testLoggerAdapter verbose: server-2af0776a-5a1b-5a54-d2d5-5626395e9402: 2024-01-08T00:31:05.248Z [INFO] server-2af0776a-5a1b-5a54-d2d5-5626395e9402: entering follower state: follower="Node at 2af0776a-5a1b-5a54-d2d5-5626395e9402 [Follower]" leader= + testing.go:149: server-2af0776a-5a1b-5a54-d2d5-5626395e9402: 2024-01-08T00:31:05.248Z [INFO] server-2af0776a-5a1b-5a54-d2d5-5626395e9402: entering follower state: follower="Node at 2af0776a-5a1b-5a54-d2d5-5626395e9402 [Follower]" leader= +testLoggerAdapter verbose: server-792e1d30-30af-6ad5-b688-5c14ba5d38c9: 2024-01-08T00:31:05.248Z [INFO] server-792e1d30-30af-6ad5-b688-5c14ba5d38c9: entering follower state: follower="Node at 792e1d30-30af-6ad5-b688-5c14ba5d38c9 [Follower]" leader= + testing.go:149: server-792e1d30-30af-6ad5-b688-5c14ba5d38c9: 2024-01-08T00:31:05.248Z [INFO] server-792e1d30-30af-6ad5-b688-5c14ba5d38c9: entering follower state: follower="Node at 792e1d30-30af-6ad5-b688-5c14ba5d38c9 [Follower]" leader= +testLoggerAdapter verbose: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.313Z [WARN] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.313Z [WARN] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.313Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: entering candidate state: node="Node at 37991d03-32aa-cfcf-1adb-12eaa3b69498 [Candidate]" term=2 + testing.go:149: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.313Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: entering candidate state: node="Node at 37991d03-32aa-cfcf-1adb-12eaa3b69498 [Candidate]" term=2 +testLoggerAdapter verbose: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.313Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: election won: tally=2 + testing.go:149: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.313Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: election won: tally=2 +testLoggerAdapter verbose: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.313Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: entering leader state: leader="Node at 37991d03-32aa-cfcf-1adb-12eaa3b69498 [Leader]" + testing.go:149: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.313Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: entering leader state: leader="Node at 37991d03-32aa-cfcf-1adb-12eaa3b69498 [Leader]" +testLoggerAdapter verbose: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.314Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: added peer, starting replication: peer=server-2af0776a-5a1b-5a54-d2d5-5626395e9402 + testing.go:149: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.314Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: added peer, starting replication: peer=server-2af0776a-5a1b-5a54-d2d5-5626395e9402 +testLoggerAdapter verbose: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.314Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: added peer, starting replication: peer=server-792e1d30-30af-6ad5-b688-5c14ba5d38c9 + testing.go:149: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.314Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: added peer, starting replication: peer=server-792e1d30-30af-6ad5-b688-5c14ba5d38c9 +testLoggerAdapter verbose: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.314Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: pipelining replication: peer="{Voter server-792e1d30-30af-6ad5-b688-5c14ba5d38c9 792e1d30-30af-6ad5-b688-5c14ba5d38c9}" + testing.go:149: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.314Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: pipelining replication: peer="{Voter server-792e1d30-30af-6ad5-b688-5c14ba5d38c9 792e1d30-30af-6ad5-b688-5c14ba5d38c9}" +testLoggerAdapter verbose: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.314Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: pipelining replication: peer="{Voter server-2af0776a-5a1b-5a54-d2d5-5626395e9402 2af0776a-5a1b-5a54-d2d5-5626395e9402}" + testing.go:149: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.314Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: pipelining replication: peer="{Voter server-2af0776a-5a1b-5a54-d2d5-5626395e9402 2af0776a-5a1b-5a54-d2d5-5626395e9402}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:05.419Z [INFO] stable state for Leader reached at 2024-01-08 00:31:05.314299318 +0000 UTC m=+11.361188364 (1 nodes), 67.073214ms from start of poll, 67.906081ms from cluster start. Timeout at 2024-01-08 00:31:05.419573012 +0000 UTC m=+11.466462053, 105.273689ms after stability + testing.go:149: cluster: 2024-01-08T00:31:05.419Z [INFO] stable state for Leader reached at 2024-01-08 00:31:05.314299318 +0000 UTC m=+11.361188364 (1 nodes), 67.073214ms from start of poll, 67.906081ms from cluster start. Timeout at 2024-01-08 00:31:05.419573012 +0000 UTC m=+11.466462053, 105.273689ms after stability +testLoggerAdapter verbose: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.420Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: starting snapshot up to: index=12 + testing.go:149: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.420Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: starting snapshot up to: index=12 + testing.go:153: 2024-01-08T00:31:05.420Z [INFO] creating new snapshot: path=/tmp/raft1640412050/snapshots/2-12-1704673865420.tmp +testLoggerAdapter verbose: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.428Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: snapshot complete up to: index=12 + testing.go:149: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.428Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: snapshot complete up to: index=12 + testing.go:153: 2024-01-08T00:31:05.429Z [INFO] creating new snapshot: path=/tmp/raft1640412050/snapshots/2-23-1704673865429.tmp +testLoggerAdapter verbose: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.437Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: copied to local snapshot: bytes=71 + testing.go:149: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.437Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.437Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: restored user snapshot: index=1 + testing.go:149: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.437Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: restored user snapshot: index=1 +testLoggerAdapter verbose: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.437Z [ERROR] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: failed to get log: index=23 error="log not found" + testing.go:149: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.437Z [ERROR] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.437Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: aborting pipeline replication: peer="{Voter server-792e1d30-30af-6ad5-b688-5c14ba5d38c9 792e1d30-30af-6ad5-b688-5c14ba5d38c9}" + testing.go:149: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.437Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: aborting pipeline replication: peer="{Voter server-792e1d30-30af-6ad5-b688-5c14ba5d38c9 792e1d30-30af-6ad5-b688-5c14ba5d38c9}" +testLoggerAdapter verbose: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.437Z [ERROR] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: failed to get log: index=23 error="log not found" + testing.go:149: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.437Z [ERROR] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.437Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: aborting pipeline replication: peer="{Voter server-2af0776a-5a1b-5a54-d2d5-5626395e9402 2af0776a-5a1b-5a54-d2d5-5626395e9402}" + testing.go:149: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.437Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: aborting pipeline replication: peer="{Voter server-2af0776a-5a1b-5a54-d2d5-5626395e9402 2af0776a-5a1b-5a54-d2d5-5626395e9402}" +testLoggerAdapter verbose: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.444Z [ERROR] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: failed to get log: index=23 error="log not found" + testing.go:149: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.444Z [ERROR] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: failed to get log: index=23 error="log not found" + testing.go:153: 2024-01-08T00:31:05.444Z [INFO] creating new snapshot: path=/tmp/raft2790635236/snapshots/2-23-1704673865444.tmp +testLoggerAdapter verbose: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.447Z [ERROR] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: failed to get log: index=23 error="log not found" + testing.go:149: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.447Z [ERROR] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: failed to get log: index=23 error="log not found" + testing.go:153: 2024-01-08T00:31:05.447Z [INFO] creating new snapshot: path=/tmp/raft889931139/snapshots/2-23-1704673865447.tmp +testLoggerAdapter verbose: server-2af0776a-5a1b-5a54-d2d5-5626395e9402: 2024-01-08T00:31:05.457Z [INFO] server-2af0776a-5a1b-5a54-d2d5-5626395e9402: copied to local snapshot: bytes=71 + testing.go:149: server-2af0776a-5a1b-5a54-d2d5-5626395e9402: 2024-01-08T00:31:05.457Z [INFO] server-2af0776a-5a1b-5a54-d2d5-5626395e9402: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-2af0776a-5a1b-5a54-d2d5-5626395e9402: 2024-01-08T00:31:05.457Z [INFO] server-2af0776a-5a1b-5a54-d2d5-5626395e9402: Installed remote snapshot + testing.go:149: server-2af0776a-5a1b-5a54-d2d5-5626395e9402: 2024-01-08T00:31:05.457Z [INFO] server-2af0776a-5a1b-5a54-d2d5-5626395e9402: Installed remote snapshot +testLoggerAdapter verbose: server-2af0776a-5a1b-5a54-d2d5-5626395e9402: 2024-01-08T00:31:05.457Z [WARN] server-2af0776a-5a1b-5a54-d2d5-5626395e9402: skipping application of old log: index=22 + testing.go:149: server-2af0776a-5a1b-5a54-d2d5-5626395e9402: 2024-01-08T00:31:05.457Z [WARN] server-2af0776a-5a1b-5a54-d2d5-5626395e9402: skipping application of old log: index=22 +testLoggerAdapter verbose: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.457Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: pipelining replication: peer="{Voter server-2af0776a-5a1b-5a54-d2d5-5626395e9402 2af0776a-5a1b-5a54-d2d5-5626395e9402}" + testing.go:149: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.457Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: pipelining replication: peer="{Voter server-2af0776a-5a1b-5a54-d2d5-5626395e9402 2af0776a-5a1b-5a54-d2d5-5626395e9402}" +testLoggerAdapter verbose: server-792e1d30-30af-6ad5-b688-5c14ba5d38c9: 2024-01-08T00:31:05.459Z [INFO] server-792e1d30-30af-6ad5-b688-5c14ba5d38c9: copied to local snapshot: bytes=71 + testing.go:149: server-792e1d30-30af-6ad5-b688-5c14ba5d38c9: 2024-01-08T00:31:05.459Z [INFO] server-792e1d30-30af-6ad5-b688-5c14ba5d38c9: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-792e1d30-30af-6ad5-b688-5c14ba5d38c9: 2024-01-08T00:31:05.459Z [INFO] server-792e1d30-30af-6ad5-b688-5c14ba5d38c9: Installed remote snapshot + testing.go:149: server-792e1d30-30af-6ad5-b688-5c14ba5d38c9: 2024-01-08T00:31:05.459Z [INFO] server-792e1d30-30af-6ad5-b688-5c14ba5d38c9: Installed remote snapshot +testLoggerAdapter verbose: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.459Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: pipelining replication: peer="{Voter server-792e1d30-30af-6ad5-b688-5c14ba5d38c9 792e1d30-30af-6ad5-b688-5c14ba5d38c9}" + testing.go:149: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.459Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: pipelining replication: peer="{Voter server-792e1d30-30af-6ad5-b688-5c14ba5d38c9 792e1d30-30af-6ad5-b688-5c14ba5d38c9}" +testLoggerAdapter verbose: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.475Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: aborting pipeline replication: peer="{Voter server-792e1d30-30af-6ad5-b688-5c14ba5d38c9 792e1d30-30af-6ad5-b688-5c14ba5d38c9}" + testing.go:149: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.475Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: aborting pipeline replication: peer="{Voter server-792e1d30-30af-6ad5-b688-5c14ba5d38c9 792e1d30-30af-6ad5-b688-5c14ba5d38c9}" +testLoggerAdapter verbose: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.475Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: aborting pipeline replication: peer="{Voter server-2af0776a-5a1b-5a54-d2d5-5626395e9402 2af0776a-5a1b-5a54-d2d5-5626395e9402}" + testing.go:149: server-37991d03-32aa-cfcf-1adb-12eaa3b69498: 2024-01-08T00:31:05.475Z [INFO] server-37991d03-32aa-cfcf-1adb-12eaa3b69498: aborting pipeline replication: peer="{Voter server-2af0776a-5a1b-5a54-d2d5-5626395e9402 2af0776a-5a1b-5a54-d2d5-5626395e9402}" +testLoggerAdapter verbose: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.476Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9353476e-b9c8-84b8-dc54-c291e86420bf Address:9353476e-b9c8-84b8-dc54-c291e86420bf} {Suffrage:Voter ID:server-237d4f9a-e353-e960-cae3-f2ef9d843b0f Address:237d4f9a-e353-e960-cae3-f2ef9d843b0f} {Suffrage:Voter ID:server-ae651032-d12d-3749-0071-dc6b6cbe41b6 Address:ae651032-d12d-3749-0071-dc6b6cbe41b6}]" + testing.go:149: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.476Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9353476e-b9c8-84b8-dc54-c291e86420bf Address:9353476e-b9c8-84b8-dc54-c291e86420bf} {Suffrage:Voter ID:server-237d4f9a-e353-e960-cae3-f2ef9d843b0f Address:237d4f9a-e353-e960-cae3-f2ef9d843b0f} {Suffrage:Voter ID:server-ae651032-d12d-3749-0071-dc6b6cbe41b6 Address:ae651032-d12d-3749-0071-dc6b6cbe41b6}]" +testLoggerAdapter verbose: server-237d4f9a-e353-e960-cae3-f2ef9d843b0f: 2024-01-08T00:31:05.476Z [INFO] server-237d4f9a-e353-e960-cae3-f2ef9d843b0f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9353476e-b9c8-84b8-dc54-c291e86420bf Address:9353476e-b9c8-84b8-dc54-c291e86420bf} {Suffrage:Voter ID:server-237d4f9a-e353-e960-cae3-f2ef9d843b0f Address:237d4f9a-e353-e960-cae3-f2ef9d843b0f} {Suffrage:Voter ID:server-ae651032-d12d-3749-0071-dc6b6cbe41b6 Address:ae651032-d12d-3749-0071-dc6b6cbe41b6}]" + testing.go:149: server-237d4f9a-e353-e960-cae3-f2ef9d843b0f: 2024-01-08T00:31:05.476Z [INFO] server-237d4f9a-e353-e960-cae3-f2ef9d843b0f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9353476e-b9c8-84b8-dc54-c291e86420bf Address:9353476e-b9c8-84b8-dc54-c291e86420bf} {Suffrage:Voter ID:server-237d4f9a-e353-e960-cae3-f2ef9d843b0f Address:237d4f9a-e353-e960-cae3-f2ef9d843b0f} {Suffrage:Voter ID:server-ae651032-d12d-3749-0071-dc6b6cbe41b6 Address:ae651032-d12d-3749-0071-dc6b6cbe41b6}]" +testLoggerAdapter verbose: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.476Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: entering follower state: follower="Node at 9353476e-b9c8-84b8-dc54-c291e86420bf [Follower]" leader= + testing.go:149: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.476Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: entering follower state: follower="Node at 9353476e-b9c8-84b8-dc54-c291e86420bf [Follower]" leader= +testLoggerAdapter verbose: server-ae651032-d12d-3749-0071-dc6b6cbe41b6: 2024-01-08T00:31:05.476Z [INFO] server-ae651032-d12d-3749-0071-dc6b6cbe41b6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9353476e-b9c8-84b8-dc54-c291e86420bf Address:9353476e-b9c8-84b8-dc54-c291e86420bf} {Suffrage:Voter ID:server-237d4f9a-e353-e960-cae3-f2ef9d843b0f Address:237d4f9a-e353-e960-cae3-f2ef9d843b0f} {Suffrage:Voter ID:server-ae651032-d12d-3749-0071-dc6b6cbe41b6 Address:ae651032-d12d-3749-0071-dc6b6cbe41b6}]" + testing.go:149: server-ae651032-d12d-3749-0071-dc6b6cbe41b6: 2024-01-08T00:31:05.476Z [INFO] server-ae651032-d12d-3749-0071-dc6b6cbe41b6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9353476e-b9c8-84b8-dc54-c291e86420bf Address:9353476e-b9c8-84b8-dc54-c291e86420bf} {Suffrage:Voter ID:server-237d4f9a-e353-e960-cae3-f2ef9d843b0f Address:237d4f9a-e353-e960-cae3-f2ef9d843b0f} {Suffrage:Voter ID:server-ae651032-d12d-3749-0071-dc6b6cbe41b6 Address:ae651032-d12d-3749-0071-dc6b6cbe41b6}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:05.477Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:05.477Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-237d4f9a-e353-e960-cae3-f2ef9d843b0f: 2024-01-08T00:31:05.476Z [INFO] server-237d4f9a-e353-e960-cae3-f2ef9d843b0f: entering follower state: follower="Node at 237d4f9a-e353-e960-cae3-f2ef9d843b0f [Follower]" leader= + testing.go:149: server-237d4f9a-e353-e960-cae3-f2ef9d843b0f: 2024-01-08T00:31:05.476Z [INFO] server-237d4f9a-e353-e960-cae3-f2ef9d843b0f: entering follower state: follower="Node at 237d4f9a-e353-e960-cae3-f2ef9d843b0f [Follower]" leader= +testLoggerAdapter verbose: server-ae651032-d12d-3749-0071-dc6b6cbe41b6: 2024-01-08T00:31:05.477Z [INFO] server-ae651032-d12d-3749-0071-dc6b6cbe41b6: entering follower state: follower="Node at ae651032-d12d-3749-0071-dc6b6cbe41b6 [Follower]" leader= + testing.go:149: server-ae651032-d12d-3749-0071-dc6b6cbe41b6: 2024-01-08T00:31:05.477Z [INFO] server-ae651032-d12d-3749-0071-dc6b6cbe41b6: entering follower state: follower="Node at ae651032-d12d-3749-0071-dc6b6cbe41b6 [Follower]" leader= +testLoggerAdapter verbose: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.534Z [WARN] server-9353476e-b9c8-84b8-dc54-c291e86420bf: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.534Z [WARN] server-9353476e-b9c8-84b8-dc54-c291e86420bf: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.534Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: entering candidate state: node="Node at 9353476e-b9c8-84b8-dc54-c291e86420bf [Candidate]" term=2 + testing.go:149: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.534Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: entering candidate state: node="Node at 9353476e-b9c8-84b8-dc54-c291e86420bf [Candidate]" term=2 +testLoggerAdapter verbose: server-ae651032-d12d-3749-0071-dc6b6cbe41b6: 2024-01-08T00:31:05.534Z [WARN] server-ae651032-d12d-3749-0071-dc6b6cbe41b6: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-ae651032-d12d-3749-0071-dc6b6cbe41b6: 2024-01-08T00:31:05.534Z [WARN] server-ae651032-d12d-3749-0071-dc6b6cbe41b6: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-ae651032-d12d-3749-0071-dc6b6cbe41b6: 2024-01-08T00:31:05.534Z [INFO] server-ae651032-d12d-3749-0071-dc6b6cbe41b6: entering candidate state: node="Node at ae651032-d12d-3749-0071-dc6b6cbe41b6 [Candidate]" term=2 + testing.go:149: server-ae651032-d12d-3749-0071-dc6b6cbe41b6: 2024-01-08T00:31:05.534Z [INFO] server-ae651032-d12d-3749-0071-dc6b6cbe41b6: entering candidate state: node="Node at ae651032-d12d-3749-0071-dc6b6cbe41b6 [Candidate]" term=2 +testLoggerAdapter verbose: server-ae651032-d12d-3749-0071-dc6b6cbe41b6: 2024-01-08T00:31:05.534Z [INFO] server-ae651032-d12d-3749-0071-dc6b6cbe41b6: duplicate requestVote for same term: term=2 + testing.go:149: server-ae651032-d12d-3749-0071-dc6b6cbe41b6: 2024-01-08T00:31:05.534Z [INFO] server-ae651032-d12d-3749-0071-dc6b6cbe41b6: duplicate requestVote for same term: term=2 +testLoggerAdapter verbose: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.534Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: election won: tally=2 + testing.go:149: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.534Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: election won: tally=2 +testLoggerAdapter verbose: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.534Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: entering leader state: leader="Node at 9353476e-b9c8-84b8-dc54-c291e86420bf [Leader]" + testing.go:149: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.534Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: entering leader state: leader="Node at 9353476e-b9c8-84b8-dc54-c291e86420bf [Leader]" +testLoggerAdapter verbose: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.534Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: added peer, starting replication: peer=server-237d4f9a-e353-e960-cae3-f2ef9d843b0f + testing.go:149: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.534Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: added peer, starting replication: peer=server-237d4f9a-e353-e960-cae3-f2ef9d843b0f +testLoggerAdapter verbose: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.534Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: added peer, starting replication: peer=server-ae651032-d12d-3749-0071-dc6b6cbe41b6 + testing.go:149: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.534Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: added peer, starting replication: peer=server-ae651032-d12d-3749-0071-dc6b6cbe41b6 +testLoggerAdapter verbose: server-ae651032-d12d-3749-0071-dc6b6cbe41b6: 2024-01-08T00:31:05.535Z [INFO] server-ae651032-d12d-3749-0071-dc6b6cbe41b6: entering follower state: follower="Node at ae651032-d12d-3749-0071-dc6b6cbe41b6 [Follower]" leader=9353476e-b9c8-84b8-dc54-c291e86420bf + testing.go:149: server-ae651032-d12d-3749-0071-dc6b6cbe41b6: 2024-01-08T00:31:05.535Z [INFO] server-ae651032-d12d-3749-0071-dc6b6cbe41b6: entering follower state: follower="Node at ae651032-d12d-3749-0071-dc6b6cbe41b6 [Follower]" leader=9353476e-b9c8-84b8-dc54-c291e86420bf +testLoggerAdapter verbose: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.535Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: pipelining replication: peer="{Voter server-ae651032-d12d-3749-0071-dc6b6cbe41b6 ae651032-d12d-3749-0071-dc6b6cbe41b6}" + testing.go:149: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.535Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: pipelining replication: peer="{Voter server-ae651032-d12d-3749-0071-dc6b6cbe41b6 ae651032-d12d-3749-0071-dc6b6cbe41b6}" +testLoggerAdapter verbose: server-237d4f9a-e353-e960-cae3-f2ef9d843b0f: 2024-01-08T00:31:05.535Z [INFO] server-237d4f9a-e353-e960-cae3-f2ef9d843b0f: duplicate requestVote for same term: term=2 + testing.go:149: server-237d4f9a-e353-e960-cae3-f2ef9d843b0f: 2024-01-08T00:31:05.535Z [INFO] server-237d4f9a-e353-e960-cae3-f2ef9d843b0f: duplicate requestVote for same term: term=2 +testLoggerAdapter verbose: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.535Z [WARN] server-9353476e-b9c8-84b8-dc54-c291e86420bf: rejecting vote request since we have a leader: from=ae651032-d12d-3749-0071-dc6b6cbe41b6 leader=9353476e-b9c8-84b8-dc54-c291e86420bf + testing.go:149: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.535Z [WARN] server-9353476e-b9c8-84b8-dc54-c291e86420bf: rejecting vote request since we have a leader: from=ae651032-d12d-3749-0071-dc6b6cbe41b6 leader=9353476e-b9c8-84b8-dc54-c291e86420bf +testLoggerAdapter verbose: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.535Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: pipelining replication: peer="{Voter server-237d4f9a-e353-e960-cae3-f2ef9d843b0f 237d4f9a-e353-e960-cae3-f2ef9d843b0f}" + testing.go:149: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.535Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: pipelining replication: peer="{Voter server-237d4f9a-e353-e960-cae3-f2ef9d843b0f 237d4f9a-e353-e960-cae3-f2ef9d843b0f}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:05.640Z [INFO] stable state for Leader reached at 2024-01-08 00:31:05.535218823 +0000 UTC m=+11.582107894 (1 nodes), 58.141283ms from start of poll, 58.746131ms from cluster start. Timeout at 2024-01-08 00:31:05.640702468 +0000 UTC m=+11.687591539, 105.483645ms after stability + testing.go:149: cluster: 2024-01-08T00:31:05.640Z [INFO] stable state for Leader reached at 2024-01-08 00:31:05.535218823 +0000 UTC m=+11.582107894 (1 nodes), 58.141283ms from start of poll, 58.746131ms from cluster start. Timeout at 2024-01-08 00:31:05.640702468 +0000 UTC m=+11.687591539, 105.483645ms after stability +testLoggerAdapter verbose: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.641Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: starting snapshot up to: index=12 + testing.go:149: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.641Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: starting snapshot up to: index=12 + testing.go:153: 2024-01-08T00:31:05.641Z [INFO] creating new snapshot: path=/tmp/raft511083705/snapshots/2-12-1704673865641.tmp +testLoggerAdapter verbose: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.650Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: snapshot complete up to: index=12 + testing.go:149: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.650Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: snapshot complete up to: index=12 + testing.go:153: 2024-01-08T00:31:05.650Z [INFO] creating new snapshot: path=/tmp/raft511083705/snapshots/2-23-1704673865650.tmp +testLoggerAdapter verbose: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.662Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: copied to local snapshot: bytes=71 + testing.go:149: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.662Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.663Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: restored user snapshot: index=1 + testing.go:149: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.663Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: restored user snapshot: index=1 +testLoggerAdapter verbose: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.663Z [ERROR] server-9353476e-b9c8-84b8-dc54-c291e86420bf: failed to get log: index=23 error="log not found" + testing.go:149: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.663Z [ERROR] server-9353476e-b9c8-84b8-dc54-c291e86420bf: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.663Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: aborting pipeline replication: peer="{Voter server-237d4f9a-e353-e960-cae3-f2ef9d843b0f 237d4f9a-e353-e960-cae3-f2ef9d843b0f}" + testing.go:149: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.663Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: aborting pipeline replication: peer="{Voter server-237d4f9a-e353-e960-cae3-f2ef9d843b0f 237d4f9a-e353-e960-cae3-f2ef9d843b0f}" +testLoggerAdapter verbose: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.663Z [ERROR] server-9353476e-b9c8-84b8-dc54-c291e86420bf: failed to get log: index=23 error="log not found" + testing.go:149: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.663Z [ERROR] server-9353476e-b9c8-84b8-dc54-c291e86420bf: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.663Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: aborting pipeline replication: peer="{Voter server-ae651032-d12d-3749-0071-dc6b6cbe41b6 ae651032-d12d-3749-0071-dc6b6cbe41b6}" + testing.go:149: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.663Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: aborting pipeline replication: peer="{Voter server-ae651032-d12d-3749-0071-dc6b6cbe41b6 ae651032-d12d-3749-0071-dc6b6cbe41b6}" +testLoggerAdapter verbose: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.669Z [ERROR] server-9353476e-b9c8-84b8-dc54-c291e86420bf: failed to get log: index=23 error="log not found" + testing.go:149: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.669Z [ERROR] server-9353476e-b9c8-84b8-dc54-c291e86420bf: failed to get log: index=23 error="log not found" + testing.go:153: 2024-01-08T00:31:05.669Z [INFO] creating new snapshot: path=/tmp/raft3518562365/snapshots/2-23-1704673865669.tmp +testLoggerAdapter verbose: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.672Z [ERROR] server-9353476e-b9c8-84b8-dc54-c291e86420bf: failed to get log: index=23 error="log not found" + testing.go:149: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.672Z [ERROR] server-9353476e-b9c8-84b8-dc54-c291e86420bf: failed to get log: index=23 error="log not found" + testing.go:153: 2024-01-08T00:31:05.673Z [INFO] creating new snapshot: path=/tmp/raft1169431483/snapshots/2-23-1704673865673.tmp +testLoggerAdapter verbose: server-237d4f9a-e353-e960-cae3-f2ef9d843b0f: 2024-01-08T00:31:05.680Z [INFO] server-237d4f9a-e353-e960-cae3-f2ef9d843b0f: copied to local snapshot: bytes=71 + testing.go:149: server-237d4f9a-e353-e960-cae3-f2ef9d843b0f: 2024-01-08T00:31:05.680Z [INFO] server-237d4f9a-e353-e960-cae3-f2ef9d843b0f: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-237d4f9a-e353-e960-cae3-f2ef9d843b0f: 2024-01-08T00:31:05.681Z [INFO] server-237d4f9a-e353-e960-cae3-f2ef9d843b0f: Installed remote snapshot + testing.go:149: server-237d4f9a-e353-e960-cae3-f2ef9d843b0f: 2024-01-08T00:31:05.681Z [INFO] server-237d4f9a-e353-e960-cae3-f2ef9d843b0f: Installed remote snapshot +testLoggerAdapter verbose: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.681Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: pipelining replication: peer="{Voter server-237d4f9a-e353-e960-cae3-f2ef9d843b0f 237d4f9a-e353-e960-cae3-f2ef9d843b0f}" + testing.go:149: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.681Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: pipelining replication: peer="{Voter server-237d4f9a-e353-e960-cae3-f2ef9d843b0f 237d4f9a-e353-e960-cae3-f2ef9d843b0f}" +testLoggerAdapter verbose: server-ae651032-d12d-3749-0071-dc6b6cbe41b6: 2024-01-08T00:31:05.683Z [INFO] server-ae651032-d12d-3749-0071-dc6b6cbe41b6: copied to local snapshot: bytes=71 + testing.go:149: server-ae651032-d12d-3749-0071-dc6b6cbe41b6: 2024-01-08T00:31:05.683Z [INFO] server-ae651032-d12d-3749-0071-dc6b6cbe41b6: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-ae651032-d12d-3749-0071-dc6b6cbe41b6: 2024-01-08T00:31:05.683Z [INFO] server-ae651032-d12d-3749-0071-dc6b6cbe41b6: Installed remote snapshot + testing.go:149: server-ae651032-d12d-3749-0071-dc6b6cbe41b6: 2024-01-08T00:31:05.683Z [INFO] server-ae651032-d12d-3749-0071-dc6b6cbe41b6: Installed remote snapshot +testLoggerAdapter verbose: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.683Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: pipelining replication: peer="{Voter server-ae651032-d12d-3749-0071-dc6b6cbe41b6 ae651032-d12d-3749-0071-dc6b6cbe41b6}" + testing.go:149: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.683Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: pipelining replication: peer="{Voter server-ae651032-d12d-3749-0071-dc6b6cbe41b6 ae651032-d12d-3749-0071-dc6b6cbe41b6}" +testLoggerAdapter verbose: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.697Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: aborting pipeline replication: peer="{Voter server-ae651032-d12d-3749-0071-dc6b6cbe41b6 ae651032-d12d-3749-0071-dc6b6cbe41b6}" + testing.go:149: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.697Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: aborting pipeline replication: peer="{Voter server-ae651032-d12d-3749-0071-dc6b6cbe41b6 ae651032-d12d-3749-0071-dc6b6cbe41b6}" +testLoggerAdapter verbose: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.697Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: aborting pipeline replication: peer="{Voter server-237d4f9a-e353-e960-cae3-f2ef9d843b0f 237d4f9a-e353-e960-cae3-f2ef9d843b0f}" + testing.go:149: server-9353476e-b9c8-84b8-dc54-c291e86420bf: 2024-01-08T00:31:05.697Z [INFO] server-9353476e-b9c8-84b8-dc54-c291e86420bf: aborting pipeline replication: peer="{Voter server-237d4f9a-e353-e960-cae3-f2ef9d843b0f 237d4f9a-e353-e960-cae3-f2ef9d843b0f}" +testLoggerAdapter verbose: server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a: 2024-01-08T00:31:05.699Z [INFO] server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a Address:42eaf54d-e01f-ab9a-3600-082b2aa9f99a} {Suffrage:Voter ID:server-8dbbcae6-dc88-aab9-703d-92afe3b4448c Address:8dbbcae6-dc88-aab9-703d-92afe3b4448c} {Suffrage:Voter ID:server-0b54f66c-220a-d219-88a5-afa3e4d794ab Address:0b54f66c-220a-d219-88a5-afa3e4d794ab}]" + testing.go:149: server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a: 2024-01-08T00:31:05.699Z [INFO] server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a Address:42eaf54d-e01f-ab9a-3600-082b2aa9f99a} {Suffrage:Voter ID:server-8dbbcae6-dc88-aab9-703d-92afe3b4448c Address:8dbbcae6-dc88-aab9-703d-92afe3b4448c} {Suffrage:Voter ID:server-0b54f66c-220a-d219-88a5-afa3e4d794ab Address:0b54f66c-220a-d219-88a5-afa3e4d794ab}]" +testLoggerAdapter verbose: server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: 2024-01-08T00:31:05.699Z [INFO] server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a Address:42eaf54d-e01f-ab9a-3600-082b2aa9f99a} {Suffrage:Voter ID:server-8dbbcae6-dc88-aab9-703d-92afe3b4448c Address:8dbbcae6-dc88-aab9-703d-92afe3b4448c} {Suffrage:Voter ID:server-0b54f66c-220a-d219-88a5-afa3e4d794ab Address:0b54f66c-220a-d219-88a5-afa3e4d794ab}]" + testing.go:149: server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: 2024-01-08T00:31:05.699Z [INFO] server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a Address:42eaf54d-e01f-ab9a-3600-082b2aa9f99a} {Suffrage:Voter ID:server-8dbbcae6-dc88-aab9-703d-92afe3b4448c Address:8dbbcae6-dc88-aab9-703d-92afe3b4448c} {Suffrage:Voter ID:server-0b54f66c-220a-d219-88a5-afa3e4d794ab Address:0b54f66c-220a-d219-88a5-afa3e4d794ab}]" +testLoggerAdapter verbose: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.699Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a Address:42eaf54d-e01f-ab9a-3600-082b2aa9f99a} {Suffrage:Voter ID:server-8dbbcae6-dc88-aab9-703d-92afe3b4448c Address:8dbbcae6-dc88-aab9-703d-92afe3b4448c} {Suffrage:Voter ID:server-0b54f66c-220a-d219-88a5-afa3e4d794ab Address:0b54f66c-220a-d219-88a5-afa3e4d794ab}]" + testing.go:149: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.699Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a Address:42eaf54d-e01f-ab9a-3600-082b2aa9f99a} {Suffrage:Voter ID:server-8dbbcae6-dc88-aab9-703d-92afe3b4448c Address:8dbbcae6-dc88-aab9-703d-92afe3b4448c} {Suffrage:Voter ID:server-0b54f66c-220a-d219-88a5-afa3e4d794ab Address:0b54f66c-220a-d219-88a5-afa3e4d794ab}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:05.699Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:05.699Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a: 2024-01-08T00:31:05.699Z [INFO] server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a: entering follower state: follower="Node at 42eaf54d-e01f-ab9a-3600-082b2aa9f99a [Follower]" leader= + testing.go:149: server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a: 2024-01-08T00:31:05.699Z [INFO] server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a: entering follower state: follower="Node at 42eaf54d-e01f-ab9a-3600-082b2aa9f99a [Follower]" leader= +testLoggerAdapter verbose: server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: 2024-01-08T00:31:05.699Z [INFO] server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: entering follower state: follower="Node at 8dbbcae6-dc88-aab9-703d-92afe3b4448c [Follower]" leader= + testing.go:149: server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: 2024-01-08T00:31:05.699Z [INFO] server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: entering follower state: follower="Node at 8dbbcae6-dc88-aab9-703d-92afe3b4448c [Follower]" leader= +testLoggerAdapter verbose: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.699Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: entering follower state: follower="Node at 0b54f66c-220a-d219-88a5-afa3e4d794ab [Follower]" leader= + testing.go:149: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.699Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: entering follower state: follower="Node at 0b54f66c-220a-d219-88a5-afa3e4d794ab [Follower]" leader= +testLoggerAdapter verbose: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.753Z [WARN] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.753Z [WARN] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.754Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: entering candidate state: node="Node at 0b54f66c-220a-d219-88a5-afa3e4d794ab [Candidate]" term=2 + testing.go:149: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.754Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: entering candidate state: node="Node at 0b54f66c-220a-d219-88a5-afa3e4d794ab [Candidate]" term=2 +testLoggerAdapter verbose: server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: 2024-01-08T00:31:05.754Z [WARN] server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: 2024-01-08T00:31:05.754Z [WARN] server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: 2024-01-08T00:31:05.754Z [INFO] server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: entering candidate state: node="Node at 8dbbcae6-dc88-aab9-703d-92afe3b4448c [Candidate]" term=2 + testing.go:149: server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: 2024-01-08T00:31:05.754Z [INFO] server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: entering candidate state: node="Node at 8dbbcae6-dc88-aab9-703d-92afe3b4448c [Candidate]" term=2 +testLoggerAdapter verbose: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.754Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: election won: tally=2 +testLoggerAdapter verbose: server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: 2024-01-08T00:31:05.754Z [INFO] server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: duplicate requestVote for same term: term=2 + testing.go:149: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.754Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: election won: tally=2 +testLoggerAdapter verbose: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.754Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: entering leader state: leader="Node at 0b54f66c-220a-d219-88a5-afa3e4d794ab [Leader]" +testLoggerAdapter verbose: server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a: 2024-01-08T00:31:05.754Z [INFO] server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a: duplicate requestVote for same term: term=2 + testing.go:149: server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: 2024-01-08T00:31:05.754Z [INFO] server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: duplicate requestVote for same term: term=2 + testing.go:149: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.754Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: entering leader state: leader="Node at 0b54f66c-220a-d219-88a5-afa3e4d794ab [Leader]" +testLoggerAdapter verbose: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.754Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: added peer, starting replication: peer=server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a + testing.go:149: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.754Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: added peer, starting replication: peer=server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a +testLoggerAdapter verbose: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.754Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: added peer, starting replication: peer=server-8dbbcae6-dc88-aab9-703d-92afe3b4448c + testing.go:149: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.754Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: added peer, starting replication: peer=server-8dbbcae6-dc88-aab9-703d-92afe3b4448c +testLoggerAdapter verbose: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.754Z [WARN] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: rejecting vote request since we have a leader: from=8dbbcae6-dc88-aab9-703d-92afe3b4448c leader=0b54f66c-220a-d219-88a5-afa3e4d794ab + testing.go:149: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.754Z [WARN] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: rejecting vote request since we have a leader: from=8dbbcae6-dc88-aab9-703d-92afe3b4448c leader=0b54f66c-220a-d219-88a5-afa3e4d794ab + testing.go:149: server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a: 2024-01-08T00:31:05.754Z [INFO] server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a: duplicate requestVote for same term: term=2 +testLoggerAdapter verbose: server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: 2024-01-08T00:31:05.754Z [INFO] server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: entering follower state: follower="Node at 8dbbcae6-dc88-aab9-703d-92afe3b4448c [Follower]" leader=0b54f66c-220a-d219-88a5-afa3e4d794ab + testing.go:149: server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: 2024-01-08T00:31:05.754Z [INFO] server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: entering follower state: follower="Node at 8dbbcae6-dc88-aab9-703d-92afe3b4448c [Follower]" leader=0b54f66c-220a-d219-88a5-afa3e4d794ab +testLoggerAdapter verbose: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.754Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: pipelining replication: peer="{Voter server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a 42eaf54d-e01f-ab9a-3600-082b2aa9f99a}" + testing.go:149: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.754Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: pipelining replication: peer="{Voter server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a 42eaf54d-e01f-ab9a-3600-082b2aa9f99a}" +testLoggerAdapter verbose: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.754Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: pipelining replication: peer="{Voter server-8dbbcae6-dc88-aab9-703d-92afe3b4448c 8dbbcae6-dc88-aab9-703d-92afe3b4448c}" + testing.go:149: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.754Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: pipelining replication: peer="{Voter server-8dbbcae6-dc88-aab9-703d-92afe3b4448c 8dbbcae6-dc88-aab9-703d-92afe3b4448c}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:05.860Z [INFO] stable state for Leader reached at 2024-01-08 00:31:05.754598958 +0000 UTC m=+11.801488019 (1 nodes), 55.116941ms from start of poll, 55.538697ms from cluster start. Timeout at 2024-01-08 00:31:05.860045736 +0000 UTC m=+11.906934818, 105.446799ms after stability + testing.go:149: cluster: 2024-01-08T00:31:05.860Z [INFO] stable state for Leader reached at 2024-01-08 00:31:05.754598958 +0000 UTC m=+11.801488019 (1 nodes), 55.116941ms from start of poll, 55.538697ms from cluster start. Timeout at 2024-01-08 00:31:05.860045736 +0000 UTC m=+11.906934818, 105.446799ms after stability +testLoggerAdapter verbose: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.860Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: starting snapshot up to: index=12 + testing.go:149: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.860Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: starting snapshot up to: index=12 + testing.go:153: 2024-01-08T00:31:05.860Z [INFO] creating new snapshot: path=/tmp/raft1611914947/snapshots/2-12-1704673865860.tmp +testLoggerAdapter verbose: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.869Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: snapshot complete up to: index=12 + testing.go:149: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.869Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: snapshot complete up to: index=12 + testing.go:153: 2024-01-08T00:31:05.869Z [INFO] creating new snapshot: path=/tmp/raft1611914947/snapshots/2-113-1704673865869.tmp +testLoggerAdapter verbose: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.878Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: copied to local snapshot: bytes=71 + testing.go:149: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.878Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.878Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: restored user snapshot: index=1 + testing.go:149: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.878Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: restored user snapshot: index=1 +testLoggerAdapter verbose: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.878Z [ERROR] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: failed to get log: index=23 error="log not found" + testing.go:149: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.878Z [ERROR] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.878Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: aborting pipeline replication: peer="{Voter server-8dbbcae6-dc88-aab9-703d-92afe3b4448c 8dbbcae6-dc88-aab9-703d-92afe3b4448c}" + testing.go:149: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.878Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: aborting pipeline replication: peer="{Voter server-8dbbcae6-dc88-aab9-703d-92afe3b4448c 8dbbcae6-dc88-aab9-703d-92afe3b4448c}" +testLoggerAdapter verbose: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.878Z [ERROR] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: failed to get log: index=23 error="log not found" + testing.go:149: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.878Z [ERROR] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.878Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: aborting pipeline replication: peer="{Voter server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a 42eaf54d-e01f-ab9a-3600-082b2aa9f99a}" + testing.go:149: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.878Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: aborting pipeline replication: peer="{Voter server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a 42eaf54d-e01f-ab9a-3600-082b2aa9f99a}" +testLoggerAdapter verbose: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.884Z [ERROR] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: failed to get log: index=23 error="log not found" + testing.go:149: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.884Z [ERROR] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: failed to get log: index=23 error="log not found" + testing.go:153: 2024-01-08T00:31:05.885Z [INFO] creating new snapshot: path=/tmp/raft3305911943/snapshots/2-113-1704673865885.tmp +testLoggerAdapter verbose: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.886Z [ERROR] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: failed to get log: index=23 error="log not found" + testing.go:149: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.886Z [ERROR] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: failed to get log: index=23 error="log not found" + testing.go:153: 2024-01-08T00:31:05.887Z [INFO] creating new snapshot: path=/tmp/raft2474434758/snapshots/2-113-1704673865887.tmp +testLoggerAdapter verbose: server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a: 2024-01-08T00:31:05.896Z [INFO] server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a: copied to local snapshot: bytes=71 + testing.go:149: server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a: 2024-01-08T00:31:05.896Z [INFO] server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a: 2024-01-08T00:31:05.896Z [INFO] server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a: Installed remote snapshot + testing.go:149: server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a: 2024-01-08T00:31:05.896Z [INFO] server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a: Installed remote snapshot +testLoggerAdapter verbose: server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a: 2024-01-08T00:31:05.896Z [WARN] server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a: skipping application of old log: index=113 + testing.go:149: server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a: 2024-01-08T00:31:05.896Z [WARN] server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a: skipping application of old log: index=113 +testLoggerAdapter verbose: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.897Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: pipelining replication: peer="{Voter server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a 42eaf54d-e01f-ab9a-3600-082b2aa9f99a}" + testing.go:149: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.897Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: pipelining replication: peer="{Voter server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a 42eaf54d-e01f-ab9a-3600-082b2aa9f99a}" +testLoggerAdapter verbose: server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: 2024-01-08T00:31:05.896Z [INFO] server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: copied to local snapshot: bytes=71 + testing.go:149: server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: 2024-01-08T00:31:05.896Z [INFO] server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: 2024-01-08T00:31:05.897Z [INFO] server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: Installed remote snapshot + testing.go:149: server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: 2024-01-08T00:31:05.897Z [INFO] server-8dbbcae6-dc88-aab9-703d-92afe3b4448c: Installed remote snapshot +testLoggerAdapter verbose: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.897Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: pipelining replication: peer="{Voter server-8dbbcae6-dc88-aab9-703d-92afe3b4448c 8dbbcae6-dc88-aab9-703d-92afe3b4448c}" + testing.go:149: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.897Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: pipelining replication: peer="{Voter server-8dbbcae6-dc88-aab9-703d-92afe3b4448c 8dbbcae6-dc88-aab9-703d-92afe3b4448c}" +testLoggerAdapter verbose: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.909Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: aborting pipeline replication: peer="{Voter server-8dbbcae6-dc88-aab9-703d-92afe3b4448c 8dbbcae6-dc88-aab9-703d-92afe3b4448c}" + testing.go:149: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.909Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: aborting pipeline replication: peer="{Voter server-8dbbcae6-dc88-aab9-703d-92afe3b4448c 8dbbcae6-dc88-aab9-703d-92afe3b4448c}" +testLoggerAdapter verbose: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.909Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: aborting pipeline replication: peer="{Voter server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a 42eaf54d-e01f-ab9a-3600-082b2aa9f99a}" + testing.go:149: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.909Z [INFO] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: aborting pipeline replication: peer="{Voter server-42eaf54d-e01f-ab9a-3600-082b2aa9f99a 42eaf54d-e01f-ab9a-3600-082b2aa9f99a}" +testLoggerAdapter verbose: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.960Z [ERROR] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: failed to heartbeat to: peer=42eaf54d-e01f-ab9a-3600-082b2aa9f99a error="command timed out" + testing.go:149: server-0b54f66c-220a-d219-88a5-afa3e4d794ab: 2024-01-08T00:31:05.960Z [ERROR] server-0b54f66c-220a-d219-88a5-afa3e4d794ab: failed to heartbeat to: peer=42eaf54d-e01f-ab9a-3600-082b2aa9f99a error="command timed out" +testLoggerAdapter verbose: server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84: 2024-01-08T00:31:05.962Z [INFO] server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84 Address:3515b721-0e5e-91ac-2fa2-49f75a5b9a84} {Suffrage:Voter ID:server-240e00b6-e276-bafc-8be2-7e927c3b9b9c Address:240e00b6-e276-bafc-8be2-7e927c3b9b9c} {Suffrage:Voter ID:server-d904b220-5b5c-c279-16b3-99f3c5035f9a Address:d904b220-5b5c-c279-16b3-99f3c5035f9a}]" + testing.go:149: server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84: 2024-01-08T00:31:05.962Z [INFO] server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84 Address:3515b721-0e5e-91ac-2fa2-49f75a5b9a84} {Suffrage:Voter ID:server-240e00b6-e276-bafc-8be2-7e927c3b9b9c Address:240e00b6-e276-bafc-8be2-7e927c3b9b9c} {Suffrage:Voter ID:server-d904b220-5b5c-c279-16b3-99f3c5035f9a Address:d904b220-5b5c-c279-16b3-99f3c5035f9a}]" +testLoggerAdapter verbose: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:05.962Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84 Address:3515b721-0e5e-91ac-2fa2-49f75a5b9a84} {Suffrage:Voter ID:server-240e00b6-e276-bafc-8be2-7e927c3b9b9c Address:240e00b6-e276-bafc-8be2-7e927c3b9b9c} {Suffrage:Voter ID:server-d904b220-5b5c-c279-16b3-99f3c5035f9a Address:d904b220-5b5c-c279-16b3-99f3c5035f9a}]" + testing.go:149: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:05.962Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84 Address:3515b721-0e5e-91ac-2fa2-49f75a5b9a84} {Suffrage:Voter ID:server-240e00b6-e276-bafc-8be2-7e927c3b9b9c Address:240e00b6-e276-bafc-8be2-7e927c3b9b9c} {Suffrage:Voter ID:server-d904b220-5b5c-c279-16b3-99f3c5035f9a Address:d904b220-5b5c-c279-16b3-99f3c5035f9a}]" +testLoggerAdapter verbose: server-d904b220-5b5c-c279-16b3-99f3c5035f9a: 2024-01-08T00:31:05.962Z [INFO] server-d904b220-5b5c-c279-16b3-99f3c5035f9a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84 Address:3515b721-0e5e-91ac-2fa2-49f75a5b9a84} {Suffrage:Voter ID:server-240e00b6-e276-bafc-8be2-7e927c3b9b9c Address:240e00b6-e276-bafc-8be2-7e927c3b9b9c} {Suffrage:Voter ID:server-d904b220-5b5c-c279-16b3-99f3c5035f9a Address:d904b220-5b5c-c279-16b3-99f3c5035f9a}]" + testing.go:149: server-d904b220-5b5c-c279-16b3-99f3c5035f9a: 2024-01-08T00:31:05.962Z [INFO] server-d904b220-5b5c-c279-16b3-99f3c5035f9a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84 Address:3515b721-0e5e-91ac-2fa2-49f75a5b9a84} {Suffrage:Voter ID:server-240e00b6-e276-bafc-8be2-7e927c3b9b9c Address:240e00b6-e276-bafc-8be2-7e927c3b9b9c} {Suffrage:Voter ID:server-d904b220-5b5c-c279-16b3-99f3c5035f9a Address:d904b220-5b5c-c279-16b3-99f3c5035f9a}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:05.962Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:05.962Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84: 2024-01-08T00:31:05.962Z [INFO] server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84: entering follower state: follower="Node at 3515b721-0e5e-91ac-2fa2-49f75a5b9a84 [Follower]" leader= + testing.go:149: server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84: 2024-01-08T00:31:05.962Z [INFO] server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84: entering follower state: follower="Node at 3515b721-0e5e-91ac-2fa2-49f75a5b9a84 [Follower]" leader= +testLoggerAdapter verbose: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:05.962Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: entering follower state: follower="Node at 240e00b6-e276-bafc-8be2-7e927c3b9b9c [Follower]" leader= + testing.go:149: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:05.962Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: entering follower state: follower="Node at 240e00b6-e276-bafc-8be2-7e927c3b9b9c [Follower]" leader= +testLoggerAdapter verbose: server-d904b220-5b5c-c279-16b3-99f3c5035f9a: 2024-01-08T00:31:05.962Z [INFO] server-d904b220-5b5c-c279-16b3-99f3c5035f9a: entering follower state: follower="Node at d904b220-5b5c-c279-16b3-99f3c5035f9a [Follower]" leader= + testing.go:149: server-d904b220-5b5c-c279-16b3-99f3c5035f9a: 2024-01-08T00:31:05.962Z [INFO] server-d904b220-5b5c-c279-16b3-99f3c5035f9a: entering follower state: follower="Node at d904b220-5b5c-c279-16b3-99f3c5035f9a [Follower]" leader= +testLoggerAdapter verbose: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.028Z [WARN] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.028Z [WARN] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.028Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: entering candidate state: node="Node at 240e00b6-e276-bafc-8be2-7e927c3b9b9c [Candidate]" term=2 + testing.go:149: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.028Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: entering candidate state: node="Node at 240e00b6-e276-bafc-8be2-7e927c3b9b9c [Candidate]" term=2 +testLoggerAdapter verbose: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.028Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: election won: tally=2 + testing.go:149: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.028Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: election won: tally=2 +testLoggerAdapter verbose: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.028Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: entering leader state: leader="Node at 240e00b6-e276-bafc-8be2-7e927c3b9b9c [Leader]" + testing.go:149: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.028Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: entering leader state: leader="Node at 240e00b6-e276-bafc-8be2-7e927c3b9b9c [Leader]" +testLoggerAdapter verbose: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.028Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: added peer, starting replication: peer=server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84 + testing.go:149: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.028Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: added peer, starting replication: peer=server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84 +testLoggerAdapter verbose: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.028Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: added peer, starting replication: peer=server-d904b220-5b5c-c279-16b3-99f3c5035f9a + testing.go:149: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.028Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: added peer, starting replication: peer=server-d904b220-5b5c-c279-16b3-99f3c5035f9a +testLoggerAdapter verbose: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.028Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: pipelining replication: peer="{Voter server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84 3515b721-0e5e-91ac-2fa2-49f75a5b9a84}" + testing.go:149: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.028Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: pipelining replication: peer="{Voter server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84 3515b721-0e5e-91ac-2fa2-49f75a5b9a84}" +testLoggerAdapter verbose: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.028Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: pipelining replication: peer="{Voter server-d904b220-5b5c-c279-16b3-99f3c5035f9a d904b220-5b5c-c279-16b3-99f3c5035f9a}" + testing.go:149: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.028Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: pipelining replication: peer="{Voter server-d904b220-5b5c-c279-16b3-99f3c5035f9a d904b220-5b5c-c279-16b3-99f3c5035f9a}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:06.134Z [INFO] stable state for Leader reached at 2024-01-08 00:31:06.028513699 +0000 UTC m=+12.075402840 (1 nodes), 65.696728ms from start of poll, 66.135668ms from cluster start. Timeout at 2024-01-08 00:31:06.134408755 +0000 UTC m=+12.181297886, 105.895046ms after stability + testing.go:149: cluster: 2024-01-08T00:31:06.134Z [INFO] stable state for Leader reached at 2024-01-08 00:31:06.028513699 +0000 UTC m=+12.075402840 (1 nodes), 65.696728ms from start of poll, 66.135668ms from cluster start. Timeout at 2024-01-08 00:31:06.134408755 +0000 UTC m=+12.181297886, 105.895046ms after stability +testLoggerAdapter verbose: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.134Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: starting snapshot up to: index=12 + testing.go:149: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.134Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: starting snapshot up to: index=12 + testing.go:153: 2024-01-08T00:31:06.135Z [INFO] creating new snapshot: path=/tmp/raft351508388/snapshots/2-12-1704673866135.tmp +testLoggerAdapter verbose: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.142Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: snapshot complete up to: index=12 + testing.go:149: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.142Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: snapshot complete up to: index=12 + testing.go:153: 2024-01-08T00:31:06.143Z [INFO] creating new snapshot: path=/tmp/raft351508388/snapshots/2-1013-1704673866143.tmp +testLoggerAdapter verbose: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.150Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: copied to local snapshot: bytes=71 + testing.go:149: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.150Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.151Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: restored user snapshot: index=1 + testing.go:149: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.151Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: restored user snapshot: index=1 +testLoggerAdapter verbose: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.151Z [ERROR] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: failed to get log: index=23 error="log not found" + testing.go:149: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.151Z [ERROR] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.151Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: aborting pipeline replication: peer="{Voter server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84 3515b721-0e5e-91ac-2fa2-49f75a5b9a84}" + testing.go:149: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.151Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: aborting pipeline replication: peer="{Voter server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84 3515b721-0e5e-91ac-2fa2-49f75a5b9a84}" +testLoggerAdapter verbose: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.151Z [ERROR] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: failed to get log: index=23 error="log not found" + testing.go:149: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.151Z [ERROR] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.151Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: aborting pipeline replication: peer="{Voter server-d904b220-5b5c-c279-16b3-99f3c5035f9a d904b220-5b5c-c279-16b3-99f3c5035f9a}" + testing.go:149: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.151Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: aborting pipeline replication: peer="{Voter server-d904b220-5b5c-c279-16b3-99f3c5035f9a d904b220-5b5c-c279-16b3-99f3c5035f9a}" +testLoggerAdapter verbose: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.157Z [ERROR] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: failed to get log: index=23 error="log not found" + testing.go:149: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.157Z [ERROR] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: failed to get log: index=23 error="log not found" + testing.go:153: 2024-01-08T00:31:06.158Z [INFO] creating new snapshot: path=/tmp/raft3009075828/snapshots/2-1013-1704673866158.tmp +testLoggerAdapter verbose: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.161Z [ERROR] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: failed to get log: index=23 error="log not found" + testing.go:149: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.161Z [ERROR] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: failed to get log: index=23 error="log not found" + testing.go:153: 2024-01-08T00:31:06.161Z [INFO] creating new snapshot: path=/tmp/raft2767649673/snapshots/2-1013-1704673866161.tmp +testLoggerAdapter verbose: server-d904b220-5b5c-c279-16b3-99f3c5035f9a: 2024-01-08T00:31:06.167Z [INFO] server-d904b220-5b5c-c279-16b3-99f3c5035f9a: copied to local snapshot: bytes=71 + testing.go:149: server-d904b220-5b5c-c279-16b3-99f3c5035f9a: 2024-01-08T00:31:06.167Z [INFO] server-d904b220-5b5c-c279-16b3-99f3c5035f9a: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-d904b220-5b5c-c279-16b3-99f3c5035f9a: 2024-01-08T00:31:06.168Z [INFO] server-d904b220-5b5c-c279-16b3-99f3c5035f9a: Installed remote snapshot + testing.go:149: server-d904b220-5b5c-c279-16b3-99f3c5035f9a: 2024-01-08T00:31:06.168Z [INFO] server-d904b220-5b5c-c279-16b3-99f3c5035f9a: Installed remote snapshot +testLoggerAdapter verbose: server-d904b220-5b5c-c279-16b3-99f3c5035f9a: 2024-01-08T00:31:06.168Z [WARN] server-d904b220-5b5c-c279-16b3-99f3c5035f9a: skipping application of old log: index=22 + testing.go:149: server-d904b220-5b5c-c279-16b3-99f3c5035f9a: 2024-01-08T00:31:06.168Z [WARN] server-d904b220-5b5c-c279-16b3-99f3c5035f9a: skipping application of old log: index=22 +testLoggerAdapter verbose: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.168Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: pipelining replication: peer="{Voter server-d904b220-5b5c-c279-16b3-99f3c5035f9a d904b220-5b5c-c279-16b3-99f3c5035f9a}" + testing.go:149: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.168Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: pipelining replication: peer="{Voter server-d904b220-5b5c-c279-16b3-99f3c5035f9a d904b220-5b5c-c279-16b3-99f3c5035f9a}" +testLoggerAdapter verbose: server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84: 2024-01-08T00:31:06.170Z [INFO] server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84: copied to local snapshot: bytes=71 + testing.go:149: server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84: 2024-01-08T00:31:06.170Z [INFO] server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84: 2024-01-08T00:31:06.170Z [INFO] server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84: Installed remote snapshot + testing.go:149: server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84: 2024-01-08T00:31:06.170Z [INFO] server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84: Installed remote snapshot +testLoggerAdapter verbose: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.171Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: pipelining replication: peer="{Voter server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84 3515b721-0e5e-91ac-2fa2-49f75a5b9a84}" + testing.go:149: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.171Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: pipelining replication: peer="{Voter server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84 3515b721-0e5e-91ac-2fa2-49f75a5b9a84}" +testLoggerAdapter verbose: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.185Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: aborting pipeline replication: peer="{Voter server-d904b220-5b5c-c279-16b3-99f3c5035f9a d904b220-5b5c-c279-16b3-99f3c5035f9a}" + testing.go:149: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.185Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: aborting pipeline replication: peer="{Voter server-d904b220-5b5c-c279-16b3-99f3c5035f9a d904b220-5b5c-c279-16b3-99f3c5035f9a}" +testLoggerAdapter verbose: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.185Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: aborting pipeline replication: peer="{Voter server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84 3515b721-0e5e-91ac-2fa2-49f75a5b9a84}" + testing.go:149: server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: 2024-01-08T00:31:06.185Z [INFO] server-240e00b6-e276-bafc-8be2-7e927c3b9b9c: aborting pipeline replication: peer="{Voter server-3515b721-0e5e-91ac-2fa2-49f75a5b9a84 3515b721-0e5e-91ac-2fa2-49f75a5b9a84}" +testLoggerAdapter verbose: server-ea94504c-b9dd-9d53-9131-959998be4670: 2024-01-08T00:31:06.187Z [INFO] server-ea94504c-b9dd-9d53-9131-959998be4670: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ea94504c-b9dd-9d53-9131-959998be4670 Address:ea94504c-b9dd-9d53-9131-959998be4670} {Suffrage:Voter ID:server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0 Address:7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0} {Suffrage:Voter ID:server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0 Address:de0fcf05-17c2-a10a-c09c-a660e2b9f5f0}]" + testing.go:149: server-ea94504c-b9dd-9d53-9131-959998be4670: 2024-01-08T00:31:06.187Z [INFO] server-ea94504c-b9dd-9d53-9131-959998be4670: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ea94504c-b9dd-9d53-9131-959998be4670 Address:ea94504c-b9dd-9d53-9131-959998be4670} {Suffrage:Voter ID:server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0 Address:7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0} {Suffrage:Voter ID:server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0 Address:de0fcf05-17c2-a10a-c09c-a660e2b9f5f0}]" +testLoggerAdapter verbose: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.187Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ea94504c-b9dd-9d53-9131-959998be4670 Address:ea94504c-b9dd-9d53-9131-959998be4670} {Suffrage:Voter ID:server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0 Address:7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0} {Suffrage:Voter ID:server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0 Address:de0fcf05-17c2-a10a-c09c-a660e2b9f5f0}]" + testing.go:149: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.187Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ea94504c-b9dd-9d53-9131-959998be4670 Address:ea94504c-b9dd-9d53-9131-959998be4670} {Suffrage:Voter ID:server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0 Address:7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0} {Suffrage:Voter ID:server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0 Address:de0fcf05-17c2-a10a-c09c-a660e2b9f5f0}]" +testLoggerAdapter verbose: server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0: 2024-01-08T00:31:06.187Z [INFO] server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ea94504c-b9dd-9d53-9131-959998be4670 Address:ea94504c-b9dd-9d53-9131-959998be4670} {Suffrage:Voter ID:server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0 Address:7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0} {Suffrage:Voter ID:server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0 Address:de0fcf05-17c2-a10a-c09c-a660e2b9f5f0}]" + testing.go:149: server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0: 2024-01-08T00:31:06.187Z [INFO] server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ea94504c-b9dd-9d53-9131-959998be4670 Address:ea94504c-b9dd-9d53-9131-959998be4670} {Suffrage:Voter ID:server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0 Address:7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0} {Suffrage:Voter ID:server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0 Address:de0fcf05-17c2-a10a-c09c-a660e2b9f5f0}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:06.187Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:06.187Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-ea94504c-b9dd-9d53-9131-959998be4670: 2024-01-08T00:31:06.187Z [INFO] server-ea94504c-b9dd-9d53-9131-959998be4670: entering follower state: follower="Node at ea94504c-b9dd-9d53-9131-959998be4670 [Follower]" leader= + testing.go:149: server-ea94504c-b9dd-9d53-9131-959998be4670: 2024-01-08T00:31:06.187Z [INFO] server-ea94504c-b9dd-9d53-9131-959998be4670: entering follower state: follower="Node at ea94504c-b9dd-9d53-9131-959998be4670 [Follower]" leader= +testLoggerAdapter verbose: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.187Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: entering follower state: follower="Node at 7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0 [Follower]" leader= + testing.go:149: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.187Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: entering follower state: follower="Node at 7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0 [Follower]" leader= +testLoggerAdapter verbose: server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0: 2024-01-08T00:31:06.187Z [INFO] server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0: entering follower state: follower="Node at de0fcf05-17c2-a10a-c09c-a660e2b9f5f0 [Follower]" leader= + testing.go:149: server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0: 2024-01-08T00:31:06.187Z [INFO] server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0: entering follower state: follower="Node at de0fcf05-17c2-a10a-c09c-a660e2b9f5f0 [Follower]" leader= +testLoggerAdapter verbose: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.244Z [WARN] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.244Z [WARN] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.244Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: entering candidate state: node="Node at 7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0 [Candidate]" term=2 + testing.go:149: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.244Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: entering candidate state: node="Node at 7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0 [Candidate]" term=2 +testLoggerAdapter verbose: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.244Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: election won: tally=2 + testing.go:149: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.244Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: election won: tally=2 +testLoggerAdapter verbose: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.245Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: entering leader state: leader="Node at 7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0 [Leader]" + testing.go:149: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.245Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: entering leader state: leader="Node at 7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0 [Leader]" +testLoggerAdapter verbose: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.245Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: added peer, starting replication: peer=server-ea94504c-b9dd-9d53-9131-959998be4670 + testing.go:149: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.245Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: added peer, starting replication: peer=server-ea94504c-b9dd-9d53-9131-959998be4670 +testLoggerAdapter verbose: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.245Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: added peer, starting replication: peer=server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0 + testing.go:149: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.245Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: added peer, starting replication: peer=server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0 +testLoggerAdapter verbose: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.245Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: pipelining replication: peer="{Voter server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0 de0fcf05-17c2-a10a-c09c-a660e2b9f5f0}" + testing.go:149: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.245Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: pipelining replication: peer="{Voter server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0 de0fcf05-17c2-a10a-c09c-a660e2b9f5f0}" +testLoggerAdapter verbose: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.245Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: pipelining replication: peer="{Voter server-ea94504c-b9dd-9d53-9131-959998be4670 ea94504c-b9dd-9d53-9131-959998be4670}" + testing.go:149: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.245Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: pipelining replication: peer="{Voter server-ea94504c-b9dd-9d53-9131-959998be4670 ea94504c-b9dd-9d53-9131-959998be4670}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:06.351Z [INFO] stable state for Leader reached at 2024-01-08 00:31:06.24541993 +0000 UTC m=+12.292308973 (1 nodes), 57.713897ms from start of poll, 58.136438ms from cluster start. Timeout at 2024-01-08 00:31:06.351375156 +0000 UTC m=+12.398264218, 105.955245ms after stability + testing.go:149: cluster: 2024-01-08T00:31:06.351Z [INFO] stable state for Leader reached at 2024-01-08 00:31:06.24541993 +0000 UTC m=+12.292308973 (1 nodes), 57.713897ms from start of poll, 58.136438ms from cluster start. Timeout at 2024-01-08 00:31:06.351375156 +0000 UTC m=+12.398264218, 105.955245ms after stability +testLoggerAdapter verbose: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.351Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: starting snapshot up to: index=12 + testing.go:149: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.351Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: starting snapshot up to: index=12 + testing.go:153: 2024-01-08T00:31:06.352Z [INFO] creating new snapshot: path=/tmp/raft415338014/snapshots/2-12-1704673866352.tmp +testLoggerAdapter verbose: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.359Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: snapshot complete up to: index=12 + testing.go:149: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.359Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: snapshot complete up to: index=12 + testing.go:153: 2024-01-08T00:31:06.360Z [INFO] creating new snapshot: path=/tmp/raft415338014/snapshots/2-10013-1704673866360.tmp +testLoggerAdapter verbose: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.368Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: copied to local snapshot: bytes=71 + testing.go:149: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.368Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.368Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: restored user snapshot: index=1 + testing.go:149: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.368Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: restored user snapshot: index=1 +testLoggerAdapter verbose: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.369Z [ERROR] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: failed to get log: index=23 error="log not found" + testing.go:149: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.369Z [ERROR] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.369Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: aborting pipeline replication: peer="{Voter server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0 de0fcf05-17c2-a10a-c09c-a660e2b9f5f0}" + testing.go:149: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.369Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: aborting pipeline replication: peer="{Voter server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0 de0fcf05-17c2-a10a-c09c-a660e2b9f5f0}" +testLoggerAdapter verbose: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.369Z [ERROR] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: failed to get log: index=23 error="log not found" + testing.go:149: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.369Z [ERROR] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.369Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: aborting pipeline replication: peer="{Voter server-ea94504c-b9dd-9d53-9131-959998be4670 ea94504c-b9dd-9d53-9131-959998be4670}" + testing.go:149: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.369Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: aborting pipeline replication: peer="{Voter server-ea94504c-b9dd-9d53-9131-959998be4670 ea94504c-b9dd-9d53-9131-959998be4670}" +testLoggerAdapter verbose: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.376Z [ERROR] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: failed to get log: index=23 error="log not found" + testing.go:149: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.376Z [ERROR] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.376Z [ERROR] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: failed to get log: index=23 error="log not found" + testing.go:149: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.376Z [ERROR] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: failed to get log: index=23 error="log not found" + testing.go:153: 2024-01-08T00:31:06.376Z [INFO] creating new snapshot: path=/tmp/raft2915594357/snapshots/2-10013-1704673866376.tmp + testing.go:153: 2024-01-08T00:31:06.377Z [INFO] creating new snapshot: path=/tmp/raft1635674279/snapshots/2-10013-1704673866377.tmp +testLoggerAdapter verbose: server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0: 2024-01-08T00:31:06.388Z [INFO] server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0: copied to local snapshot: bytes=71 + testing.go:149: server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0: 2024-01-08T00:31:06.388Z [INFO] server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0: 2024-01-08T00:31:06.388Z [INFO] server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0: Installed remote snapshot + testing.go:149: server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0: 2024-01-08T00:31:06.388Z [INFO] server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0: Installed remote snapshot +testLoggerAdapter verbose: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.388Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: pipelining replication: peer="{Voter server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0 de0fcf05-17c2-a10a-c09c-a660e2b9f5f0}" + testing.go:149: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.388Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: pipelining replication: peer="{Voter server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0 de0fcf05-17c2-a10a-c09c-a660e2b9f5f0}" +testLoggerAdapter verbose: server-ea94504c-b9dd-9d53-9131-959998be4670: 2024-01-08T00:31:06.388Z [INFO] server-ea94504c-b9dd-9d53-9131-959998be4670: copied to local snapshot: bytes=71 + testing.go:149: server-ea94504c-b9dd-9d53-9131-959998be4670: 2024-01-08T00:31:06.388Z [INFO] server-ea94504c-b9dd-9d53-9131-959998be4670: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-ea94504c-b9dd-9d53-9131-959998be4670: 2024-01-08T00:31:06.388Z [INFO] server-ea94504c-b9dd-9d53-9131-959998be4670: Installed remote snapshot + testing.go:149: server-ea94504c-b9dd-9d53-9131-959998be4670: 2024-01-08T00:31:06.388Z [INFO] server-ea94504c-b9dd-9d53-9131-959998be4670: Installed remote snapshot +testLoggerAdapter verbose: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.389Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: pipelining replication: peer="{Voter server-ea94504c-b9dd-9d53-9131-959998be4670 ea94504c-b9dd-9d53-9131-959998be4670}" + testing.go:149: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.389Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: pipelining replication: peer="{Voter server-ea94504c-b9dd-9d53-9131-959998be4670 ea94504c-b9dd-9d53-9131-959998be4670}" +testLoggerAdapter verbose: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.405Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: aborting pipeline replication: peer="{Voter server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0 de0fcf05-17c2-a10a-c09c-a660e2b9f5f0}" + testing.go:149: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.405Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: aborting pipeline replication: peer="{Voter server-de0fcf05-17c2-a10a-c09c-a660e2b9f5f0 de0fcf05-17c2-a10a-c09c-a660e2b9f5f0}" +testLoggerAdapter verbose: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.405Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: aborting pipeline replication: peer="{Voter server-ea94504c-b9dd-9d53-9131-959998be4670 ea94504c-b9dd-9d53-9131-959998be4670}" + testing.go:149: server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: 2024-01-08T00:31:06.405Z [INFO] server-7a391fc5-a4ec-4b75-abc3-0871ff3f0ec0: aborting pipeline replication: peer="{Voter server-ea94504c-b9dd-9d53-9131-959998be4670 ea94504c-b9dd-9d53-9131-959998be4670}" +--- PASS: TestRaft_UserRestore (1.39s) === RUN TestRaft_SendSnapshotFollower -testLoggerAdapter verbose: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.341Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f27e3834-1e25-de4f-7758-0769fcf30041 Address:f27e3834-1e25-de4f-7758-0769fcf30041} {Suffrage:Voter ID:server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f Address:9a864b8b-b5cb-2054-a3b5-4d03564f8f1f} {Suffrage:Voter ID:server-2948bb85-9435-564c-6883-21ba28fb987b Address:2948bb85-9435-564c-6883-21ba28fb987b}]" - testing.go:149: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.341Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f27e3834-1e25-de4f-7758-0769fcf30041 Address:f27e3834-1e25-de4f-7758-0769fcf30041} {Suffrage:Voter ID:server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f Address:9a864b8b-b5cb-2054-a3b5-4d03564f8f1f} {Suffrage:Voter ID:server-2948bb85-9435-564c-6883-21ba28fb987b Address:2948bb85-9435-564c-6883-21ba28fb987b}]" -testLoggerAdapter verbose: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.342Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: entering follower state: follower="Node at f27e3834-1e25-de4f-7758-0769fcf30041 [Follower]" leader= - testing.go:149: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.342Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: entering follower state: follower="Node at f27e3834-1e25-de4f-7758-0769fcf30041 [Follower]" leader= -testLoggerAdapter verbose: server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f: 2025-02-09T06:52:31.342Z [INFO] server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f27e3834-1e25-de4f-7758-0769fcf30041 Address:f27e3834-1e25-de4f-7758-0769fcf30041} {Suffrage:Voter ID:server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f Address:9a864b8b-b5cb-2054-a3b5-4d03564f8f1f} {Suffrage:Voter ID:server-2948bb85-9435-564c-6883-21ba28fb987b Address:2948bb85-9435-564c-6883-21ba28fb987b}]" - testing.go:149: server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f: 2025-02-09T06:52:31.342Z [INFO] server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f27e3834-1e25-de4f-7758-0769fcf30041 Address:f27e3834-1e25-de4f-7758-0769fcf30041} {Suffrage:Voter ID:server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f Address:9a864b8b-b5cb-2054-a3b5-4d03564f8f1f} {Suffrage:Voter ID:server-2948bb85-9435-564c-6883-21ba28fb987b Address:2948bb85-9435-564c-6883-21ba28fb987b}]" -testLoggerAdapter verbose: server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f: 2025-02-09T06:52:31.342Z [INFO] server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f: entering follower state: follower="Node at 9a864b8b-b5cb-2054-a3b5-4d03564f8f1f [Follower]" leader= - testing.go:149: server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f: 2025-02-09T06:52:31.342Z [INFO] server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f: entering follower state: follower="Node at 9a864b8b-b5cb-2054-a3b5-4d03564f8f1f [Follower]" leader= -testLoggerAdapter verbose: server-2948bb85-9435-564c-6883-21ba28fb987b: 2025-02-09T06:52:31.342Z [INFO] server-2948bb85-9435-564c-6883-21ba28fb987b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f27e3834-1e25-de4f-7758-0769fcf30041 Address:f27e3834-1e25-de4f-7758-0769fcf30041} {Suffrage:Voter ID:server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f Address:9a864b8b-b5cb-2054-a3b5-4d03564f8f1f} {Suffrage:Voter ID:server-2948bb85-9435-564c-6883-21ba28fb987b Address:2948bb85-9435-564c-6883-21ba28fb987b}]" - testing.go:149: server-2948bb85-9435-564c-6883-21ba28fb987b: 2025-02-09T06:52:31.342Z [INFO] server-2948bb85-9435-564c-6883-21ba28fb987b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f27e3834-1e25-de4f-7758-0769fcf30041 Address:f27e3834-1e25-de4f-7758-0769fcf30041} {Suffrage:Voter ID:server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f Address:9a864b8b-b5cb-2054-a3b5-4d03564f8f1f} {Suffrage:Voter ID:server-2948bb85-9435-564c-6883-21ba28fb987b Address:2948bb85-9435-564c-6883-21ba28fb987b}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:31.342Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T06:52:31.342Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-2948bb85-9435-564c-6883-21ba28fb987b: 2025-02-09T06:52:31.342Z [INFO] server-2948bb85-9435-564c-6883-21ba28fb987b: entering follower state: follower="Node at 2948bb85-9435-564c-6883-21ba28fb987b [Follower]" leader= - testing.go:149: server-2948bb85-9435-564c-6883-21ba28fb987b: 2025-02-09T06:52:31.342Z [INFO] server-2948bb85-9435-564c-6883-21ba28fb987b: entering follower state: follower="Node at 2948bb85-9435-564c-6883-21ba28fb987b [Follower]" leader= -testLoggerAdapter verbose: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.406Z [WARN] server-f27e3834-1e25-de4f-7758-0769fcf30041: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.406Z [WARN] server-f27e3834-1e25-de4f-7758-0769fcf30041: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.406Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: entering candidate state: node="Node at f27e3834-1e25-de4f-7758-0769fcf30041 [Candidate]" term=2 - testing.go:149: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.406Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: entering candidate state: node="Node at f27e3834-1e25-de4f-7758-0769fcf30041 [Candidate]" term=2 -testLoggerAdapter verbose: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.406Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: election won: tally=2 - testing.go:149: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.406Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: election won: tally=2 -testLoggerAdapter verbose: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.406Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: entering leader state: leader="Node at f27e3834-1e25-de4f-7758-0769fcf30041 [Leader]" - testing.go:149: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.406Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: entering leader state: leader="Node at f27e3834-1e25-de4f-7758-0769fcf30041 [Leader]" -testLoggerAdapter verbose: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.406Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: added peer, starting replication: peer=server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f - testing.go:149: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.406Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: added peer, starting replication: peer=server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f -testLoggerAdapter verbose: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.406Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: added peer, starting replication: peer=server-2948bb85-9435-564c-6883-21ba28fb987b - testing.go:149: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.406Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: added peer, starting replication: peer=server-2948bb85-9435-564c-6883-21ba28fb987b -testLoggerAdapter verbose: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.406Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: pipelining replication: peer="{Voter server-2948bb85-9435-564c-6883-21ba28fb987b 2948bb85-9435-564c-6883-21ba28fb987b}" - testing.go:149: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.406Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: pipelining replication: peer="{Voter server-2948bb85-9435-564c-6883-21ba28fb987b 2948bb85-9435-564c-6883-21ba28fb987b}" -testLoggerAdapter verbose: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.406Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: pipelining replication: peer="{Voter server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f 9a864b8b-b5cb-2054-a3b5-4d03564f8f1f}" - testing.go:149: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.406Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: pipelining replication: peer="{Voter server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f 9a864b8b-b5cb-2054-a3b5-4d03564f8f1f}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:31.511Z [INFO] stable state for Follower reached at 2025-02-09 06:52:31.406548301 +0000 UTC m=+12.385357065 (2 nodes), 64.2697ms from start of poll, 64.654679ms from cluster start. Timeout at 2025-02-09 06:52:31.511859974 +0000 UTC m=+12.490668741, 105.311676ms after stability - testing.go:149: cluster: 2025-02-09T06:52:31.511Z [INFO] stable state for Follower reached at 2025-02-09 06:52:31.406548301 +0000 UTC m=+12.385357065 (2 nodes), 64.2697ms from start of poll, 64.654679ms from cluster start. Timeout at 2025-02-09 06:52:31.511859974 +0000 UTC m=+12.490668741, 105.311676ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:31.511Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:31.511Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:31.617Z [INFO] stable state for Leader reached at 2025-02-09 06:52:31.511933975 +0000 UTC m=+12.490742738 (1 nodes), 481ns from start of poll, 170.040352ms from cluster start. Timeout at 2025-02-09 06:52:31.617750979 +0000 UTC m=+12.596559739, 105.817001ms after stability - testing.go:149: cluster: 2025-02-09T06:52:31.617Z [INFO] stable state for Leader reached at 2025-02-09 06:52:31.511933975 +0000 UTC m=+12.490742738 (1 nodes), 481ns from start of poll, 170.040352ms from cluster start. Timeout at 2025-02-09 06:52:31.617750979 +0000 UTC m=+12.596559739, 105.817001ms after stability -testLoggerAdapter verbose: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.617Z [ERROR] server-f27e3834-1e25-de4f-7758-0769fcf30041: failed to pipeline appendEntries: peer="{Voter server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f 9a864b8b-b5cb-2054-a3b5-4d03564f8f1f}" error="append pipeline closed" - testing.go:149: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.617Z [ERROR] server-f27e3834-1e25-de4f-7758-0769fcf30041: failed to pipeline appendEntries: peer="{Voter server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f 9a864b8b-b5cb-2054-a3b5-4d03564f8f1f}" error="append pipeline closed" -testLoggerAdapter verbose: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.617Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: aborting pipeline replication: peer="{Voter server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f 9a864b8b-b5cb-2054-a3b5-4d03564f8f1f}" - testing.go:149: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.617Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: aborting pipeline replication: peer="{Voter server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f 9a864b8b-b5cb-2054-a3b5-4d03564f8f1f}" -testLoggerAdapter verbose: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.617Z [ERROR] server-f27e3834-1e25-de4f-7758-0769fcf30041: failed to appendEntries to: peer="{Voter server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f 9a864b8b-b5cb-2054-a3b5-4d03564f8f1f}" error="failed to connect to peer: 9a864b8b-b5cb-2054-a3b5-4d03564f8f1f" - testing.go:149: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.617Z [ERROR] server-f27e3834-1e25-de4f-7758-0769fcf30041: failed to appendEntries to: peer="{Voter server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f 9a864b8b-b5cb-2054-a3b5-4d03564f8f1f}" error="failed to connect to peer: 9a864b8b-b5cb-2054-a3b5-4d03564f8f1f" +testLoggerAdapter verbose: server-e85a8395-e223-4c71-1a24-3b8df5b08a46: 2024-01-08T00:31:06.407Z [INFO] server-e85a8395-e223-4c71-1a24-3b8df5b08a46: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e85a8395-e223-4c71-1a24-3b8df5b08a46 Address:e85a8395-e223-4c71-1a24-3b8df5b08a46} {Suffrage:Voter ID:server-cd07f062-9c4f-8158-b15c-e5b208152459 Address:cd07f062-9c4f-8158-b15c-e5b208152459} {Suffrage:Voter ID:server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89 Address:1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89}]" + testing.go:149: server-e85a8395-e223-4c71-1a24-3b8df5b08a46: 2024-01-08T00:31:06.407Z [INFO] server-e85a8395-e223-4c71-1a24-3b8df5b08a46: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e85a8395-e223-4c71-1a24-3b8df5b08a46 Address:e85a8395-e223-4c71-1a24-3b8df5b08a46} {Suffrage:Voter ID:server-cd07f062-9c4f-8158-b15c-e5b208152459 Address:cd07f062-9c4f-8158-b15c-e5b208152459} {Suffrage:Voter ID:server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89 Address:1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89}]" +testLoggerAdapter verbose: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.407Z [INFO] server-cd07f062-9c4f-8158-b15c-e5b208152459: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e85a8395-e223-4c71-1a24-3b8df5b08a46 Address:e85a8395-e223-4c71-1a24-3b8df5b08a46} {Suffrage:Voter ID:server-cd07f062-9c4f-8158-b15c-e5b208152459 Address:cd07f062-9c4f-8158-b15c-e5b208152459} {Suffrage:Voter ID:server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89 Address:1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89}]" + testing.go:149: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.407Z [INFO] server-cd07f062-9c4f-8158-b15c-e5b208152459: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e85a8395-e223-4c71-1a24-3b8df5b08a46 Address:e85a8395-e223-4c71-1a24-3b8df5b08a46} {Suffrage:Voter ID:server-cd07f062-9c4f-8158-b15c-e5b208152459 Address:cd07f062-9c4f-8158-b15c-e5b208152459} {Suffrage:Voter ID:server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89 Address:1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89}]" +testLoggerAdapter verbose: server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89: 2024-01-08T00:31:06.407Z [INFO] server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e85a8395-e223-4c71-1a24-3b8df5b08a46 Address:e85a8395-e223-4c71-1a24-3b8df5b08a46} {Suffrage:Voter ID:server-cd07f062-9c4f-8158-b15c-e5b208152459 Address:cd07f062-9c4f-8158-b15c-e5b208152459} {Suffrage:Voter ID:server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89 Address:1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89}]" + testing.go:149: server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89: 2024-01-08T00:31:06.407Z [INFO] server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e85a8395-e223-4c71-1a24-3b8df5b08a46 Address:e85a8395-e223-4c71-1a24-3b8df5b08a46} {Suffrage:Voter ID:server-cd07f062-9c4f-8158-b15c-e5b208152459 Address:cd07f062-9c4f-8158-b15c-e5b208152459} {Suffrage:Voter ID:server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89 Address:1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:06.407Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T00:31:06.407Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: server-e85a8395-e223-4c71-1a24-3b8df5b08a46: 2024-01-08T00:31:06.407Z [INFO] server-e85a8395-e223-4c71-1a24-3b8df5b08a46: entering follower state: follower="Node at e85a8395-e223-4c71-1a24-3b8df5b08a46 [Follower]" leader= + testing.go:149: server-e85a8395-e223-4c71-1a24-3b8df5b08a46: 2024-01-08T00:31:06.407Z [INFO] server-e85a8395-e223-4c71-1a24-3b8df5b08a46: entering follower state: follower="Node at e85a8395-e223-4c71-1a24-3b8df5b08a46 [Follower]" leader= +testLoggerAdapter verbose: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.407Z [INFO] server-cd07f062-9c4f-8158-b15c-e5b208152459: entering follower state: follower="Node at cd07f062-9c4f-8158-b15c-e5b208152459 [Follower]" leader= + testing.go:149: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.407Z [INFO] server-cd07f062-9c4f-8158-b15c-e5b208152459: entering follower state: follower="Node at cd07f062-9c4f-8158-b15c-e5b208152459 [Follower]" leader= +testLoggerAdapter verbose: server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89: 2024-01-08T00:31:06.407Z [INFO] server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89: entering follower state: follower="Node at 1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89 [Follower]" leader= + testing.go:149: server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89: 2024-01-08T00:31:06.407Z [INFO] server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89: entering follower state: follower="Node at 1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89 [Follower]" leader= +testLoggerAdapter verbose: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.470Z [WARN] server-cd07f062-9c4f-8158-b15c-e5b208152459: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.470Z [WARN] server-cd07f062-9c4f-8158-b15c-e5b208152459: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.470Z [INFO] server-cd07f062-9c4f-8158-b15c-e5b208152459: entering candidate state: node="Node at cd07f062-9c4f-8158-b15c-e5b208152459 [Candidate]" term=2 + testing.go:149: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.470Z [INFO] server-cd07f062-9c4f-8158-b15c-e5b208152459: entering candidate state: node="Node at cd07f062-9c4f-8158-b15c-e5b208152459 [Candidate]" term=2 +testLoggerAdapter verbose: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.470Z [INFO] server-cd07f062-9c4f-8158-b15c-e5b208152459: election won: tally=2 + testing.go:149: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.470Z [INFO] server-cd07f062-9c4f-8158-b15c-e5b208152459: election won: tally=2 +testLoggerAdapter verbose: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.470Z [INFO] server-cd07f062-9c4f-8158-b15c-e5b208152459: entering leader state: leader="Node at cd07f062-9c4f-8158-b15c-e5b208152459 [Leader]" + testing.go:149: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.470Z [INFO] server-cd07f062-9c4f-8158-b15c-e5b208152459: entering leader state: leader="Node at cd07f062-9c4f-8158-b15c-e5b208152459 [Leader]" +testLoggerAdapter verbose: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.470Z [INFO] server-cd07f062-9c4f-8158-b15c-e5b208152459: added peer, starting replication: peer=server-e85a8395-e223-4c71-1a24-3b8df5b08a46 + testing.go:149: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.470Z [INFO] server-cd07f062-9c4f-8158-b15c-e5b208152459: added peer, starting replication: peer=server-e85a8395-e223-4c71-1a24-3b8df5b08a46 +testLoggerAdapter verbose: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.470Z [INFO] server-cd07f062-9c4f-8158-b15c-e5b208152459: added peer, starting replication: peer=server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89 + testing.go:149: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.470Z [INFO] server-cd07f062-9c4f-8158-b15c-e5b208152459: added peer, starting replication: peer=server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89 +testLoggerAdapter verbose: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.470Z [INFO] server-cd07f062-9c4f-8158-b15c-e5b208152459: pipelining replication: peer="{Voter server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89 1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89}" + testing.go:149: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.470Z [INFO] server-cd07f062-9c4f-8158-b15c-e5b208152459: pipelining replication: peer="{Voter server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89 1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89}" +testLoggerAdapter verbose: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.471Z [INFO] server-cd07f062-9c4f-8158-b15c-e5b208152459: pipelining replication: peer="{Voter server-e85a8395-e223-4c71-1a24-3b8df5b08a46 e85a8395-e223-4c71-1a24-3b8df5b08a46}" + testing.go:149: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.471Z [INFO] server-cd07f062-9c4f-8158-b15c-e5b208152459: pipelining replication: peer="{Voter server-e85a8395-e223-4c71-1a24-3b8df5b08a46 e85a8395-e223-4c71-1a24-3b8df5b08a46}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:06.576Z [INFO] stable state for Follower reached at 2024-01-08 00:31:06.471107604 +0000 UTC m=+12.517996669 (2 nodes), 63.365948ms from start of poll, 63.80465ms from cluster start. Timeout at 2024-01-08 00:31:06.576409529 +0000 UTC m=+12.623298585, 105.301916ms after stability + testing.go:149: cluster: 2024-01-08T00:31:06.576Z [INFO] stable state for Follower reached at 2024-01-08 00:31:06.471107604 +0000 UTC m=+12.517996669 (2 nodes), 63.365948ms from start of poll, 63.80465ms from cluster start. Timeout at 2024-01-08 00:31:06.576409529 +0000 UTC m=+12.623298585, 105.301916ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:06.576Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:06.576Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:06.681Z [INFO] stable state for Leader reached at 2024-01-08 00:31:06.576607318 +0000 UTC m=+12.623496368 (1 nodes), 2.053µs from start of poll, 169.304349ms from cluster start. Timeout at 2024-01-08 00:31:06.681932607 +0000 UTC m=+12.728821674, 105.325306ms after stability + testing.go:149: cluster: 2024-01-08T00:31:06.681Z [INFO] stable state for Leader reached at 2024-01-08 00:31:06.576607318 +0000 UTC m=+12.623496368 (1 nodes), 2.053µs from start of poll, 169.304349ms from cluster start. Timeout at 2024-01-08 00:31:06.681932607 +0000 UTC m=+12.728821674, 105.325306ms after stability +testLoggerAdapter verbose: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.682Z [ERROR] server-cd07f062-9c4f-8158-b15c-e5b208152459: failed to pipeline appendEntries: peer="{Voter server-e85a8395-e223-4c71-1a24-3b8df5b08a46 e85a8395-e223-4c71-1a24-3b8df5b08a46}" error="append pipeline closed" + testing.go:149: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.682Z [ERROR] server-cd07f062-9c4f-8158-b15c-e5b208152459: failed to pipeline appendEntries: peer="{Voter server-e85a8395-e223-4c71-1a24-3b8df5b08a46 e85a8395-e223-4c71-1a24-3b8df5b08a46}" error="append pipeline closed" +testLoggerAdapter verbose: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.682Z [INFO] server-cd07f062-9c4f-8158-b15c-e5b208152459: aborting pipeline replication: peer="{Voter server-e85a8395-e223-4c71-1a24-3b8df5b08a46 e85a8395-e223-4c71-1a24-3b8df5b08a46}" + testing.go:149: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.682Z [INFO] server-cd07f062-9c4f-8158-b15c-e5b208152459: aborting pipeline replication: peer="{Voter server-e85a8395-e223-4c71-1a24-3b8df5b08a46 e85a8395-e223-4c71-1a24-3b8df5b08a46}" +testLoggerAdapter verbose: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.682Z [ERROR] server-cd07f062-9c4f-8158-b15c-e5b208152459: failed to appendEntries to: peer="{Voter server-e85a8395-e223-4c71-1a24-3b8df5b08a46 e85a8395-e223-4c71-1a24-3b8df5b08a46}" error="failed to connect to peer: e85a8395-e223-4c71-1a24-3b8df5b08a46" + testing.go:149: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.682Z [ERROR] server-cd07f062-9c4f-8158-b15c-e5b208152459: failed to appendEntries to: peer="{Voter server-e85a8395-e223-4c71-1a24-3b8df5b08a46 e85a8395-e223-4c71-1a24-3b8df5b08a46}" error="failed to connect to peer: e85a8395-e223-4c71-1a24-3b8df5b08a46" raft_test.go:1320: [INFO] Finished apply without behind follower -testLoggerAdapter verbose: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.619Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: starting snapshot up to: index=102 - testing.go:149: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.619Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: starting snapshot up to: index=102 - testing.go:153: 2025-02-09T06:52:31.619Z [INFO] creating new snapshot: path=/tmp/raft4257923744/snapshots/2-102-1739083951619.tmp -testLoggerAdapter verbose: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.623Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: compacting logs: from=1 to=92 - testing.go:149: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.623Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: compacting logs: from=1 to=92 -testLoggerAdapter verbose: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.623Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: snapshot complete up to: index=102 - testing.go:149: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.623Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: snapshot complete up to: index=102 -testLoggerAdapter verbose: server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f: 2025-02-09T06:52:31.623Z [ERROR] server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f: failed to take snapshot: error="nothing new to snapshot" - testing.go:149: server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f: 2025-02-09T06:52:31.623Z [ERROR] server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f: failed to take snapshot: error="nothing new to snapshot" -testLoggerAdapter verbose: server-2948bb85-9435-564c-6883-21ba28fb987b: 2025-02-09T06:52:31.623Z [INFO] server-2948bb85-9435-564c-6883-21ba28fb987b: starting snapshot up to: index=91 - testing.go:149: server-2948bb85-9435-564c-6883-21ba28fb987b: 2025-02-09T06:52:31.623Z [INFO] server-2948bb85-9435-564c-6883-21ba28fb987b: starting snapshot up to: index=91 - testing.go:153: 2025-02-09T06:52:31.623Z [INFO] creating new snapshot: path=/tmp/raft2201791858/snapshots/2-91-1739083951623.tmp -testLoggerAdapter verbose: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.624Z [ERROR] server-f27e3834-1e25-de4f-7758-0769fcf30041: failed to heartbeat to: peer=9a864b8b-b5cb-2054-a3b5-4d03564f8f1f error="failed to connect to peer: 9a864b8b-b5cb-2054-a3b5-4d03564f8f1f" - testing.go:149: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.624Z [ERROR] server-f27e3834-1e25-de4f-7758-0769fcf30041: failed to heartbeat to: peer=9a864b8b-b5cb-2054-a3b5-4d03564f8f1f error="failed to connect to peer: 9a864b8b-b5cb-2054-a3b5-4d03564f8f1f" -testLoggerAdapter verbose: server-2948bb85-9435-564c-6883-21ba28fb987b: 2025-02-09T06:52:31.625Z [INFO] server-2948bb85-9435-564c-6883-21ba28fb987b: compacting logs: from=1 to=91 - testing.go:149: server-2948bb85-9435-564c-6883-21ba28fb987b: 2025-02-09T06:52:31.625Z [INFO] server-2948bb85-9435-564c-6883-21ba28fb987b: compacting logs: from=1 to=91 -testLoggerAdapter verbose: server-2948bb85-9435-564c-6883-21ba28fb987b: 2025-02-09T06:52:31.625Z [INFO] server-2948bb85-9435-564c-6883-21ba28fb987b: snapshot complete up to: index=91 - testing.go:149: server-2948bb85-9435-564c-6883-21ba28fb987b: 2025-02-09T06:52:31.625Z [INFO] server-2948bb85-9435-564c-6883-21ba28fb987b: snapshot complete up to: index=91 -testLoggerAdapter verbose: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.628Z [ERROR] server-f27e3834-1e25-de4f-7758-0769fcf30041: failed to get log: index=2 error="log not found" - testing.go:149: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.628Z [ERROR] server-f27e3834-1e25-de4f-7758-0769fcf30041: failed to get log: index=2 error="log not found" - testing.go:153: 2025-02-09T06:52:31.629Z [INFO] creating new snapshot: path=/tmp/raft1433449189/snapshots/2-102-1739083951629.tmp -testLoggerAdapter verbose: server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f: 2025-02-09T06:52:31.632Z [INFO] server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f: copied to local snapshot: bytes=693 - testing.go:149: server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f: 2025-02-09T06:52:31.632Z [INFO] server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f: copied to local snapshot: bytes=693 -testLoggerAdapter verbose: server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f: 2025-02-09T06:52:31.632Z [INFO] server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f: Installed remote snapshot - testing.go:149: server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f: 2025-02-09T06:52:31.632Z [INFO] server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f: Installed remote snapshot -testLoggerAdapter verbose: server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f: 2025-02-09T06:52:31.632Z [WARN] server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f: skipping application of old log: index=102 - testing.go:149: server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f: 2025-02-09T06:52:31.632Z [WARN] server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f: skipping application of old log: index=102 -testLoggerAdapter verbose: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.632Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: pipelining replication: peer="{Voter server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f 9a864b8b-b5cb-2054-a3b5-4d03564f8f1f}" - testing.go:149: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.632Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: pipelining replication: peer="{Voter server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f 9a864b8b-b5cb-2054-a3b5-4d03564f8f1f}" -testLoggerAdapter verbose: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.636Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: aborting pipeline replication: peer="{Voter server-2948bb85-9435-564c-6883-21ba28fb987b 2948bb85-9435-564c-6883-21ba28fb987b}" - testing.go:149: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.636Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: aborting pipeline replication: peer="{Voter server-2948bb85-9435-564c-6883-21ba28fb987b 2948bb85-9435-564c-6883-21ba28fb987b}" -testLoggerAdapter verbose: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.636Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: aborting pipeline replication: peer="{Voter server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f 9a864b8b-b5cb-2054-a3b5-4d03564f8f1f}" - testing.go:149: server-f27e3834-1e25-de4f-7758-0769fcf30041: 2025-02-09T06:52:31.636Z [INFO] server-f27e3834-1e25-de4f-7758-0769fcf30041: aborting pipeline replication: peer="{Voter server-9a864b8b-b5cb-2054-a3b5-4d03564f8f1f 9a864b8b-b5cb-2054-a3b5-4d03564f8f1f}" ---- PASS: TestRaft_SendSnapshotFollower (0.30s) +testLoggerAdapter verbose: server-e85a8395-e223-4c71-1a24-3b8df5b08a46: 2024-01-08T00:31:06.684Z [ERROR] server-e85a8395-e223-4c71-1a24-3b8df5b08a46: failed to take snapshot: error="nothing new to snapshot" + testing.go:149: server-e85a8395-e223-4c71-1a24-3b8df5b08a46: 2024-01-08T00:31:06.684Z [ERROR] server-e85a8395-e223-4c71-1a24-3b8df5b08a46: failed to take snapshot: error="nothing new to snapshot" +testLoggerAdapter verbose: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.684Z [INFO] server-cd07f062-9c4f-8158-b15c-e5b208152459: starting snapshot up to: index=102 + testing.go:149: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.684Z [INFO] server-cd07f062-9c4f-8158-b15c-e5b208152459: starting snapshot up to: index=102 + testing.go:153: 2024-01-08T00:31:06.684Z [INFO] creating new snapshot: path=/tmp/raft2173245344/snapshots/2-102-1704673866684.tmp +testLoggerAdapter verbose: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.691Z [ERROR] server-cd07f062-9c4f-8158-b15c-e5b208152459: failed to heartbeat to: peer=e85a8395-e223-4c71-1a24-3b8df5b08a46 error="failed to connect to peer: e85a8395-e223-4c71-1a24-3b8df5b08a46" + testing.go:149: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.691Z [ERROR] server-cd07f062-9c4f-8158-b15c-e5b208152459: failed to heartbeat to: peer=e85a8395-e223-4c71-1a24-3b8df5b08a46 error="failed to connect to peer: e85a8395-e223-4c71-1a24-3b8df5b08a46" +testLoggerAdapter verbose: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.692Z [INFO] server-cd07f062-9c4f-8158-b15c-e5b208152459: compacting logs: from=1 to=92 + testing.go:149: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.692Z [INFO] server-cd07f062-9c4f-8158-b15c-e5b208152459: compacting logs: from=1 to=92 +testLoggerAdapter verbose: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.692Z [INFO] server-cd07f062-9c4f-8158-b15c-e5b208152459: snapshot complete up to: index=102 + testing.go:149: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.692Z [INFO] server-cd07f062-9c4f-8158-b15c-e5b208152459: snapshot complete up to: index=102 +testLoggerAdapter verbose: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.692Z [ERROR] server-cd07f062-9c4f-8158-b15c-e5b208152459: failed to get log: index=2 error="log not found" + testing.go:149: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.692Z [ERROR] server-cd07f062-9c4f-8158-b15c-e5b208152459: failed to get log: index=2 error="log not found" +testLoggerAdapter verbose: server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89: 2024-01-08T00:31:06.692Z [INFO] server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89: starting snapshot up to: index=97 + testing.go:149: server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89: 2024-01-08T00:31:06.692Z [INFO] server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89: starting snapshot up to: index=97 + testing.go:153: 2024-01-08T00:31:06.692Z [INFO] creating new snapshot: path=/tmp/raft4234533615/snapshots/2-97-1704673866692.tmp +testLoggerAdapter verbose: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.693Z [ERROR] server-cd07f062-9c4f-8158-b15c-e5b208152459: failed to install snapshot: id=2-102-1704673866684 error="failed to connect to peer: e85a8395-e223-4c71-1a24-3b8df5b08a46" + testing.go:149: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.693Z [ERROR] server-cd07f062-9c4f-8158-b15c-e5b208152459: failed to install snapshot: id=2-102-1704673866684 error="failed to connect to peer: e85a8395-e223-4c71-1a24-3b8df5b08a46" +testLoggerAdapter verbose: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.693Z [ERROR] server-cd07f062-9c4f-8158-b15c-e5b208152459: failed to send snapshot to: peer="{Voter server-e85a8395-e223-4c71-1a24-3b8df5b08a46 e85a8395-e223-4c71-1a24-3b8df5b08a46}" error="failed to connect to peer: e85a8395-e223-4c71-1a24-3b8df5b08a46" + testing.go:149: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.693Z [ERROR] server-cd07f062-9c4f-8158-b15c-e5b208152459: failed to send snapshot to: peer="{Voter server-e85a8395-e223-4c71-1a24-3b8df5b08a46 e85a8395-e223-4c71-1a24-3b8df5b08a46}" error="failed to connect to peer: e85a8395-e223-4c71-1a24-3b8df5b08a46" +testLoggerAdapter verbose: server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89: 2024-01-08T00:31:06.701Z [INFO] server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89: compacting logs: from=1 to=92 + testing.go:149: server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89: 2024-01-08T00:31:06.701Z [INFO] server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89: compacting logs: from=1 to=92 +testLoggerAdapter verbose: server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89: 2024-01-08T00:31:06.701Z [INFO] server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89: snapshot complete up to: index=97 + testing.go:149: server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89: 2024-01-08T00:31:06.701Z [INFO] server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89: snapshot complete up to: index=97 +testLoggerAdapter verbose: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.703Z [ERROR] server-cd07f062-9c4f-8158-b15c-e5b208152459: failed to get log: index=2 error="log not found" + testing.go:149: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.703Z [ERROR] server-cd07f062-9c4f-8158-b15c-e5b208152459: failed to get log: index=2 error="log not found" + testing.go:153: 2024-01-08T00:31:06.703Z [INFO] creating new snapshot: path=/tmp/raft986211076/snapshots/2-102-1704673866703.tmp +testLoggerAdapter verbose: server-e85a8395-e223-4c71-1a24-3b8df5b08a46: 2024-01-08T00:31:06.710Z [INFO] server-e85a8395-e223-4c71-1a24-3b8df5b08a46: copied to local snapshot: bytes=693 + testing.go:149: server-e85a8395-e223-4c71-1a24-3b8df5b08a46: 2024-01-08T00:31:06.710Z [INFO] server-e85a8395-e223-4c71-1a24-3b8df5b08a46: copied to local snapshot: bytes=693 +testLoggerAdapter verbose: server-e85a8395-e223-4c71-1a24-3b8df5b08a46: 2024-01-08T00:31:06.711Z [INFO] server-e85a8395-e223-4c71-1a24-3b8df5b08a46: Installed remote snapshot + testing.go:149: server-e85a8395-e223-4c71-1a24-3b8df5b08a46: 2024-01-08T00:31:06.711Z [INFO] server-e85a8395-e223-4c71-1a24-3b8df5b08a46: Installed remote snapshot +testLoggerAdapter verbose: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.712Z [INFO] server-cd07f062-9c4f-8158-b15c-e5b208152459: aborting pipeline replication: peer="{Voter server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89 1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89}" + testing.go:149: server-cd07f062-9c4f-8158-b15c-e5b208152459: 2024-01-08T00:31:06.712Z [INFO] server-cd07f062-9c4f-8158-b15c-e5b208152459: aborting pipeline replication: peer="{Voter server-1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89 1ebeebc9-fef4-e5bb-6ef5-64c33fe46a89}" +--- PASS: TestRaft_SendSnapshotFollower (0.31s) === RUN TestRaft_SendSnapshotAndLogsFollower -testLoggerAdapter verbose: server-16a7755d-3460-4753-8090-799f13ced7da: 2025-02-09T06:52:31.637Z [INFO] server-16a7755d-3460-4753-8090-799f13ced7da: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-16a7755d-3460-4753-8090-799f13ced7da Address:16a7755d-3460-4753-8090-799f13ced7da} {Suffrage:Voter ID:server-113f0681-7acf-736a-2a6e-1cfb35ac2cce Address:113f0681-7acf-736a-2a6e-1cfb35ac2cce} {Suffrage:Voter ID:server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54 Address:0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54}]" - testing.go:149: server-16a7755d-3460-4753-8090-799f13ced7da: 2025-02-09T06:52:31.637Z [INFO] server-16a7755d-3460-4753-8090-799f13ced7da: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-16a7755d-3460-4753-8090-799f13ced7da Address:16a7755d-3460-4753-8090-799f13ced7da} {Suffrage:Voter ID:server-113f0681-7acf-736a-2a6e-1cfb35ac2cce Address:113f0681-7acf-736a-2a6e-1cfb35ac2cce} {Suffrage:Voter ID:server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54 Address:0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54}]" -testLoggerAdapter verbose: server-16a7755d-3460-4753-8090-799f13ced7da: 2025-02-09T06:52:31.637Z [INFO] server-16a7755d-3460-4753-8090-799f13ced7da: entering follower state: follower="Node at 16a7755d-3460-4753-8090-799f13ced7da [Follower]" leader= -testLoggerAdapter verbose: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.637Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-16a7755d-3460-4753-8090-799f13ced7da Address:16a7755d-3460-4753-8090-799f13ced7da} {Suffrage:Voter ID:server-113f0681-7acf-736a-2a6e-1cfb35ac2cce Address:113f0681-7acf-736a-2a6e-1cfb35ac2cce} {Suffrage:Voter ID:server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54 Address:0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54}]" - testing.go:149: server-16a7755d-3460-4753-8090-799f13ced7da: 2025-02-09T06:52:31.637Z [INFO] server-16a7755d-3460-4753-8090-799f13ced7da: entering follower state: follower="Node at 16a7755d-3460-4753-8090-799f13ced7da [Follower]" leader= - testing.go:149: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.637Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-16a7755d-3460-4753-8090-799f13ced7da Address:16a7755d-3460-4753-8090-799f13ced7da} {Suffrage:Voter ID:server-113f0681-7acf-736a-2a6e-1cfb35ac2cce Address:113f0681-7acf-736a-2a6e-1cfb35ac2cce} {Suffrage:Voter ID:server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54 Address:0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54}]" -testLoggerAdapter verbose: server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54: 2025-02-09T06:52:31.638Z [INFO] server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-16a7755d-3460-4753-8090-799f13ced7da Address:16a7755d-3460-4753-8090-799f13ced7da} {Suffrage:Voter ID:server-113f0681-7acf-736a-2a6e-1cfb35ac2cce Address:113f0681-7acf-736a-2a6e-1cfb35ac2cce} {Suffrage:Voter ID:server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54 Address:0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54}]" - testing.go:149: server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54: 2025-02-09T06:52:31.638Z [INFO] server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-16a7755d-3460-4753-8090-799f13ced7da Address:16a7755d-3460-4753-8090-799f13ced7da} {Suffrage:Voter ID:server-113f0681-7acf-736a-2a6e-1cfb35ac2cce Address:113f0681-7acf-736a-2a6e-1cfb35ac2cce} {Suffrage:Voter ID:server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54 Address:0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:31.638Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T06:52:31.638Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.638Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: entering follower state: follower="Node at 113f0681-7acf-736a-2a6e-1cfb35ac2cce [Follower]" leader= - testing.go:149: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.638Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: entering follower state: follower="Node at 113f0681-7acf-736a-2a6e-1cfb35ac2cce [Follower]" leader= -testLoggerAdapter verbose: server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54: 2025-02-09T06:52:31.638Z [INFO] server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54: entering follower state: follower="Node at 0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54 [Follower]" leader= - testing.go:149: server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54: 2025-02-09T06:52:31.638Z [INFO] server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54: entering follower state: follower="Node at 0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54 [Follower]" leader= -testLoggerAdapter verbose: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.691Z [WARN] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.691Z [WARN] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.691Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: entering candidate state: node="Node at 113f0681-7acf-736a-2a6e-1cfb35ac2cce [Candidate]" term=2 - testing.go:149: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.691Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: entering candidate state: node="Node at 113f0681-7acf-736a-2a6e-1cfb35ac2cce [Candidate]" term=2 -testLoggerAdapter verbose: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.691Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: election won: tally=2 - testing.go:149: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.691Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: election won: tally=2 -testLoggerAdapter verbose: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.691Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: entering leader state: leader="Node at 113f0681-7acf-736a-2a6e-1cfb35ac2cce [Leader]" - testing.go:149: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.691Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: entering leader state: leader="Node at 113f0681-7acf-736a-2a6e-1cfb35ac2cce [Leader]" -testLoggerAdapter verbose: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.691Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: added peer, starting replication: peer=server-16a7755d-3460-4753-8090-799f13ced7da - testing.go:149: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.691Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: added peer, starting replication: peer=server-16a7755d-3460-4753-8090-799f13ced7da -testLoggerAdapter verbose: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.691Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: added peer, starting replication: peer=server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54 - testing.go:149: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.691Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: added peer, starting replication: peer=server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54 -testLoggerAdapter verbose: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.691Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: pipelining replication: peer="{Voter server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54 0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54}" - testing.go:149: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.691Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: pipelining replication: peer="{Voter server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54 0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54}" -testLoggerAdapter verbose: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.691Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: pipelining replication: peer="{Voter server-16a7755d-3460-4753-8090-799f13ced7da 16a7755d-3460-4753-8090-799f13ced7da}" - testing.go:149: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.691Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: pipelining replication: peer="{Voter server-16a7755d-3460-4753-8090-799f13ced7da 16a7755d-3460-4753-8090-799f13ced7da}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:31.797Z [INFO] stable state for Follower reached at 2025-02-09 06:52:31.691773644 +0000 UTC m=+12.670582415 (2 nodes), 53.590529ms from start of poll, 54.090753ms from cluster start. Timeout at 2025-02-09 06:52:31.79767778 +0000 UTC m=+12.776486537, 105.904122ms after stability - testing.go:149: cluster: 2025-02-09T06:52:31.797Z [INFO] stable state for Follower reached at 2025-02-09 06:52:31.691773644 +0000 UTC m=+12.670582415 (2 nodes), 53.590529ms from start of poll, 54.090753ms from cluster start. Timeout at 2025-02-09 06:52:31.79767778 +0000 UTC m=+12.776486537, 105.904122ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:31.797Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:31.797Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:31.903Z [INFO] stable state for Leader reached at 2025-02-09 06:52:31.797761631 +0000 UTC m=+12.776570404 (1 nodes), 529ns from start of poll, 160.078742ms from cluster start. Timeout at 2025-02-09 06:52:31.903304496 +0000 UTC m=+12.882113254, 105.54285ms after stability - testing.go:149: cluster: 2025-02-09T06:52:31.903Z [INFO] stable state for Leader reached at 2025-02-09 06:52:31.797761631 +0000 UTC m=+12.776570404 (1 nodes), 529ns from start of poll, 160.078742ms from cluster start. Timeout at 2025-02-09 06:52:31.903304496 +0000 UTC m=+12.882113254, 105.54285ms after stability -testLoggerAdapter verbose: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.903Z [ERROR] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: failed to pipeline appendEntries: peer="{Voter server-16a7755d-3460-4753-8090-799f13ced7da 16a7755d-3460-4753-8090-799f13ced7da}" error="append pipeline closed" - testing.go:149: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.903Z [ERROR] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: failed to pipeline appendEntries: peer="{Voter server-16a7755d-3460-4753-8090-799f13ced7da 16a7755d-3460-4753-8090-799f13ced7da}" error="append pipeline closed" -testLoggerAdapter verbose: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.903Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: aborting pipeline replication: peer="{Voter server-16a7755d-3460-4753-8090-799f13ced7da 16a7755d-3460-4753-8090-799f13ced7da}" - testing.go:149: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.903Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: aborting pipeline replication: peer="{Voter server-16a7755d-3460-4753-8090-799f13ced7da 16a7755d-3460-4753-8090-799f13ced7da}" -testLoggerAdapter verbose: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.903Z [ERROR] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: failed to appendEntries to: peer="{Voter server-16a7755d-3460-4753-8090-799f13ced7da 16a7755d-3460-4753-8090-799f13ced7da}" error="failed to connect to peer: 16a7755d-3460-4753-8090-799f13ced7da" - testing.go:149: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.903Z [ERROR] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: failed to appendEntries to: peer="{Voter server-16a7755d-3460-4753-8090-799f13ced7da 16a7755d-3460-4753-8090-799f13ced7da}" error="failed to connect to peer: 16a7755d-3460-4753-8090-799f13ced7da" +testLoggerAdapter verbose: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:06.713Z [INFO] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e Address:d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e} {Suffrage:Voter ID:server-bdbd9fe5-b7e0-3828-169f-9446390443e2 Address:bdbd9fe5-b7e0-3828-169f-9446390443e2} {Suffrage:Voter ID:server-2b405c0f-7a8f-8c3d-909a-721aefefba5f Address:2b405c0f-7a8f-8c3d-909a-721aefefba5f}]" + testing.go:149: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:06.713Z [INFO] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e Address:d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e} {Suffrage:Voter ID:server-bdbd9fe5-b7e0-3828-169f-9446390443e2 Address:bdbd9fe5-b7e0-3828-169f-9446390443e2} {Suffrage:Voter ID:server-2b405c0f-7a8f-8c3d-909a-721aefefba5f Address:2b405c0f-7a8f-8c3d-909a-721aefefba5f}]" +testLoggerAdapter verbose: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:06.713Z [INFO] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: entering follower state: follower="Node at d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e [Follower]" leader= + testing.go:149: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:06.713Z [INFO] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: entering follower state: follower="Node at d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e [Follower]" leader= +testLoggerAdapter verbose: server-bdbd9fe5-b7e0-3828-169f-9446390443e2: 2024-01-08T00:31:06.713Z [INFO] server-bdbd9fe5-b7e0-3828-169f-9446390443e2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e Address:d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e} {Suffrage:Voter ID:server-bdbd9fe5-b7e0-3828-169f-9446390443e2 Address:bdbd9fe5-b7e0-3828-169f-9446390443e2} {Suffrage:Voter ID:server-2b405c0f-7a8f-8c3d-909a-721aefefba5f Address:2b405c0f-7a8f-8c3d-909a-721aefefba5f}]" + testing.go:149: server-bdbd9fe5-b7e0-3828-169f-9446390443e2: 2024-01-08T00:31:06.713Z [INFO] server-bdbd9fe5-b7e0-3828-169f-9446390443e2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e Address:d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e} {Suffrage:Voter ID:server-bdbd9fe5-b7e0-3828-169f-9446390443e2 Address:bdbd9fe5-b7e0-3828-169f-9446390443e2} {Suffrage:Voter ID:server-2b405c0f-7a8f-8c3d-909a-721aefefba5f Address:2b405c0f-7a8f-8c3d-909a-721aefefba5f}]" +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:06.714Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e Address:d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e} {Suffrage:Voter ID:server-bdbd9fe5-b7e0-3828-169f-9446390443e2 Address:bdbd9fe5-b7e0-3828-169f-9446390443e2} {Suffrage:Voter ID:server-2b405c0f-7a8f-8c3d-909a-721aefefba5f Address:2b405c0f-7a8f-8c3d-909a-721aefefba5f}]" + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:06.714Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e Address:d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e} {Suffrage:Voter ID:server-bdbd9fe5-b7e0-3828-169f-9446390443e2 Address:bdbd9fe5-b7e0-3828-169f-9446390443e2} {Suffrage:Voter ID:server-2b405c0f-7a8f-8c3d-909a-721aefefba5f Address:2b405c0f-7a8f-8c3d-909a-721aefefba5f}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:06.714Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T00:31:06.714Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: server-bdbd9fe5-b7e0-3828-169f-9446390443e2: 2024-01-08T00:31:06.715Z [INFO] server-bdbd9fe5-b7e0-3828-169f-9446390443e2: entering follower state: follower="Node at bdbd9fe5-b7e0-3828-169f-9446390443e2 [Follower]" leader= + testing.go:149: server-bdbd9fe5-b7e0-3828-169f-9446390443e2: 2024-01-08T00:31:06.715Z [INFO] server-bdbd9fe5-b7e0-3828-169f-9446390443e2: entering follower state: follower="Node at bdbd9fe5-b7e0-3828-169f-9446390443e2 [Follower]" leader= +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:06.715Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: entering follower state: follower="Node at 2b405c0f-7a8f-8c3d-909a-721aefefba5f [Follower]" leader= + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:06.715Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: entering follower state: follower="Node at 2b405c0f-7a8f-8c3d-909a-721aefefba5f [Follower]" leader= +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:06.777Z [WARN] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:06.777Z [WARN] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:06.777Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: entering candidate state: node="Node at 2b405c0f-7a8f-8c3d-909a-721aefefba5f [Candidate]" term=2 + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:06.777Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: entering candidate state: node="Node at 2b405c0f-7a8f-8c3d-909a-721aefefba5f [Candidate]" term=2 +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:06.777Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: election won: tally=2 + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:06.777Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: election won: tally=2 +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:06.778Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: entering leader state: leader="Node at 2b405c0f-7a8f-8c3d-909a-721aefefba5f [Leader]" + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:06.778Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: entering leader state: leader="Node at 2b405c0f-7a8f-8c3d-909a-721aefefba5f [Leader]" +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:06.778Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: added peer, starting replication: peer=server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:06.778Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: added peer, starting replication: peer=server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:06.778Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: added peer, starting replication: peer=server-bdbd9fe5-b7e0-3828-169f-9446390443e2 + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:06.778Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: added peer, starting replication: peer=server-bdbd9fe5-b7e0-3828-169f-9446390443e2 +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:06.778Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: pipelining replication: peer="{Voter server-bdbd9fe5-b7e0-3828-169f-9446390443e2 bdbd9fe5-b7e0-3828-169f-9446390443e2}" + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:06.778Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: pipelining replication: peer="{Voter server-bdbd9fe5-b7e0-3828-169f-9446390443e2 bdbd9fe5-b7e0-3828-169f-9446390443e2}" +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:06.778Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: pipelining replication: peer="{Voter server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e}" + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:06.778Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: pipelining replication: peer="{Voter server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:06.883Z [INFO] stable state for Follower reached at 2024-01-08 00:31:06.778412978 +0000 UTC m=+12.825302022 (2 nodes), 63.911984ms from start of poll, 64.785513ms from cluster start. Timeout at 2024-01-08 00:31:06.883471893 +0000 UTC m=+12.930360922, 105.0589ms after stability + testing.go:149: cluster: 2024-01-08T00:31:06.883Z [INFO] stable state for Follower reached at 2024-01-08 00:31:06.778412978 +0000 UTC m=+12.825302022 (2 nodes), 63.911984ms from start of poll, 64.785513ms from cluster start. Timeout at 2024-01-08 00:31:06.883471893 +0000 UTC m=+12.930360922, 105.0589ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:06.883Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:06.883Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:06.989Z [INFO] stable state for Leader reached at 2024-01-08 00:31:06.883609032 +0000 UTC m=+12.930498064 (1 nodes), 1.99µs from start of poll, 169.981555ms from cluster start. Timeout at 2024-01-08 00:31:06.98933743 +0000 UTC m=+13.036226498, 105.728434ms after stability + testing.go:149: cluster: 2024-01-08T00:31:06.989Z [INFO] stable state for Leader reached at 2024-01-08 00:31:06.883609032 +0000 UTC m=+12.930498064 (1 nodes), 1.99µs from start of poll, 169.981555ms from cluster start. Timeout at 2024-01-08 00:31:06.98933743 +0000 UTC m=+13.036226498, 105.728434ms after stability +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:06.989Z [ERROR] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: failed to pipeline appendEntries: peer="{Voter server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e}" error="append pipeline closed" + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:06.989Z [ERROR] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: failed to pipeline appendEntries: peer="{Voter server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e}" error="append pipeline closed" +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:06.989Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: aborting pipeline replication: peer="{Voter server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e}" + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:06.989Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: aborting pipeline replication: peer="{Voter server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e}" +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:06.989Z [ERROR] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: failed to appendEntries to: peer="{Voter server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e}" error="failed to connect to peer: d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e" + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:06.989Z [ERROR] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: failed to appendEntries to: peer="{Voter server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e}" error="failed to connect to peer: d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e" raft_test.go:1362: [INFO] Finished apply without behind follower -testLoggerAdapter verbose: server-16a7755d-3460-4753-8090-799f13ced7da: 2025-02-09T06:52:31.904Z [ERROR] server-16a7755d-3460-4753-8090-799f13ced7da: failed to take snapshot: error="nothing new to snapshot" - testing.go:149: server-16a7755d-3460-4753-8090-799f13ced7da: 2025-02-09T06:52:31.904Z [ERROR] server-16a7755d-3460-4753-8090-799f13ced7da: failed to take snapshot: error="nothing new to snapshot" -testLoggerAdapter verbose: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.904Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: starting snapshot up to: index=102 - testing.go:149: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.904Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: starting snapshot up to: index=102 - testing.go:153: 2025-02-09T06:52:31.904Z [INFO] creating new snapshot: path=/tmp/raft403308337/snapshots/2-102-1739083951904.tmp -testLoggerAdapter verbose: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.908Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: compacting logs: from=1 to=92 - testing.go:149: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.908Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: compacting logs: from=1 to=92 -testLoggerAdapter verbose: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.908Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: snapshot complete up to: index=102 - testing.go:149: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.908Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: snapshot complete up to: index=102 -testLoggerAdapter verbose: server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54: 2025-02-09T06:52:31.908Z [INFO] server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54: starting snapshot up to: index=98 - testing.go:149: server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54: 2025-02-09T06:52:31.908Z [INFO] server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54: starting snapshot up to: index=98 - testing.go:153: 2025-02-09T06:52:31.908Z [INFO] creating new snapshot: path=/tmp/raft3275006271/snapshots/2-98-1739083951908.tmp -testLoggerAdapter verbose: server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54: 2025-02-09T06:52:31.910Z [INFO] server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54: compacting logs: from=1 to=92 - testing.go:149: server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54: 2025-02-09T06:52:31.910Z [INFO] server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54: compacting logs: from=1 to=92 -testLoggerAdapter verbose: server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54: 2025-02-09T06:52:31.910Z [INFO] server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54: snapshot complete up to: index=98 - testing.go:149: server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54: 2025-02-09T06:52:31.910Z [INFO] server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54: snapshot complete up to: index=98 -testLoggerAdapter verbose: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.911Z [ERROR] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: failed to heartbeat to: peer=16a7755d-3460-4753-8090-799f13ced7da error="failed to connect to peer: 16a7755d-3460-4753-8090-799f13ced7da" - testing.go:149: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.911Z [ERROR] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: failed to heartbeat to: peer=16a7755d-3460-4753-8090-799f13ced7da error="failed to connect to peer: 16a7755d-3460-4753-8090-799f13ced7da" +testLoggerAdapter verbose: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:06.991Z [ERROR] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: failed to take snapshot: error="nothing new to snapshot" + testing.go:149: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:06.991Z [ERROR] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: failed to take snapshot: error="nothing new to snapshot" +testLoggerAdapter verbose: server-bdbd9fe5-b7e0-3828-169f-9446390443e2: 2024-01-08T00:31:06.992Z [INFO] server-bdbd9fe5-b7e0-3828-169f-9446390443e2: starting snapshot up to: index=90 + testing.go:149: server-bdbd9fe5-b7e0-3828-169f-9446390443e2: 2024-01-08T00:31:06.992Z [INFO] server-bdbd9fe5-b7e0-3828-169f-9446390443e2: starting snapshot up to: index=90 + testing.go:153: 2024-01-08T00:31:06.992Z [INFO] creating new snapshot: path=/tmp/raft1382983678/snapshots/2-90-1704673866992.tmp +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:06.994Z [ERROR] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: failed to heartbeat to: peer=d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e error="failed to connect to peer: d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e" + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:06.994Z [ERROR] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: failed to heartbeat to: peer=d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e error="failed to connect to peer: d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e" +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.000Z [ERROR] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: failed to appendEntries to: peer="{Voter server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e}" error="failed to connect to peer: d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e" + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.000Z [ERROR] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: failed to appendEntries to: peer="{Voter server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e}" error="failed to connect to peer: d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e" +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.011Z [ERROR] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: failed to appendEntries to: peer="{Voter server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e}" error="failed to connect to peer: d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e" + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.011Z [ERROR] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: failed to appendEntries to: peer="{Voter server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e}" error="failed to connect to peer: d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e" +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.014Z [ERROR] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: failed to heartbeat to: peer=d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e error="failed to connect to peer: d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e" + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.014Z [ERROR] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: failed to heartbeat to: peer=d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e error="failed to connect to peer: d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e" +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.033Z [ERROR] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: failed to heartbeat to: peer=d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e error="failed to connect to peer: d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e" + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.033Z [ERROR] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: failed to heartbeat to: peer=d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e error="failed to connect to peer: d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e" +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.037Z [WARN] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: failed to contact: server-id=server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e time=50.034278ms + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.037Z [WARN] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: failed to contact: server-id=server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e time=50.034278ms +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.038Z [ERROR] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: failed to appendEntries to: peer="{Voter server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e}" error="failed to connect to peer: d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e" + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.038Z [ERROR] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: failed to appendEntries to: peer="{Voter server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e}" error="failed to connect to peer: d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e" +testLoggerAdapter verbose: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.039Z [WARN] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: heartbeat timeout reached, starting election: last-leader=2b405c0f-7a8f-8c3d-909a-721aefefba5f + testing.go:149: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.039Z [WARN] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: heartbeat timeout reached, starting election: last-leader=2b405c0f-7a8f-8c3d-909a-721aefefba5f +testLoggerAdapter verbose: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.039Z [INFO] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: entering candidate state: node="Node at d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e [Candidate]" term=3 + testing.go:149: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.039Z [INFO] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: entering candidate state: node="Node at d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e [Candidate]" term=3 +testLoggerAdapter verbose: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.039Z [ERROR] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: failed to make requestVote RPC: target="{Voter server-2b405c0f-7a8f-8c3d-909a-721aefefba5f 2b405c0f-7a8f-8c3d-909a-721aefefba5f}" error="failed to connect to peer: 2b405c0f-7a8f-8c3d-909a-721aefefba5f" + testing.go:149: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.039Z [ERROR] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: failed to make requestVote RPC: target="{Voter server-2b405c0f-7a8f-8c3d-909a-721aefefba5f 2b405c0f-7a8f-8c3d-909a-721aefefba5f}" error="failed to connect to peer: 2b405c0f-7a8f-8c3d-909a-721aefefba5f" +testLoggerAdapter verbose: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.039Z [ERROR] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: failed to make requestVote RPC: target="{Voter server-bdbd9fe5-b7e0-3828-169f-9446390443e2 bdbd9fe5-b7e0-3828-169f-9446390443e2}" error="failed to connect to peer: bdbd9fe5-b7e0-3828-169f-9446390443e2" + testing.go:149: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.039Z [ERROR] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: failed to make requestVote RPC: target="{Voter server-bdbd9fe5-b7e0-3828-169f-9446390443e2 bdbd9fe5-b7e0-3828-169f-9446390443e2}" error="failed to connect to peer: bdbd9fe5-b7e0-3828-169f-9446390443e2" +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.065Z [ERROR] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: failed to heartbeat to: peer=d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e error="failed to connect to peer: d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e" + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.065Z [ERROR] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: failed to heartbeat to: peer=d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e error="failed to connect to peer: d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e" +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.083Z [WARN] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: failed to contact: server-id=server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e time=96.863125ms + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.083Z [WARN] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: failed to contact: server-id=server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e time=96.863125ms +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.085Z [ERROR] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: failed to appendEntries to: peer="{Voter server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e}" error="failed to connect to peer: d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e" + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.085Z [ERROR] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: failed to appendEntries to: peer="{Voter server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e}" error="failed to connect to peer: d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e" +testLoggerAdapter verbose: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.103Z [WARN] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: Election timeout reached, restarting election + testing.go:149: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.103Z [WARN] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: Election timeout reached, restarting election +testLoggerAdapter verbose: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.103Z [INFO] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: entering candidate state: node="Node at d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e [Candidate]" term=4 + testing.go:149: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.103Z [INFO] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: entering candidate state: node="Node at d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e [Candidate]" term=4 +testLoggerAdapter verbose: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.104Z [ERROR] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: failed to make requestVote RPC: target="{Voter server-2b405c0f-7a8f-8c3d-909a-721aefefba5f 2b405c0f-7a8f-8c3d-909a-721aefefba5f}" error="failed to connect to peer: 2b405c0f-7a8f-8c3d-909a-721aefefba5f" + testing.go:149: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.104Z [ERROR] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: failed to make requestVote RPC: target="{Voter server-2b405c0f-7a8f-8c3d-909a-721aefefba5f 2b405c0f-7a8f-8c3d-909a-721aefefba5f}" error="failed to connect to peer: 2b405c0f-7a8f-8c3d-909a-721aefefba5f" +testLoggerAdapter verbose: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.104Z [ERROR] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: failed to make requestVote RPC: target="{Voter server-bdbd9fe5-b7e0-3828-169f-9446390443e2 bdbd9fe5-b7e0-3828-169f-9446390443e2}" error="failed to connect to peer: bdbd9fe5-b7e0-3828-169f-9446390443e2" + testing.go:149: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.104Z [ERROR] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: failed to make requestVote RPC: target="{Voter server-bdbd9fe5-b7e0-3828-169f-9446390443e2 bdbd9fe5-b7e0-3828-169f-9446390443e2}" error="failed to connect to peer: bdbd9fe5-b7e0-3828-169f-9446390443e2" +testLoggerAdapter verbose: server-bdbd9fe5-b7e0-3828-169f-9446390443e2: 2024-01-08T00:31:07.104Z [INFO] server-bdbd9fe5-b7e0-3828-169f-9446390443e2: compacting logs: from=1 to=90 + testing.go:149: server-bdbd9fe5-b7e0-3828-169f-9446390443e2: 2024-01-08T00:31:07.104Z [INFO] server-bdbd9fe5-b7e0-3828-169f-9446390443e2: compacting logs: from=1 to=90 +testLoggerAdapter verbose: server-bdbd9fe5-b7e0-3828-169f-9446390443e2: 2024-01-08T00:31:07.104Z [INFO] server-bdbd9fe5-b7e0-3828-169f-9446390443e2: snapshot complete up to: index=90 + testing.go:149: server-bdbd9fe5-b7e0-3828-169f-9446390443e2: 2024-01-08T00:31:07.104Z [INFO] server-bdbd9fe5-b7e0-3828-169f-9446390443e2: snapshot complete up to: index=90 +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.104Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: starting snapshot up to: index=102 + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.104Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: starting snapshot up to: index=102 + testing.go:153: 2024-01-08T00:31:07.104Z [INFO] creating new snapshot: path=/tmp/raft4239871505/snapshots/2-102-1704673867104.tmp +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.113Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: compacting logs: from=1 to=92 + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.113Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: compacting logs: from=1 to=92 +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.113Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: snapshot complete up to: index=102 + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.113Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: snapshot complete up to: index=102 raft_test.go:1383: [INFO] Finished apply without behind follower -testLoggerAdapter verbose: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.914Z [ERROR] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: failed to get log: index=2 error="log not found" - testing.go:149: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.914Z [ERROR] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: failed to get log: index=2 error="log not found" - testing.go:153: 2025-02-09T06:52:31.914Z [INFO] creating new snapshot: path=/tmp/raft1873730546/snapshots/2-102-1739083951914.tmp -testLoggerAdapter verbose: server-16a7755d-3460-4753-8090-799f13ced7da: 2025-02-09T06:52:31.917Z [INFO] server-16a7755d-3460-4753-8090-799f13ced7da: copied to local snapshot: bytes=693 - testing.go:149: server-16a7755d-3460-4753-8090-799f13ced7da: 2025-02-09T06:52:31.917Z [INFO] server-16a7755d-3460-4753-8090-799f13ced7da: copied to local snapshot: bytes=693 -testLoggerAdapter verbose: server-16a7755d-3460-4753-8090-799f13ced7da: 2025-02-09T06:52:31.917Z [INFO] server-16a7755d-3460-4753-8090-799f13ced7da: Installed remote snapshot - testing.go:149: server-16a7755d-3460-4753-8090-799f13ced7da: 2025-02-09T06:52:31.917Z [INFO] server-16a7755d-3460-4753-8090-799f13ced7da: Installed remote snapshot -testLoggerAdapter verbose: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.917Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: pipelining replication: peer="{Voter server-16a7755d-3460-4753-8090-799f13ced7da 16a7755d-3460-4753-8090-799f13ced7da}" - testing.go:149: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.917Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: pipelining replication: peer="{Voter server-16a7755d-3460-4753-8090-799f13ced7da 16a7755d-3460-4753-8090-799f13ced7da}" -testLoggerAdapter verbose: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.923Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: aborting pipeline replication: peer="{Voter server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54 0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54}" - testing.go:149: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.923Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: aborting pipeline replication: peer="{Voter server-0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54 0f3aeac5-6e53-cb1e-55ef-d477f4dbcc54}" -testLoggerAdapter verbose: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.923Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: aborting pipeline replication: peer="{Voter server-16a7755d-3460-4753-8090-799f13ced7da 16a7755d-3460-4753-8090-799f13ced7da}" - testing.go:149: server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: 2025-02-09T06:52:31.923Z [INFO] server-113f0681-7acf-736a-2a6e-1cfb35ac2cce: aborting pipeline replication: peer="{Voter server-16a7755d-3460-4753-8090-799f13ced7da 16a7755d-3460-4753-8090-799f13ced7da}" ---- PASS: TestRaft_SendSnapshotAndLogsFollower (0.29s) +testLoggerAdapter verbose: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.169Z [WARN] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: Election timeout reached, restarting election + testing.go:149: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.169Z [WARN] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: Election timeout reached, restarting election +testLoggerAdapter verbose: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.169Z [INFO] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: entering candidate state: node="Node at d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e [Candidate]" term=5 + testing.go:149: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.169Z [INFO] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: entering candidate state: node="Node at d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e [Candidate]" term=5 +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.169Z [WARN] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: rejecting vote request since we have a leader: from=d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e leader=2b405c0f-7a8f-8c3d-909a-721aefefba5f + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.169Z [WARN] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: rejecting vote request since we have a leader: from=d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e leader=2b405c0f-7a8f-8c3d-909a-721aefefba5f +testLoggerAdapter verbose: server-bdbd9fe5-b7e0-3828-169f-9446390443e2: 2024-01-08T00:31:07.169Z [WARN] server-bdbd9fe5-b7e0-3828-169f-9446390443e2: rejecting vote request since we have a leader: from=d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e leader=2b405c0f-7a8f-8c3d-909a-721aefefba5f + testing.go:149: server-bdbd9fe5-b7e0-3828-169f-9446390443e2: 2024-01-08T00:31:07.169Z [WARN] server-bdbd9fe5-b7e0-3828-169f-9446390443e2: rejecting vote request since we have a leader: from=d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e leader=2b405c0f-7a8f-8c3d-909a-721aefefba5f +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.170Z [ERROR] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: failed to get log: index=2 error="log not found" + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.170Z [ERROR] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: failed to get log: index=2 error="log not found" +testLoggerAdapter verbose: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.171Z [INFO] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: ignoring installSnapshot request with older term than current term: request-term=2 current-term=5 + testing.go:149: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.171Z [INFO] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: ignoring installSnapshot request with older term than current term: request-term=2 current-term=5 +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.171Z [ERROR] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: peer has newer term, stopping replication: peer="{Voter server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e}" + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.171Z [ERROR] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: peer has newer term, stopping replication: peer="{Voter server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e}" +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.171Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: entering follower state: follower="Node at 2b405c0f-7a8f-8c3d-909a-721aefefba5f [Follower]" leader= + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.171Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: entering follower state: follower="Node at 2b405c0f-7a8f-8c3d-909a-721aefefba5f [Follower]" leader= +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.171Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: aborting pipeline replication: peer="{Voter server-bdbd9fe5-b7e0-3828-169f-9446390443e2 bdbd9fe5-b7e0-3828-169f-9446390443e2}" + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.171Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: aborting pipeline replication: peer="{Voter server-bdbd9fe5-b7e0-3828-169f-9446390443e2 bdbd9fe5-b7e0-3828-169f-9446390443e2}" +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.234Z [WARN] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.234Z [WARN] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.234Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: entering candidate state: node="Node at 2b405c0f-7a8f-8c3d-909a-721aefefba5f [Candidate]" term=3 + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.234Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: entering candidate state: node="Node at 2b405c0f-7a8f-8c3d-909a-721aefefba5f [Candidate]" term=3 +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.234Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: election won: tally=2 + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.234Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: election won: tally=2 +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.234Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: entering leader state: leader="Node at 2b405c0f-7a8f-8c3d-909a-721aefefba5f [Leader]" + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.234Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: entering leader state: leader="Node at 2b405c0f-7a8f-8c3d-909a-721aefefba5f [Leader]" +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.234Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: added peer, starting replication: peer=server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.234Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: added peer, starting replication: peer=server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.235Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: added peer, starting replication: peer=server-bdbd9fe5-b7e0-3828-169f-9446390443e2 + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.235Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: added peer, starting replication: peer=server-bdbd9fe5-b7e0-3828-169f-9446390443e2 +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.235Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: pipelining replication: peer="{Voter server-bdbd9fe5-b7e0-3828-169f-9446390443e2 bdbd9fe5-b7e0-3828-169f-9446390443e2}" + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.235Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: pipelining replication: peer="{Voter server-bdbd9fe5-b7e0-3828-169f-9446390443e2 bdbd9fe5-b7e0-3828-169f-9446390443e2}" +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.235Z [ERROR] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: peer has newer term, stopping replication: peer="{Voter server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e}" + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.235Z [ERROR] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: peer has newer term, stopping replication: peer="{Voter server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e}" +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.235Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: entering follower state: follower="Node at 2b405c0f-7a8f-8c3d-909a-721aefefba5f [Follower]" leader= + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.235Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: entering follower state: follower="Node at 2b405c0f-7a8f-8c3d-909a-721aefefba5f [Follower]" leader= +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.235Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: aborting pipeline replication: peer="{Voter server-bdbd9fe5-b7e0-3828-169f-9446390443e2 bdbd9fe5-b7e0-3828-169f-9446390443e2}" + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.235Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: aborting pipeline replication: peer="{Voter server-bdbd9fe5-b7e0-3828-169f-9446390443e2 bdbd9fe5-b7e0-3828-169f-9446390443e2}" +testLoggerAdapter verbose: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.242Z [WARN] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: Election timeout reached, restarting election + testing.go:149: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.242Z [WARN] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: Election timeout reached, restarting election +testLoggerAdapter verbose: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.242Z [INFO] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: entering candidate state: node="Node at d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e [Candidate]" term=6 + testing.go:149: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.242Z [INFO] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: entering candidate state: node="Node at d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e [Candidate]" term=6 +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.242Z [WARN] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: rejecting vote request since our last term is greater: candidate=d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e last-term=3 last-candidate-term=2 + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.242Z [WARN] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: rejecting vote request since our last term is greater: candidate=d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e last-term=3 last-candidate-term=2 +testLoggerAdapter verbose: server-bdbd9fe5-b7e0-3828-169f-9446390443e2: 2024-01-08T00:31:07.242Z [WARN] server-bdbd9fe5-b7e0-3828-169f-9446390443e2: rejecting vote request since we have a leader: from=d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e leader=2b405c0f-7a8f-8c3d-909a-721aefefba5f + testing.go:149: server-bdbd9fe5-b7e0-3828-169f-9446390443e2: 2024-01-08T00:31:07.242Z [WARN] server-bdbd9fe5-b7e0-3828-169f-9446390443e2: rejecting vote request since we have a leader: from=d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e leader=2b405c0f-7a8f-8c3d-909a-721aefefba5f +testLoggerAdapter verbose: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.316Z [WARN] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: Election timeout reached, restarting election + testing.go:149: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.316Z [WARN] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: Election timeout reached, restarting election +testLoggerAdapter verbose: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.316Z [INFO] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: entering candidate state: node="Node at d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e [Candidate]" term=7 + testing.go:149: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.316Z [INFO] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: entering candidate state: node="Node at d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e [Candidate]" term=7 +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.316Z [WARN] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: rejecting vote request since our last term is greater: candidate=d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e last-term=3 last-candidate-term=2 + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.316Z [WARN] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: rejecting vote request since our last term is greater: candidate=d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e last-term=3 last-candidate-term=2 +testLoggerAdapter verbose: server-bdbd9fe5-b7e0-3828-169f-9446390443e2: 2024-01-08T00:31:07.316Z [WARN] server-bdbd9fe5-b7e0-3828-169f-9446390443e2: rejecting vote request since we have a leader: from=d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e leader=2b405c0f-7a8f-8c3d-909a-721aefefba5f + testing.go:149: server-bdbd9fe5-b7e0-3828-169f-9446390443e2: 2024-01-08T00:31:07.316Z [WARN] server-bdbd9fe5-b7e0-3828-169f-9446390443e2: rejecting vote request since we have a leader: from=d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e leader=2b405c0f-7a8f-8c3d-909a-721aefefba5f +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.331Z [WARN] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.331Z [WARN] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.331Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: entering candidate state: node="Node at 2b405c0f-7a8f-8c3d-909a-721aefefba5f [Candidate]" term=8 + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.331Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: entering candidate state: node="Node at 2b405c0f-7a8f-8c3d-909a-721aefefba5f [Candidate]" term=8 +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.331Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: election won: tally=2 + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.331Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: election won: tally=2 +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.331Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: entering leader state: leader="Node at 2b405c0f-7a8f-8c3d-909a-721aefefba5f [Leader]" + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.331Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: entering leader state: leader="Node at 2b405c0f-7a8f-8c3d-909a-721aefefba5f [Leader]" +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.331Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: added peer, starting replication: peer=server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.331Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: added peer, starting replication: peer=server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.331Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: added peer, starting replication: peer=server-bdbd9fe5-b7e0-3828-169f-9446390443e2 + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.331Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: added peer, starting replication: peer=server-bdbd9fe5-b7e0-3828-169f-9446390443e2 +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.331Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: pipelining replication: peer="{Voter server-bdbd9fe5-b7e0-3828-169f-9446390443e2 bdbd9fe5-b7e0-3828-169f-9446390443e2}" + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.331Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: pipelining replication: peer="{Voter server-bdbd9fe5-b7e0-3828-169f-9446390443e2 bdbd9fe5-b7e0-3828-169f-9446390443e2}" +testLoggerAdapter verbose: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.331Z [INFO] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: entering follower state: follower="Node at d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e [Follower]" leader= + testing.go:149: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.331Z [INFO] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: entering follower state: follower="Node at d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e [Follower]" leader= +testLoggerAdapter verbose: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.331Z [WARN] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: failed to get previous log: previous-index=203 last-index=2 error="log not found" + testing.go:149: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.331Z [WARN] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: failed to get previous log: previous-index=203 last-index=2 error="log not found" +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.331Z [WARN] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: appendEntries rejected, sending older logs: peer="{Voter server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e}" next=3 + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.331Z [WARN] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: appendEntries rejected, sending older logs: peer="{Voter server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e}" next=3 +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.331Z [ERROR] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: failed to get log: index=2 error="log not found" + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.331Z [ERROR] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: failed to get log: index=2 error="log not found" + testing.go:153: 2024-01-08T00:31:07.332Z [INFO] creating new snapshot: path=/tmp/raft3284140110/snapshots/2-102-1704673867332.tmp +testLoggerAdapter verbose: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.341Z [INFO] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: copied to local snapshot: bytes=693 + testing.go:149: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.341Z [INFO] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: copied to local snapshot: bytes=693 +testLoggerAdapter verbose: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.342Z [INFO] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: Installed remote snapshot + testing.go:149: server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: 2024-01-08T00:31:07.342Z [INFO] server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e: Installed remote snapshot +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.342Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: pipelining replication: peer="{Voter server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e}" + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.342Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: pipelining replication: peer="{Voter server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e}" +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.343Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: aborting pipeline replication: peer="{Voter server-bdbd9fe5-b7e0-3828-169f-9446390443e2 bdbd9fe5-b7e0-3828-169f-9446390443e2}" + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.343Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: aborting pipeline replication: peer="{Voter server-bdbd9fe5-b7e0-3828-169f-9446390443e2 bdbd9fe5-b7e0-3828-169f-9446390443e2}" +testLoggerAdapter verbose: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.343Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: aborting pipeline replication: peer="{Voter server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e}" + testing.go:149: server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: 2024-01-08T00:31:07.343Z [INFO] server-2b405c0f-7a8f-8c3d-909a-721aefefba5f: aborting pipeline replication: peer="{Voter server-d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e d65b2d96-cf22-f00b-f1c2-dd7f8e5daf2e}" +--- PASS: TestRaft_SendSnapshotAndLogsFollower (0.63s) === RUN TestRaft_ReJoinFollower -testLoggerAdapter verbose: server-60273cff-8d8e-d83e-9886-d08cd479f1e7: 2025-02-09T06:52:31.924Z [INFO] server-60273cff-8d8e-d83e-9886-d08cd479f1e7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-60273cff-8d8e-d83e-9886-d08cd479f1e7 Address:60273cff-8d8e-d83e-9886-d08cd479f1e7} {Suffrage:Voter ID:server-8de14cca-22ff-e432-6df6-16f277011295 Address:8de14cca-22ff-e432-6df6-16f277011295} {Suffrage:Voter ID:server-7db1d716-1992-8bf9-f1c0-322e0890f7f2 Address:7db1d716-1992-8bf9-f1c0-322e0890f7f2}]" - testing.go:149: server-60273cff-8d8e-d83e-9886-d08cd479f1e7: 2025-02-09T06:52:31.924Z [INFO] server-60273cff-8d8e-d83e-9886-d08cd479f1e7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-60273cff-8d8e-d83e-9886-d08cd479f1e7 Address:60273cff-8d8e-d83e-9886-d08cd479f1e7} {Suffrage:Voter ID:server-8de14cca-22ff-e432-6df6-16f277011295 Address:8de14cca-22ff-e432-6df6-16f277011295} {Suffrage:Voter ID:server-7db1d716-1992-8bf9-f1c0-322e0890f7f2 Address:7db1d716-1992-8bf9-f1c0-322e0890f7f2}]" -testLoggerAdapter verbose: server-60273cff-8d8e-d83e-9886-d08cd479f1e7: 2025-02-09T06:52:31.925Z [INFO] server-60273cff-8d8e-d83e-9886-d08cd479f1e7: entering follower state: follower="Node at 60273cff-8d8e-d83e-9886-d08cd479f1e7 [Follower]" leader= - testing.go:149: server-60273cff-8d8e-d83e-9886-d08cd479f1e7: 2025-02-09T06:52:31.925Z [INFO] server-60273cff-8d8e-d83e-9886-d08cd479f1e7: entering follower state: follower="Node at 60273cff-8d8e-d83e-9886-d08cd479f1e7 [Follower]" leader= -testLoggerAdapter verbose: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:31.925Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-60273cff-8d8e-d83e-9886-d08cd479f1e7 Address:60273cff-8d8e-d83e-9886-d08cd479f1e7} {Suffrage:Voter ID:server-8de14cca-22ff-e432-6df6-16f277011295 Address:8de14cca-22ff-e432-6df6-16f277011295} {Suffrage:Voter ID:server-7db1d716-1992-8bf9-f1c0-322e0890f7f2 Address:7db1d716-1992-8bf9-f1c0-322e0890f7f2}]" - testing.go:149: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:31.925Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-60273cff-8d8e-d83e-9886-d08cd479f1e7 Address:60273cff-8d8e-d83e-9886-d08cd479f1e7} {Suffrage:Voter ID:server-8de14cca-22ff-e432-6df6-16f277011295 Address:8de14cca-22ff-e432-6df6-16f277011295} {Suffrage:Voter ID:server-7db1d716-1992-8bf9-f1c0-322e0890f7f2 Address:7db1d716-1992-8bf9-f1c0-322e0890f7f2}]" -testLoggerAdapter verbose: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:31.925Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: entering follower state: follower="Node at 8de14cca-22ff-e432-6df6-16f277011295 [Follower]" leader= - testing.go:149: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:31.925Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: entering follower state: follower="Node at 8de14cca-22ff-e432-6df6-16f277011295 [Follower]" leader= -testLoggerAdapter verbose: server-7db1d716-1992-8bf9-f1c0-322e0890f7f2: 2025-02-09T06:52:31.925Z [INFO] server-7db1d716-1992-8bf9-f1c0-322e0890f7f2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-60273cff-8d8e-d83e-9886-d08cd479f1e7 Address:60273cff-8d8e-d83e-9886-d08cd479f1e7} {Suffrage:Voter ID:server-8de14cca-22ff-e432-6df6-16f277011295 Address:8de14cca-22ff-e432-6df6-16f277011295} {Suffrage:Voter ID:server-7db1d716-1992-8bf9-f1c0-322e0890f7f2 Address:7db1d716-1992-8bf9-f1c0-322e0890f7f2}]" - testing.go:149: server-7db1d716-1992-8bf9-f1c0-322e0890f7f2: 2025-02-09T06:52:31.925Z [INFO] server-7db1d716-1992-8bf9-f1c0-322e0890f7f2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-60273cff-8d8e-d83e-9886-d08cd479f1e7 Address:60273cff-8d8e-d83e-9886-d08cd479f1e7} {Suffrage:Voter ID:server-8de14cca-22ff-e432-6df6-16f277011295 Address:8de14cca-22ff-e432-6df6-16f277011295} {Suffrage:Voter ID:server-7db1d716-1992-8bf9-f1c0-322e0890f7f2 Address:7db1d716-1992-8bf9-f1c0-322e0890f7f2}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:31.925Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:31.925Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-7db1d716-1992-8bf9-f1c0-322e0890f7f2: 2025-02-09T06:52:31.925Z [INFO] server-7db1d716-1992-8bf9-f1c0-322e0890f7f2: entering follower state: follower="Node at 7db1d716-1992-8bf9-f1c0-322e0890f7f2 [Follower]" leader= - testing.go:149: server-7db1d716-1992-8bf9-f1c0-322e0890f7f2: 2025-02-09T06:52:31.925Z [INFO] server-7db1d716-1992-8bf9-f1c0-322e0890f7f2: entering follower state: follower="Node at 7db1d716-1992-8bf9-f1c0-322e0890f7f2 [Follower]" leader= -testLoggerAdapter verbose: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.013Z [WARN] server-8de14cca-22ff-e432-6df6-16f277011295: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.013Z [WARN] server-8de14cca-22ff-e432-6df6-16f277011295: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.013Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: entering candidate state: node="Node at 8de14cca-22ff-e432-6df6-16f277011295 [Candidate]" term=2 - testing.go:149: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.013Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: entering candidate state: node="Node at 8de14cca-22ff-e432-6df6-16f277011295 [Candidate]" term=2 -testLoggerAdapter verbose: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.013Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: election won: tally=2 - testing.go:149: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.013Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: election won: tally=2 -testLoggerAdapter verbose: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.013Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: entering leader state: leader="Node at 8de14cca-22ff-e432-6df6-16f277011295 [Leader]" - testing.go:149: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.013Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: entering leader state: leader="Node at 8de14cca-22ff-e432-6df6-16f277011295 [Leader]" -testLoggerAdapter verbose: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.013Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: added peer, starting replication: peer=server-60273cff-8d8e-d83e-9886-d08cd479f1e7 - testing.go:149: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.013Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: added peer, starting replication: peer=server-60273cff-8d8e-d83e-9886-d08cd479f1e7 -testLoggerAdapter verbose: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.013Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: added peer, starting replication: peer=server-7db1d716-1992-8bf9-f1c0-322e0890f7f2 - testing.go:149: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.013Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: added peer, starting replication: peer=server-7db1d716-1992-8bf9-f1c0-322e0890f7f2 -testLoggerAdapter verbose: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.013Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: pipelining replication: peer="{Voter server-60273cff-8d8e-d83e-9886-d08cd479f1e7 60273cff-8d8e-d83e-9886-d08cd479f1e7}" - testing.go:149: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.013Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: pipelining replication: peer="{Voter server-60273cff-8d8e-d83e-9886-d08cd479f1e7 60273cff-8d8e-d83e-9886-d08cd479f1e7}" -testLoggerAdapter verbose: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.013Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: pipelining replication: peer="{Voter server-7db1d716-1992-8bf9-f1c0-322e0890f7f2 7db1d716-1992-8bf9-f1c0-322e0890f7f2}" - testing.go:149: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.013Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: pipelining replication: peer="{Voter server-7db1d716-1992-8bf9-f1c0-322e0890f7f2 7db1d716-1992-8bf9-f1c0-322e0890f7f2}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:32.118Z [INFO] stable state for Leader reached at 2025-02-09 06:52:32.013160956 +0000 UTC m=+12.991969741 (1 nodes), 87.761598ms from start of poll, 88.340621ms from cluster start. Timeout at 2025-02-09 06:52:32.118886188 +0000 UTC m=+13.097694947, 105.725206ms after stability - testing.go:149: cluster: 2025-02-09T06:52:32.118Z [INFO] stable state for Leader reached at 2025-02-09 06:52:32.013160956 +0000 UTC m=+12.991969741 (1 nodes), 87.761598ms from start of poll, 88.340621ms from cluster start. Timeout at 2025-02-09 06:52:32.118886188 +0000 UTC m=+13.097694947, 105.725206ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:32.124Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T06:52:32.124Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:32.229Z [INFO] stable state for Follower reached at 2025-02-09 06:52:32.124503987 +0000 UTC m=+13.103312765 (2 nodes), 699ns from start of poll, 199.683645ms from cluster start. Timeout at 2025-02-09 06:52:32.229749769 +0000 UTC m=+13.208558541, 105.245776ms after stability - testing.go:149: cluster: 2025-02-09T06:52:32.229Z [INFO] stable state for Follower reached at 2025-02-09 06:52:32.124503987 +0000 UTC m=+13.103312765 (2 nodes), 699ns from start of poll, 199.683645ms from cluster start. Timeout at 2025-02-09 06:52:32.229749769 +0000 UTC m=+13.208558541, 105.245776ms after stability -testLoggerAdapter verbose: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.229Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: updating configuration: command=RemoveServer server-id=server-60273cff-8d8e-d83e-9886-d08cd479f1e7 server-addr= servers="[{Suffrage:Voter ID:server-8de14cca-22ff-e432-6df6-16f277011295 Address:8de14cca-22ff-e432-6df6-16f277011295} {Suffrage:Voter ID:server-7db1d716-1992-8bf9-f1c0-322e0890f7f2 Address:7db1d716-1992-8bf9-f1c0-322e0890f7f2}]" - testing.go:149: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.229Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: updating configuration: command=RemoveServer server-id=server-60273cff-8d8e-d83e-9886-d08cd479f1e7 server-addr= servers="[{Suffrage:Voter ID:server-8de14cca-22ff-e432-6df6-16f277011295 Address:8de14cca-22ff-e432-6df6-16f277011295} {Suffrage:Voter ID:server-7db1d716-1992-8bf9-f1c0-322e0890f7f2 Address:7db1d716-1992-8bf9-f1c0-322e0890f7f2}]" -testLoggerAdapter verbose: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.229Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: removed peer, stopping replication: peer=server-60273cff-8d8e-d83e-9886-d08cd479f1e7 last-index=3 - testing.go:149: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.229Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: removed peer, stopping replication: peer=server-60273cff-8d8e-d83e-9886-d08cd479f1e7 last-index=3 -testLoggerAdapter verbose: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.229Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: aborting pipeline replication: peer="{Voter server-60273cff-8d8e-d83e-9886-d08cd479f1e7 60273cff-8d8e-d83e-9886-d08cd479f1e7}" - testing.go:149: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.229Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: aborting pipeline replication: peer="{Voter server-60273cff-8d8e-d83e-9886-d08cd479f1e7 60273cff-8d8e-d83e-9886-d08cd479f1e7}" -testLoggerAdapter verbose: server-60273cff-8d8e-d83e-9886-d08cd479f1e7: 2025-02-09T06:52:32.335Z [WARN] server-60273cff-8d8e-d83e-9886-d08cd479f1e7: heartbeat timeout reached, starting election: last-leader=8de14cca-22ff-e432-6df6-16f277011295 - testing.go:149: server-60273cff-8d8e-d83e-9886-d08cd479f1e7: 2025-02-09T06:52:32.335Z [WARN] server-60273cff-8d8e-d83e-9886-d08cd479f1e7: heartbeat timeout reached, starting election: last-leader=8de14cca-22ff-e432-6df6-16f277011295 -testLoggerAdapter verbose: server-60273cff-8d8e-d83e-9886-d08cd479f1e7: 2025-02-09T06:52:32.335Z [INFO] server-60273cff-8d8e-d83e-9886-d08cd479f1e7: entering candidate state: node="Node at 60273cff-8d8e-d83e-9886-d08cd479f1e7 [Candidate]" term=3 - testing.go:149: server-60273cff-8d8e-d83e-9886-d08cd479f1e7: 2025-02-09T06:52:32.335Z [INFO] server-60273cff-8d8e-d83e-9886-d08cd479f1e7: entering candidate state: node="Node at 60273cff-8d8e-d83e-9886-d08cd479f1e7 [Candidate]" term=3 -testLoggerAdapter verbose: server-7db1d716-1992-8bf9-f1c0-322e0890f7f2: 2025-02-09T06:52:32.335Z [WARN] server-7db1d716-1992-8bf9-f1c0-322e0890f7f2: rejecting vote request since we have a leader: from=60273cff-8d8e-d83e-9886-d08cd479f1e7 leader=8de14cca-22ff-e432-6df6-16f277011295 -testLoggerAdapter verbose: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.335Z [WARN] server-8de14cca-22ff-e432-6df6-16f277011295: rejecting vote request since we have a leader: from=60273cff-8d8e-d83e-9886-d08cd479f1e7 leader=8de14cca-22ff-e432-6df6-16f277011295 - testing.go:149: server-7db1d716-1992-8bf9-f1c0-322e0890f7f2: 2025-02-09T06:52:32.335Z [WARN] server-7db1d716-1992-8bf9-f1c0-322e0890f7f2: rejecting vote request since we have a leader: from=60273cff-8d8e-d83e-9886-d08cd479f1e7 leader=8de14cca-22ff-e432-6df6-16f277011295 - testing.go:149: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.335Z [WARN] server-8de14cca-22ff-e432-6df6-16f277011295: rejecting vote request since we have a leader: from=60273cff-8d8e-d83e-9886-d08cd479f1e7 leader=8de14cca-22ff-e432-6df6-16f277011295 -testLoggerAdapter verbose: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.335Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: updating configuration: command=AddStaging server-id=server-60273cff-8d8e-d83e-9886-d08cd479f1e7 server-addr=60273cff-8d8e-d83e-9886-d08cd479f1e7 servers="[{Suffrage:Voter ID:server-8de14cca-22ff-e432-6df6-16f277011295 Address:8de14cca-22ff-e432-6df6-16f277011295} {Suffrage:Voter ID:server-7db1d716-1992-8bf9-f1c0-322e0890f7f2 Address:7db1d716-1992-8bf9-f1c0-322e0890f7f2} {Suffrage:Voter ID:server-60273cff-8d8e-d83e-9886-d08cd479f1e7 Address:60273cff-8d8e-d83e-9886-d08cd479f1e7}]" - testing.go:149: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.335Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: updating configuration: command=AddStaging server-id=server-60273cff-8d8e-d83e-9886-d08cd479f1e7 server-addr=60273cff-8d8e-d83e-9886-d08cd479f1e7 servers="[{Suffrage:Voter ID:server-8de14cca-22ff-e432-6df6-16f277011295 Address:8de14cca-22ff-e432-6df6-16f277011295} {Suffrage:Voter ID:server-7db1d716-1992-8bf9-f1c0-322e0890f7f2 Address:7db1d716-1992-8bf9-f1c0-322e0890f7f2} {Suffrage:Voter ID:server-60273cff-8d8e-d83e-9886-d08cd479f1e7 Address:60273cff-8d8e-d83e-9886-d08cd479f1e7}]" -testLoggerAdapter verbose: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.335Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: added peer, starting replication: peer=server-60273cff-8d8e-d83e-9886-d08cd479f1e7 - testing.go:149: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.335Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: added peer, starting replication: peer=server-60273cff-8d8e-d83e-9886-d08cd479f1e7 -testLoggerAdapter verbose: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.335Z [ERROR] server-8de14cca-22ff-e432-6df6-16f277011295: peer has newer term, stopping replication: peer="{Voter server-60273cff-8d8e-d83e-9886-d08cd479f1e7 60273cff-8d8e-d83e-9886-d08cd479f1e7}" - testing.go:149: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.335Z [ERROR] server-8de14cca-22ff-e432-6df6-16f277011295: peer has newer term, stopping replication: peer="{Voter server-60273cff-8d8e-d83e-9886-d08cd479f1e7 60273cff-8d8e-d83e-9886-d08cd479f1e7}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:32.335Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.335Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: entering follower state: follower="Node at 8de14cca-22ff-e432-6df6-16f277011295 [Follower]" leader= - testing.go:149: cluster: 2025-02-09T06:52:32.335Z [INFO] starting stability test: raft-state=Leader - testing.go:149: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.335Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: entering follower state: follower="Node at 8de14cca-22ff-e432-6df6-16f277011295 [Follower]" leader= -testLoggerAdapter verbose: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.335Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: aborting pipeline replication: peer="{Voter server-7db1d716-1992-8bf9-f1c0-322e0890f7f2 7db1d716-1992-8bf9-f1c0-322e0890f7f2}" - testing.go:149: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.335Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: aborting pipeline replication: peer="{Voter server-7db1d716-1992-8bf9-f1c0-322e0890f7f2 7db1d716-1992-8bf9-f1c0-322e0890f7f2}" -testLoggerAdapter verbose: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.425Z [WARN] server-8de14cca-22ff-e432-6df6-16f277011295: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.425Z [WARN] server-8de14cca-22ff-e432-6df6-16f277011295: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.426Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: entering candidate state: node="Node at 8de14cca-22ff-e432-6df6-16f277011295 [Candidate]" term=3 - testing.go:149: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.426Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: entering candidate state: node="Node at 8de14cca-22ff-e432-6df6-16f277011295 [Candidate]" term=3 -testLoggerAdapter verbose: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.426Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: election won: tally=2 - testing.go:149: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.426Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: election won: tally=2 -testLoggerAdapter verbose: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.426Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: entering leader state: leader="Node at 8de14cca-22ff-e432-6df6-16f277011295 [Leader]" - testing.go:149: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.426Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: entering leader state: leader="Node at 8de14cca-22ff-e432-6df6-16f277011295 [Leader]" -testLoggerAdapter verbose: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.426Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: added peer, starting replication: peer=server-7db1d716-1992-8bf9-f1c0-322e0890f7f2 - testing.go:149: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.426Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: added peer, starting replication: peer=server-7db1d716-1992-8bf9-f1c0-322e0890f7f2 -testLoggerAdapter verbose: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.426Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: added peer, starting replication: peer=server-60273cff-8d8e-d83e-9886-d08cd479f1e7 - testing.go:149: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.426Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: added peer, starting replication: peer=server-60273cff-8d8e-d83e-9886-d08cd479f1e7 -testLoggerAdapter verbose: server-60273cff-8d8e-d83e-9886-d08cd479f1e7: 2025-02-09T06:52:32.426Z [WARN] server-60273cff-8d8e-d83e-9886-d08cd479f1e7: failed to get previous log: previous-index=4 last-index=3 error="log not found" - testing.go:149: server-60273cff-8d8e-d83e-9886-d08cd479f1e7: 2025-02-09T06:52:32.426Z [WARN] server-60273cff-8d8e-d83e-9886-d08cd479f1e7: failed to get previous log: previous-index=4 last-index=3 error="log not found" -testLoggerAdapter verbose: server-60273cff-8d8e-d83e-9886-d08cd479f1e7: 2025-02-09T06:52:32.426Z [INFO] server-60273cff-8d8e-d83e-9886-d08cd479f1e7: entering follower state: follower="Node at 60273cff-8d8e-d83e-9886-d08cd479f1e7 [Follower]" leader=8de14cca-22ff-e432-6df6-16f277011295 - testing.go:149: server-60273cff-8d8e-d83e-9886-d08cd479f1e7: 2025-02-09T06:52:32.426Z [INFO] server-60273cff-8d8e-d83e-9886-d08cd479f1e7: entering follower state: follower="Node at 60273cff-8d8e-d83e-9886-d08cd479f1e7 [Follower]" leader=8de14cca-22ff-e432-6df6-16f277011295 -testLoggerAdapter verbose: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.426Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: pipelining replication: peer="{Voter server-7db1d716-1992-8bf9-f1c0-322e0890f7f2 7db1d716-1992-8bf9-f1c0-322e0890f7f2}" - testing.go:149: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.426Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: pipelining replication: peer="{Voter server-7db1d716-1992-8bf9-f1c0-322e0890f7f2 7db1d716-1992-8bf9-f1c0-322e0890f7f2}" -testLoggerAdapter verbose: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.426Z [WARN] server-8de14cca-22ff-e432-6df6-16f277011295: appendEntries rejected, sending older logs: peer="{Voter server-60273cff-8d8e-d83e-9886-d08cd479f1e7 60273cff-8d8e-d83e-9886-d08cd479f1e7}" next=4 - testing.go:149: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.426Z [WARN] server-8de14cca-22ff-e432-6df6-16f277011295: appendEntries rejected, sending older logs: peer="{Voter server-60273cff-8d8e-d83e-9886-d08cd479f1e7 60273cff-8d8e-d83e-9886-d08cd479f1e7}" next=4 -testLoggerAdapter verbose: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.426Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: pipelining replication: peer="{Voter server-60273cff-8d8e-d83e-9886-d08cd479f1e7 60273cff-8d8e-d83e-9886-d08cd479f1e7}" - testing.go:149: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.426Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: pipelining replication: peer="{Voter server-60273cff-8d8e-d83e-9886-d08cd479f1e7 60273cff-8d8e-d83e-9886-d08cd479f1e7}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:32.532Z [INFO] stable state for Leader reached at 2025-02-09 06:52:32.426209818 +0000 UTC m=+13.405018581 (1 nodes), 90.227403ms from start of poll, 501.389461ms from cluster start. Timeout at 2025-02-09 06:52:32.532046595 +0000 UTC m=+13.510855353, 105.836772ms after stability - testing.go:149: cluster: 2025-02-09T06:52:32.532Z [INFO] stable state for Leader reached at 2025-02-09 06:52:32.426209818 +0000 UTC m=+13.405018581 (1 nodes), 90.227403ms from start of poll, 501.389461ms from cluster start. Timeout at 2025-02-09 06:52:32.532046595 +0000 UTC m=+13.510855353, 105.836772ms after stability -testLoggerAdapter verbose: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.532Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: aborting pipeline replication: peer="{Voter server-60273cff-8d8e-d83e-9886-d08cd479f1e7 60273cff-8d8e-d83e-9886-d08cd479f1e7}" - testing.go:149: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.532Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: aborting pipeline replication: peer="{Voter server-60273cff-8d8e-d83e-9886-d08cd479f1e7 60273cff-8d8e-d83e-9886-d08cd479f1e7}" -testLoggerAdapter verbose: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.532Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: aborting pipeline replication: peer="{Voter server-7db1d716-1992-8bf9-f1c0-322e0890f7f2 7db1d716-1992-8bf9-f1c0-322e0890f7f2}" - testing.go:149: server-8de14cca-22ff-e432-6df6-16f277011295: 2025-02-09T06:52:32.532Z [INFO] server-8de14cca-22ff-e432-6df6-16f277011295: aborting pipeline replication: peer="{Voter server-7db1d716-1992-8bf9-f1c0-322e0890f7f2 7db1d716-1992-8bf9-f1c0-322e0890f7f2}" ---- PASS: TestRaft_ReJoinFollower (0.61s) +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.345Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3f73f033-b0e0-fea2-6f6d-489334b4a58b Address:3f73f033-b0e0-fea2-6f6d-489334b4a58b} {Suffrage:Voter ID:server-18fcd28f-913b-5bc0-190a-6785fa6fd2db Address:18fcd28f-913b-5bc0-190a-6785fa6fd2db} {Suffrage:Voter ID:server-84780128-5416-124f-ec25-338c09410456 Address:84780128-5416-124f-ec25-338c09410456}]" + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.345Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3f73f033-b0e0-fea2-6f6d-489334b4a58b Address:3f73f033-b0e0-fea2-6f6d-489334b4a58b} {Suffrage:Voter ID:server-18fcd28f-913b-5bc0-190a-6785fa6fd2db Address:18fcd28f-913b-5bc0-190a-6785fa6fd2db} {Suffrage:Voter ID:server-84780128-5416-124f-ec25-338c09410456 Address:84780128-5416-124f-ec25-338c09410456}]" +testLoggerAdapter verbose: server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: 2024-01-08T00:31:07.345Z [INFO] server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3f73f033-b0e0-fea2-6f6d-489334b4a58b Address:3f73f033-b0e0-fea2-6f6d-489334b4a58b} {Suffrage:Voter ID:server-18fcd28f-913b-5bc0-190a-6785fa6fd2db Address:18fcd28f-913b-5bc0-190a-6785fa6fd2db} {Suffrage:Voter ID:server-84780128-5416-124f-ec25-338c09410456 Address:84780128-5416-124f-ec25-338c09410456}]" + testing.go:149: server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: 2024-01-08T00:31:07.345Z [INFO] server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3f73f033-b0e0-fea2-6f6d-489334b4a58b Address:3f73f033-b0e0-fea2-6f6d-489334b4a58b} {Suffrage:Voter ID:server-18fcd28f-913b-5bc0-190a-6785fa6fd2db Address:18fcd28f-913b-5bc0-190a-6785fa6fd2db} {Suffrage:Voter ID:server-84780128-5416-124f-ec25-338c09410456 Address:84780128-5416-124f-ec25-338c09410456}]" +testLoggerAdapter verbose: server-84780128-5416-124f-ec25-338c09410456: 2024-01-08T00:31:07.345Z [INFO] server-84780128-5416-124f-ec25-338c09410456: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3f73f033-b0e0-fea2-6f6d-489334b4a58b Address:3f73f033-b0e0-fea2-6f6d-489334b4a58b} {Suffrage:Voter ID:server-18fcd28f-913b-5bc0-190a-6785fa6fd2db Address:18fcd28f-913b-5bc0-190a-6785fa6fd2db} {Suffrage:Voter ID:server-84780128-5416-124f-ec25-338c09410456 Address:84780128-5416-124f-ec25-338c09410456}]" + testing.go:149: server-84780128-5416-124f-ec25-338c09410456: 2024-01-08T00:31:07.345Z [INFO] server-84780128-5416-124f-ec25-338c09410456: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3f73f033-b0e0-fea2-6f6d-489334b4a58b Address:3f73f033-b0e0-fea2-6f6d-489334b4a58b} {Suffrage:Voter ID:server-18fcd28f-913b-5bc0-190a-6785fa6fd2db Address:18fcd28f-913b-5bc0-190a-6785fa6fd2db} {Suffrage:Voter ID:server-84780128-5416-124f-ec25-338c09410456 Address:84780128-5416-124f-ec25-338c09410456}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:07.345Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:07.345Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.345Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: entering follower state: follower="Node at 3f73f033-b0e0-fea2-6f6d-489334b4a58b [Follower]" leader= + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.345Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: entering follower state: follower="Node at 3f73f033-b0e0-fea2-6f6d-489334b4a58b [Follower]" leader= +testLoggerAdapter verbose: server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: 2024-01-08T00:31:07.345Z [INFO] server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: entering follower state: follower="Node at 18fcd28f-913b-5bc0-190a-6785fa6fd2db [Follower]" leader= + testing.go:149: server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: 2024-01-08T00:31:07.345Z [INFO] server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: entering follower state: follower="Node at 18fcd28f-913b-5bc0-190a-6785fa6fd2db [Follower]" leader= +testLoggerAdapter verbose: server-84780128-5416-124f-ec25-338c09410456: 2024-01-08T00:31:07.345Z [INFO] server-84780128-5416-124f-ec25-338c09410456: entering follower state: follower="Node at 84780128-5416-124f-ec25-338c09410456 [Follower]" leader= + testing.go:149: server-84780128-5416-124f-ec25-338c09410456: 2024-01-08T00:31:07.345Z [INFO] server-84780128-5416-124f-ec25-338c09410456: entering follower state: follower="Node at 84780128-5416-124f-ec25-338c09410456 [Follower]" leader= +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.410Z [WARN] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.410Z [WARN] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.410Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: entering candidate state: node="Node at 3f73f033-b0e0-fea2-6f6d-489334b4a58b [Candidate]" term=2 + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.410Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: entering candidate state: node="Node at 3f73f033-b0e0-fea2-6f6d-489334b4a58b [Candidate]" term=2 +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.410Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: election won: tally=2 + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.410Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: election won: tally=2 +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.410Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: entering leader state: leader="Node at 3f73f033-b0e0-fea2-6f6d-489334b4a58b [Leader]" + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.410Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: entering leader state: leader="Node at 3f73f033-b0e0-fea2-6f6d-489334b4a58b [Leader]" +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.410Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: added peer, starting replication: peer=server-18fcd28f-913b-5bc0-190a-6785fa6fd2db + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.410Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: added peer, starting replication: peer=server-18fcd28f-913b-5bc0-190a-6785fa6fd2db +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.411Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: added peer, starting replication: peer=server-84780128-5416-124f-ec25-338c09410456 + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.411Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: added peer, starting replication: peer=server-84780128-5416-124f-ec25-338c09410456 +testLoggerAdapter verbose: server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: 2024-01-08T00:31:07.411Z [WARN] server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: 2024-01-08T00:31:07.411Z [WARN] server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: 2024-01-08T00:31:07.411Z [INFO] server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: entering candidate state: node="Node at 18fcd28f-913b-5bc0-190a-6785fa6fd2db [Candidate]" term=2 + testing.go:149: server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: 2024-01-08T00:31:07.411Z [INFO] server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: entering candidate state: node="Node at 18fcd28f-913b-5bc0-190a-6785fa6fd2db [Candidate]" term=2 +testLoggerAdapter verbose: server-84780128-5416-124f-ec25-338c09410456: 2024-01-08T00:31:07.411Z [INFO] server-84780128-5416-124f-ec25-338c09410456: duplicate requestVote for same term: term=2 + testing.go:149: server-84780128-5416-124f-ec25-338c09410456: 2024-01-08T00:31:07.411Z [INFO] server-84780128-5416-124f-ec25-338c09410456: duplicate requestVote for same term: term=2 +testLoggerAdapter verbose: server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: 2024-01-08T00:31:07.411Z [INFO] server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: duplicate requestVote for same term: term=2 + testing.go:149: server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: 2024-01-08T00:31:07.411Z [INFO] server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: duplicate requestVote for same term: term=2 +testLoggerAdapter verbose: server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: 2024-01-08T00:31:07.411Z [INFO] server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: entering follower state: follower="Node at 18fcd28f-913b-5bc0-190a-6785fa6fd2db [Follower]" leader=3f73f033-b0e0-fea2-6f6d-489334b4a58b + testing.go:149: server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: 2024-01-08T00:31:07.411Z [INFO] server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: entering follower state: follower="Node at 18fcd28f-913b-5bc0-190a-6785fa6fd2db [Follower]" leader=3f73f033-b0e0-fea2-6f6d-489334b4a58b +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.411Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: pipelining replication: peer="{Voter server-18fcd28f-913b-5bc0-190a-6785fa6fd2db 18fcd28f-913b-5bc0-190a-6785fa6fd2db}" + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.411Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: pipelining replication: peer="{Voter server-18fcd28f-913b-5bc0-190a-6785fa6fd2db 18fcd28f-913b-5bc0-190a-6785fa6fd2db}" +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.411Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: pipelining replication: peer="{Voter server-84780128-5416-124f-ec25-338c09410456 84780128-5416-124f-ec25-338c09410456}" + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.411Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: pipelining replication: peer="{Voter server-84780128-5416-124f-ec25-338c09410456 84780128-5416-124f-ec25-338c09410456}" +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.411Z [WARN] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: rejecting vote request since we have a leader: from=18fcd28f-913b-5bc0-190a-6785fa6fd2db leader=3f73f033-b0e0-fea2-6f6d-489334b4a58b + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.411Z [WARN] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: rejecting vote request since we have a leader: from=18fcd28f-913b-5bc0-190a-6785fa6fd2db leader=3f73f033-b0e0-fea2-6f6d-489334b4a58b +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:07.517Z [INFO] stable state for Leader reached at 2024-01-08 00:31:07.411660657 +0000 UTC m=+13.458549716 (1 nodes), 66.231366ms from start of poll, 66.65976ms from cluster start. Timeout at 2024-01-08 00:31:07.517311534 +0000 UTC m=+13.564200610, 105.650894ms after stability + testing.go:149: cluster: 2024-01-08T00:31:07.517Z [INFO] stable state for Leader reached at 2024-01-08 00:31:07.411660657 +0000 UTC m=+13.458549716 (1 nodes), 66.231366ms from start of poll, 66.65976ms from cluster start. Timeout at 2024-01-08 00:31:07.517311534 +0000 UTC m=+13.564200610, 105.650894ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:07.523Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T00:31:07.523Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:07.629Z [INFO] stable state for Follower reached at 2024-01-08 00:31:07.52317304 +0000 UTC m=+13.570062089 (2 nodes), 2.159µs from start of poll, 178.172133ms from cluster start. Timeout at 2024-01-08 00:31:07.629069058 +0000 UTC m=+13.675958141, 105.896052ms after stability + testing.go:149: cluster: 2024-01-08T00:31:07.629Z [INFO] stable state for Follower reached at 2024-01-08 00:31:07.52317304 +0000 UTC m=+13.570062089 (2 nodes), 2.159µs from start of poll, 178.172133ms from cluster start. Timeout at 2024-01-08 00:31:07.629069058 +0000 UTC m=+13.675958141, 105.896052ms after stability +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.629Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: updating configuration: command=RemoveServer server-id=server-18fcd28f-913b-5bc0-190a-6785fa6fd2db server-addr= servers="[{Suffrage:Voter ID:server-3f73f033-b0e0-fea2-6f6d-489334b4a58b Address:3f73f033-b0e0-fea2-6f6d-489334b4a58b} {Suffrage:Voter ID:server-84780128-5416-124f-ec25-338c09410456 Address:84780128-5416-124f-ec25-338c09410456}]" + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.629Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: updating configuration: command=RemoveServer server-id=server-18fcd28f-913b-5bc0-190a-6785fa6fd2db server-addr= servers="[{Suffrage:Voter ID:server-3f73f033-b0e0-fea2-6f6d-489334b4a58b Address:3f73f033-b0e0-fea2-6f6d-489334b4a58b} {Suffrage:Voter ID:server-84780128-5416-124f-ec25-338c09410456 Address:84780128-5416-124f-ec25-338c09410456}]" +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.629Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: removed peer, stopping replication: peer=server-18fcd28f-913b-5bc0-190a-6785fa6fd2db last-index=3 + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.629Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: removed peer, stopping replication: peer=server-18fcd28f-913b-5bc0-190a-6785fa6fd2db last-index=3 +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.629Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: aborting pipeline replication: peer="{Voter server-18fcd28f-913b-5bc0-190a-6785fa6fd2db 18fcd28f-913b-5bc0-190a-6785fa6fd2db}" + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.629Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: aborting pipeline replication: peer="{Voter server-18fcd28f-913b-5bc0-190a-6785fa6fd2db 18fcd28f-913b-5bc0-190a-6785fa6fd2db}" +testLoggerAdapter verbose: server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: 2024-01-08T00:31:07.725Z [WARN] server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: heartbeat timeout reached, starting election: last-leader=3f73f033-b0e0-fea2-6f6d-489334b4a58b + testing.go:149: server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: 2024-01-08T00:31:07.725Z [WARN] server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: heartbeat timeout reached, starting election: last-leader=3f73f033-b0e0-fea2-6f6d-489334b4a58b +testLoggerAdapter verbose: server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: 2024-01-08T00:31:07.725Z [INFO] server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: entering candidate state: node="Node at 18fcd28f-913b-5bc0-190a-6785fa6fd2db [Candidate]" term=3 + testing.go:149: server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: 2024-01-08T00:31:07.725Z [INFO] server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: entering candidate state: node="Node at 18fcd28f-913b-5bc0-190a-6785fa6fd2db [Candidate]" term=3 +testLoggerAdapter verbose: server-84780128-5416-124f-ec25-338c09410456: 2024-01-08T00:31:07.725Z [WARN] server-84780128-5416-124f-ec25-338c09410456: rejecting vote request since we have a leader: from=18fcd28f-913b-5bc0-190a-6785fa6fd2db leader=3f73f033-b0e0-fea2-6f6d-489334b4a58b + testing.go:149: server-84780128-5416-124f-ec25-338c09410456: 2024-01-08T00:31:07.725Z [WARN] server-84780128-5416-124f-ec25-338c09410456: rejecting vote request since we have a leader: from=18fcd28f-913b-5bc0-190a-6785fa6fd2db leader=3f73f033-b0e0-fea2-6f6d-489334b4a58b +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.725Z [WARN] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: rejecting vote request since we have a leader: from=18fcd28f-913b-5bc0-190a-6785fa6fd2db leader=3f73f033-b0e0-fea2-6f6d-489334b4a58b + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.725Z [WARN] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: rejecting vote request since we have a leader: from=18fcd28f-913b-5bc0-190a-6785fa6fd2db leader=3f73f033-b0e0-fea2-6f6d-489334b4a58b +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.734Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: updating configuration: command=AddStaging server-id=server-18fcd28f-913b-5bc0-190a-6785fa6fd2db server-addr=18fcd28f-913b-5bc0-190a-6785fa6fd2db servers="[{Suffrage:Voter ID:server-3f73f033-b0e0-fea2-6f6d-489334b4a58b Address:3f73f033-b0e0-fea2-6f6d-489334b4a58b} {Suffrage:Voter ID:server-84780128-5416-124f-ec25-338c09410456 Address:84780128-5416-124f-ec25-338c09410456} {Suffrage:Voter ID:server-18fcd28f-913b-5bc0-190a-6785fa6fd2db Address:18fcd28f-913b-5bc0-190a-6785fa6fd2db}]" + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.734Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: updating configuration: command=AddStaging server-id=server-18fcd28f-913b-5bc0-190a-6785fa6fd2db server-addr=18fcd28f-913b-5bc0-190a-6785fa6fd2db servers="[{Suffrage:Voter ID:server-3f73f033-b0e0-fea2-6f6d-489334b4a58b Address:3f73f033-b0e0-fea2-6f6d-489334b4a58b} {Suffrage:Voter ID:server-84780128-5416-124f-ec25-338c09410456 Address:84780128-5416-124f-ec25-338c09410456} {Suffrage:Voter ID:server-18fcd28f-913b-5bc0-190a-6785fa6fd2db Address:18fcd28f-913b-5bc0-190a-6785fa6fd2db}]" +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.734Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: added peer, starting replication: peer=server-18fcd28f-913b-5bc0-190a-6785fa6fd2db + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.734Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: added peer, starting replication: peer=server-18fcd28f-913b-5bc0-190a-6785fa6fd2db +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.734Z [ERROR] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: peer has newer term, stopping replication: peer="{Voter server-18fcd28f-913b-5bc0-190a-6785fa6fd2db 18fcd28f-913b-5bc0-190a-6785fa6fd2db}" + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.734Z [ERROR] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: peer has newer term, stopping replication: peer="{Voter server-18fcd28f-913b-5bc0-190a-6785fa6fd2db 18fcd28f-913b-5bc0-190a-6785fa6fd2db}" +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.734Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: entering follower state: follower="Node at 3f73f033-b0e0-fea2-6f6d-489334b4a58b [Follower]" leader= + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.734Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: entering follower state: follower="Node at 3f73f033-b0e0-fea2-6f6d-489334b4a58b [Follower]" leader= +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:07.734Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:07.734Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.735Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: aborting pipeline replication: peer="{Voter server-84780128-5416-124f-ec25-338c09410456 84780128-5416-124f-ec25-338c09410456}" + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.735Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: aborting pipeline replication: peer="{Voter server-84780128-5416-124f-ec25-338c09410456 84780128-5416-124f-ec25-338c09410456}" +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.792Z [WARN] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.792Z [WARN] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.792Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: entering candidate state: node="Node at 3f73f033-b0e0-fea2-6f6d-489334b4a58b [Candidate]" term=3 + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.792Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: entering candidate state: node="Node at 3f73f033-b0e0-fea2-6f6d-489334b4a58b [Candidate]" term=3 +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.792Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: election won: tally=2 + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.792Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: election won: tally=2 +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.792Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: entering leader state: leader="Node at 3f73f033-b0e0-fea2-6f6d-489334b4a58b [Leader]" + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.792Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: entering leader state: leader="Node at 3f73f033-b0e0-fea2-6f6d-489334b4a58b [Leader]" +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.792Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: added peer, starting replication: peer=server-84780128-5416-124f-ec25-338c09410456 + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.792Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: added peer, starting replication: peer=server-84780128-5416-124f-ec25-338c09410456 +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.792Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: added peer, starting replication: peer=server-18fcd28f-913b-5bc0-190a-6785fa6fd2db + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.792Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: added peer, starting replication: peer=server-18fcd28f-913b-5bc0-190a-6785fa6fd2db +testLoggerAdapter verbose: server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: 2024-01-08T00:31:07.792Z [WARN] server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: failed to get previous log: previous-index=4 last-index=3 error="log not found" + testing.go:149: server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: 2024-01-08T00:31:07.792Z [WARN] server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: failed to get previous log: previous-index=4 last-index=3 error="log not found" +testLoggerAdapter verbose: server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: 2024-01-08T00:31:07.792Z [INFO] server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: entering follower state: follower="Node at 18fcd28f-913b-5bc0-190a-6785fa6fd2db [Follower]" leader=3f73f033-b0e0-fea2-6f6d-489334b4a58b + testing.go:149: server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: 2024-01-08T00:31:07.792Z [INFO] server-18fcd28f-913b-5bc0-190a-6785fa6fd2db: entering follower state: follower="Node at 18fcd28f-913b-5bc0-190a-6785fa6fd2db [Follower]" leader=3f73f033-b0e0-fea2-6f6d-489334b4a58b +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.792Z [WARN] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: appendEntries rejected, sending older logs: peer="{Voter server-18fcd28f-913b-5bc0-190a-6785fa6fd2db 18fcd28f-913b-5bc0-190a-6785fa6fd2db}" next=4 + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.792Z [WARN] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: appendEntries rejected, sending older logs: peer="{Voter server-18fcd28f-913b-5bc0-190a-6785fa6fd2db 18fcd28f-913b-5bc0-190a-6785fa6fd2db}" next=4 +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.792Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: pipelining replication: peer="{Voter server-18fcd28f-913b-5bc0-190a-6785fa6fd2db 18fcd28f-913b-5bc0-190a-6785fa6fd2db}" + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.792Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: pipelining replication: peer="{Voter server-18fcd28f-913b-5bc0-190a-6785fa6fd2db 18fcd28f-913b-5bc0-190a-6785fa6fd2db}" +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.793Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: pipelining replication: peer="{Voter server-84780128-5416-124f-ec25-338c09410456 84780128-5416-124f-ec25-338c09410456}" + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.793Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: pipelining replication: peer="{Voter server-84780128-5416-124f-ec25-338c09410456 84780128-5416-124f-ec25-338c09410456}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:07.898Z [INFO] stable state for Leader reached at 2024-01-08 00:31:07.793079797 +0000 UTC m=+13.839968838 (1 nodes), 58.109013ms from start of poll, 448.078882ms from cluster start. Timeout at 2024-01-08 00:31:07.898427875 +0000 UTC m=+13.945317109, 105.348271ms after stability + testing.go:149: cluster: 2024-01-08T00:31:07.898Z [INFO] stable state for Leader reached at 2024-01-08 00:31:07.793079797 +0000 UTC m=+13.839968838 (1 nodes), 58.109013ms from start of poll, 448.078882ms from cluster start. Timeout at 2024-01-08 00:31:07.898427875 +0000 UTC m=+13.945317109, 105.348271ms after stability +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.898Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: aborting pipeline replication: peer="{Voter server-18fcd28f-913b-5bc0-190a-6785fa6fd2db 18fcd28f-913b-5bc0-190a-6785fa6fd2db}" + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.898Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: aborting pipeline replication: peer="{Voter server-18fcd28f-913b-5bc0-190a-6785fa6fd2db 18fcd28f-913b-5bc0-190a-6785fa6fd2db}" +testLoggerAdapter verbose: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.898Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: aborting pipeline replication: peer="{Voter server-84780128-5416-124f-ec25-338c09410456 84780128-5416-124f-ec25-338c09410456}" + testing.go:149: server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: 2024-01-08T00:31:07.898Z [INFO] server-3f73f033-b0e0-fea2-6f6d-489334b4a58b: aborting pipeline replication: peer="{Voter server-84780128-5416-124f-ec25-338c09410456 84780128-5416-124f-ec25-338c09410456}" +--- PASS: TestRaft_ReJoinFollower (0.55s) === RUN TestRaft_LeaderLeaseExpire -testLoggerAdapter verbose: server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: 2025-02-09T06:52:32.533Z [INFO] server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2f7294cd-deaa-11d5-cabd-b4cfb810832f Address:2f7294cd-deaa-11d5-cabd-b4cfb810832f} {Suffrage:Voter ID:server-b85161ed-805a-30a5-72d2-168ba2464710 Address:b85161ed-805a-30a5-72d2-168ba2464710}]" - testing.go:149: server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: 2025-02-09T06:52:32.533Z [INFO] server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2f7294cd-deaa-11d5-cabd-b4cfb810832f Address:2f7294cd-deaa-11d5-cabd-b4cfb810832f} {Suffrage:Voter ID:server-b85161ed-805a-30a5-72d2-168ba2464710 Address:b85161ed-805a-30a5-72d2-168ba2464710}]" -testLoggerAdapter verbose: server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: 2025-02-09T06:52:32.533Z [INFO] server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: entering follower state: follower="Node at 2f7294cd-deaa-11d5-cabd-b4cfb810832f [Follower]" leader= - testing.go:149: server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: 2025-02-09T06:52:32.533Z [INFO] server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: entering follower state: follower="Node at 2f7294cd-deaa-11d5-cabd-b4cfb810832f [Follower]" leader= -testLoggerAdapter verbose: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.533Z [INFO] server-b85161ed-805a-30a5-72d2-168ba2464710: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2f7294cd-deaa-11d5-cabd-b4cfb810832f Address:2f7294cd-deaa-11d5-cabd-b4cfb810832f} {Suffrage:Voter ID:server-b85161ed-805a-30a5-72d2-168ba2464710 Address:b85161ed-805a-30a5-72d2-168ba2464710}]" - testing.go:149: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.533Z [INFO] server-b85161ed-805a-30a5-72d2-168ba2464710: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2f7294cd-deaa-11d5-cabd-b4cfb810832f Address:2f7294cd-deaa-11d5-cabd-b4cfb810832f} {Suffrage:Voter ID:server-b85161ed-805a-30a5-72d2-168ba2464710 Address:b85161ed-805a-30a5-72d2-168ba2464710}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:32.533Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:32.533Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.533Z [INFO] server-b85161ed-805a-30a5-72d2-168ba2464710: entering follower state: follower="Node at b85161ed-805a-30a5-72d2-168ba2464710 [Follower]" leader= - testing.go:149: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.533Z [INFO] server-b85161ed-805a-30a5-72d2-168ba2464710: entering follower state: follower="Node at b85161ed-805a-30a5-72d2-168ba2464710 [Follower]" leader= -testLoggerAdapter verbose: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.619Z [WARN] server-b85161ed-805a-30a5-72d2-168ba2464710: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.619Z [WARN] server-b85161ed-805a-30a5-72d2-168ba2464710: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.620Z [INFO] server-b85161ed-805a-30a5-72d2-168ba2464710: entering candidate state: node="Node at b85161ed-805a-30a5-72d2-168ba2464710 [Candidate]" term=2 - testing.go:149: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.620Z [INFO] server-b85161ed-805a-30a5-72d2-168ba2464710: entering candidate state: node="Node at b85161ed-805a-30a5-72d2-168ba2464710 [Candidate]" term=2 -testLoggerAdapter verbose: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.620Z [INFO] server-b85161ed-805a-30a5-72d2-168ba2464710: election won: tally=2 - testing.go:149: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.620Z [INFO] server-b85161ed-805a-30a5-72d2-168ba2464710: election won: tally=2 -testLoggerAdapter verbose: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.620Z [INFO] server-b85161ed-805a-30a5-72d2-168ba2464710: entering leader state: leader="Node at b85161ed-805a-30a5-72d2-168ba2464710 [Leader]" - testing.go:149: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.620Z [INFO] server-b85161ed-805a-30a5-72d2-168ba2464710: entering leader state: leader="Node at b85161ed-805a-30a5-72d2-168ba2464710 [Leader]" -testLoggerAdapter verbose: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.620Z [INFO] server-b85161ed-805a-30a5-72d2-168ba2464710: added peer, starting replication: peer=server-2f7294cd-deaa-11d5-cabd-b4cfb810832f - testing.go:149: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.620Z [INFO] server-b85161ed-805a-30a5-72d2-168ba2464710: added peer, starting replication: peer=server-2f7294cd-deaa-11d5-cabd-b4cfb810832f -testLoggerAdapter verbose: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.620Z [INFO] server-b85161ed-805a-30a5-72d2-168ba2464710: pipelining replication: peer="{Voter server-2f7294cd-deaa-11d5-cabd-b4cfb810832f 2f7294cd-deaa-11d5-cabd-b4cfb810832f}" - testing.go:149: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.620Z [INFO] server-b85161ed-805a-30a5-72d2-168ba2464710: pipelining replication: peer="{Voter server-2f7294cd-deaa-11d5-cabd-b4cfb810832f 2f7294cd-deaa-11d5-cabd-b4cfb810832f}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:32.725Z [INFO] stable state for Leader reached at 2025-02-09 06:52:32.62016912 +0000 UTC m=+13.598977897 (1 nodes), 86.907775ms from start of poll, 87.214905ms from cluster start. Timeout at 2025-02-09 06:52:32.725517666 +0000 UTC m=+13.704326424, 105.348527ms after stability - testing.go:149: cluster: 2025-02-09T06:52:32.725Z [INFO] stable state for Leader reached at 2025-02-09 06:52:32.62016912 +0000 UTC m=+13.598977897 (1 nodes), 86.907775ms from start of poll, 87.214905ms from cluster start. Timeout at 2025-02-09 06:52:32.725517666 +0000 UTC m=+13.704326424, 105.348527ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:32.730Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T06:52:32.730Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:32.836Z [INFO] stable state for Follower reached at 2025-02-09 06:52:32.730997312 +0000 UTC m=+13.709806077 (1 nodes), 442ns from start of poll, 198.043085ms from cluster start. Timeout at 2025-02-09 06:52:32.836008408 +0000 UTC m=+13.814817176, 105.011099ms after stability - testing.go:149: cluster: 2025-02-09T06:52:32.836Z [INFO] stable state for Follower reached at 2025-02-09 06:52:32.730997312 +0000 UTC m=+13.709806077 (1 nodes), 442ns from start of poll, 198.043085ms from cluster start. Timeout at 2025-02-09 06:52:32.836008408 +0000 UTC m=+13.814817176, 105.011099ms after stability - raft_test.go:1492: [INFO] Disconnecting Node at 2f7294cd-deaa-11d5-cabd-b4cfb810832f [Follower] -testLoggerAdapter verbose: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.840Z [ERROR] server-b85161ed-805a-30a5-72d2-168ba2464710: failed to heartbeat to: peer=2f7294cd-deaa-11d5-cabd-b4cfb810832f error="failed to connect to peer: 2f7294cd-deaa-11d5-cabd-b4cfb810832f" - testing.go:149: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.840Z [ERROR] server-b85161ed-805a-30a5-72d2-168ba2464710: failed to heartbeat to: peer=2f7294cd-deaa-11d5-cabd-b4cfb810832f error="failed to connect to peer: 2f7294cd-deaa-11d5-cabd-b4cfb810832f" -testLoggerAdapter verbose: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.840Z [ERROR] server-b85161ed-805a-30a5-72d2-168ba2464710: failed to pipeline appendEntries: peer="{Voter server-2f7294cd-deaa-11d5-cabd-b4cfb810832f 2f7294cd-deaa-11d5-cabd-b4cfb810832f}" error="append pipeline closed" - testing.go:149: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.840Z [ERROR] server-b85161ed-805a-30a5-72d2-168ba2464710: failed to pipeline appendEntries: peer="{Voter server-2f7294cd-deaa-11d5-cabd-b4cfb810832f 2f7294cd-deaa-11d5-cabd-b4cfb810832f}" error="append pipeline closed" -testLoggerAdapter verbose: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.840Z [INFO] server-b85161ed-805a-30a5-72d2-168ba2464710: aborting pipeline replication: peer="{Voter server-2f7294cd-deaa-11d5-cabd-b4cfb810832f 2f7294cd-deaa-11d5-cabd-b4cfb810832f}" - testing.go:149: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.840Z [INFO] server-b85161ed-805a-30a5-72d2-168ba2464710: aborting pipeline replication: peer="{Voter server-2f7294cd-deaa-11d5-cabd-b4cfb810832f 2f7294cd-deaa-11d5-cabd-b4cfb810832f}" -testLoggerAdapter verbose: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.847Z [ERROR] server-b85161ed-805a-30a5-72d2-168ba2464710: failed to appendEntries to: peer="{Voter server-2f7294cd-deaa-11d5-cabd-b4cfb810832f 2f7294cd-deaa-11d5-cabd-b4cfb810832f}" error="failed to connect to peer: 2f7294cd-deaa-11d5-cabd-b4cfb810832f" - testing.go:149: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.847Z [ERROR] server-b85161ed-805a-30a5-72d2-168ba2464710: failed to appendEntries to: peer="{Voter server-2f7294cd-deaa-11d5-cabd-b4cfb810832f 2f7294cd-deaa-11d5-cabd-b4cfb810832f}" error="failed to connect to peer: 2f7294cd-deaa-11d5-cabd-b4cfb810832f" -testLoggerAdapter verbose: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.858Z [ERROR] server-b85161ed-805a-30a5-72d2-168ba2464710: failed to heartbeat to: peer=2f7294cd-deaa-11d5-cabd-b4cfb810832f error="failed to connect to peer: 2f7294cd-deaa-11d5-cabd-b4cfb810832f" - testing.go:149: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.858Z [ERROR] server-b85161ed-805a-30a5-72d2-168ba2464710: failed to heartbeat to: peer=2f7294cd-deaa-11d5-cabd-b4cfb810832f error="failed to connect to peer: 2f7294cd-deaa-11d5-cabd-b4cfb810832f" -testLoggerAdapter verbose: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.866Z [ERROR] server-b85161ed-805a-30a5-72d2-168ba2464710: failed to appendEntries to: peer="{Voter server-2f7294cd-deaa-11d5-cabd-b4cfb810832f 2f7294cd-deaa-11d5-cabd-b4cfb810832f}" error="failed to connect to peer: 2f7294cd-deaa-11d5-cabd-b4cfb810832f" - testing.go:149: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.866Z [ERROR] server-b85161ed-805a-30a5-72d2-168ba2464710: failed to appendEntries to: peer="{Voter server-2f7294cd-deaa-11d5-cabd-b4cfb810832f 2f7294cd-deaa-11d5-cabd-b4cfb810832f}" error="failed to connect to peer: 2f7294cd-deaa-11d5-cabd-b4cfb810832f" -testLoggerAdapter verbose: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.873Z [ERROR] server-b85161ed-805a-30a5-72d2-168ba2464710: failed to heartbeat to: peer=2f7294cd-deaa-11d5-cabd-b4cfb810832f error="failed to connect to peer: 2f7294cd-deaa-11d5-cabd-b4cfb810832f" - testing.go:149: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.873Z [ERROR] server-b85161ed-805a-30a5-72d2-168ba2464710: failed to heartbeat to: peer=2f7294cd-deaa-11d5-cabd-b4cfb810832f error="failed to connect to peer: 2f7294cd-deaa-11d5-cabd-b4cfb810832f" -testLoggerAdapter verbose: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.881Z [WARN] server-b85161ed-805a-30a5-72d2-168ba2464710: failed to contact: server-id=server-2f7294cd-deaa-11d5-cabd-b4cfb810832f time=50.588458ms - testing.go:149: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.881Z [WARN] server-b85161ed-805a-30a5-72d2-168ba2464710: failed to contact: server-id=server-2f7294cd-deaa-11d5-cabd-b4cfb810832f time=50.588458ms -testLoggerAdapter verbose: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.881Z [WARN] server-b85161ed-805a-30a5-72d2-168ba2464710: failed to contact quorum of nodes, stepping down - testing.go:149: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.881Z [WARN] server-b85161ed-805a-30a5-72d2-168ba2464710: failed to contact quorum of nodes, stepping down -testLoggerAdapter verbose: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.881Z [INFO] server-b85161ed-805a-30a5-72d2-168ba2464710: entering follower state: follower="Node at b85161ed-805a-30a5-72d2-168ba2464710 [Follower]" leader= - testing.go:149: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.881Z [INFO] server-b85161ed-805a-30a5-72d2-168ba2464710: entering follower state: follower="Node at b85161ed-805a-30a5-72d2-168ba2464710 [Follower]" leader= -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:32.881Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:32.881Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.882Z [ERROR] server-b85161ed-805a-30a5-72d2-168ba2464710: failed to appendEntries to: peer="{Voter server-2f7294cd-deaa-11d5-cabd-b4cfb810832f 2f7294cd-deaa-11d5-cabd-b4cfb810832f}" error="failed to connect to peer: 2f7294cd-deaa-11d5-cabd-b4cfb810832f" - testing.go:149: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.882Z [ERROR] server-b85161ed-805a-30a5-72d2-168ba2464710: failed to appendEntries to: peer="{Voter server-2f7294cd-deaa-11d5-cabd-b4cfb810832f 2f7294cd-deaa-11d5-cabd-b4cfb810832f}" error="failed to connect to peer: 2f7294cd-deaa-11d5-cabd-b4cfb810832f" -testLoggerAdapter verbose: server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: 2025-02-09T06:52:32.908Z [WARN] server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: heartbeat timeout reached, starting election: last-leader=b85161ed-805a-30a5-72d2-168ba2464710 - testing.go:149: server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: 2025-02-09T06:52:32.908Z [WARN] server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: heartbeat timeout reached, starting election: last-leader=b85161ed-805a-30a5-72d2-168ba2464710 -testLoggerAdapter verbose: server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: 2025-02-09T06:52:32.908Z [INFO] server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: entering candidate state: node="Node at 2f7294cd-deaa-11d5-cabd-b4cfb810832f [Candidate]" term=3 - testing.go:149: server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: 2025-02-09T06:52:32.908Z [INFO] server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: entering candidate state: node="Node at 2f7294cd-deaa-11d5-cabd-b4cfb810832f [Candidate]" term=3 -testLoggerAdapter verbose: server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: 2025-02-09T06:52:32.908Z [ERROR] server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: failed to make requestVote RPC: target="{Voter server-b85161ed-805a-30a5-72d2-168ba2464710 b85161ed-805a-30a5-72d2-168ba2464710}" error="failed to connect to peer: b85161ed-805a-30a5-72d2-168ba2464710" - testing.go:149: server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: 2025-02-09T06:52:32.908Z [ERROR] server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: failed to make requestVote RPC: target="{Voter server-b85161ed-805a-30a5-72d2-168ba2464710 b85161ed-805a-30a5-72d2-168ba2464710}" error="failed to connect to peer: b85161ed-805a-30a5-72d2-168ba2464710" -testLoggerAdapter verbose: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.972Z [WARN] server-b85161ed-805a-30a5-72d2-168ba2464710: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.972Z [WARN] server-b85161ed-805a-30a5-72d2-168ba2464710: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.972Z [INFO] server-b85161ed-805a-30a5-72d2-168ba2464710: entering candidate state: node="Node at b85161ed-805a-30a5-72d2-168ba2464710 [Candidate]" term=3 - testing.go:149: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.972Z [INFO] server-b85161ed-805a-30a5-72d2-168ba2464710: entering candidate state: node="Node at b85161ed-805a-30a5-72d2-168ba2464710 [Candidate]" term=3 -testLoggerAdapter verbose: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.972Z [ERROR] server-b85161ed-805a-30a5-72d2-168ba2464710: failed to make requestVote RPC: target="{Voter server-2f7294cd-deaa-11d5-cabd-b4cfb810832f 2f7294cd-deaa-11d5-cabd-b4cfb810832f}" error="failed to connect to peer: 2f7294cd-deaa-11d5-cabd-b4cfb810832f" - testing.go:149: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:32.972Z [ERROR] server-b85161ed-805a-30a5-72d2-168ba2464710: failed to make requestVote RPC: target="{Voter server-2f7294cd-deaa-11d5-cabd-b4cfb810832f 2f7294cd-deaa-11d5-cabd-b4cfb810832f}" error="failed to connect to peer: 2f7294cd-deaa-11d5-cabd-b4cfb810832f" -testLoggerAdapter verbose: server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: 2025-02-09T06:52:32.980Z [WARN] server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: Election timeout reached, restarting election - testing.go:149: server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: 2025-02-09T06:52:32.980Z [WARN] server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: Election timeout reached, restarting election -testLoggerAdapter verbose: server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: 2025-02-09T06:52:32.980Z [INFO] server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: entering candidate state: node="Node at 2f7294cd-deaa-11d5-cabd-b4cfb810832f [Candidate]" term=4 - testing.go:149: server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: 2025-02-09T06:52:32.980Z [INFO] server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: entering candidate state: node="Node at 2f7294cd-deaa-11d5-cabd-b4cfb810832f [Candidate]" term=4 -testLoggerAdapter verbose: server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: 2025-02-09T06:52:32.980Z [ERROR] server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: failed to make requestVote RPC: target="{Voter server-b85161ed-805a-30a5-72d2-168ba2464710 b85161ed-805a-30a5-72d2-168ba2464710}" error="failed to connect to peer: b85161ed-805a-30a5-72d2-168ba2464710" - testing.go:149: server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: 2025-02-09T06:52:32.980Z [ERROR] server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: failed to make requestVote RPC: target="{Voter server-b85161ed-805a-30a5-72d2-168ba2464710 b85161ed-805a-30a5-72d2-168ba2464710}" error="failed to connect to peer: b85161ed-805a-30a5-72d2-168ba2464710" -testLoggerAdapter verbose: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:33.064Z [WARN] server-b85161ed-805a-30a5-72d2-168ba2464710: Election timeout reached, restarting election - testing.go:149: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:33.064Z [WARN] server-b85161ed-805a-30a5-72d2-168ba2464710: Election timeout reached, restarting election -testLoggerAdapter verbose: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:33.064Z [INFO] server-b85161ed-805a-30a5-72d2-168ba2464710: entering candidate state: node="Node at b85161ed-805a-30a5-72d2-168ba2464710 [Candidate]" term=4 - testing.go:149: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:33.064Z [INFO] server-b85161ed-805a-30a5-72d2-168ba2464710: entering candidate state: node="Node at b85161ed-805a-30a5-72d2-168ba2464710 [Candidate]" term=4 -testLoggerAdapter verbose: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:33.064Z [ERROR] server-b85161ed-805a-30a5-72d2-168ba2464710: failed to make requestVote RPC: target="{Voter server-2f7294cd-deaa-11d5-cabd-b4cfb810832f 2f7294cd-deaa-11d5-cabd-b4cfb810832f}" error="failed to connect to peer: 2f7294cd-deaa-11d5-cabd-b4cfb810832f" - testing.go:149: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:33.064Z [ERROR] server-b85161ed-805a-30a5-72d2-168ba2464710: failed to make requestVote RPC: target="{Voter server-2f7294cd-deaa-11d5-cabd-b4cfb810832f 2f7294cd-deaa-11d5-cabd-b4cfb810832f}" error="failed to connect to peer: 2f7294cd-deaa-11d5-cabd-b4cfb810832f" -testLoggerAdapter verbose: server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: 2025-02-09T06:52:33.070Z [WARN] server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: Election timeout reached, restarting election - testing.go:149: server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: 2025-02-09T06:52:33.070Z [WARN] server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: Election timeout reached, restarting election -testLoggerAdapter verbose: server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: 2025-02-09T06:52:33.070Z [INFO] server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: entering candidate state: node="Node at 2f7294cd-deaa-11d5-cabd-b4cfb810832f [Candidate]" term=5 - testing.go:149: server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: 2025-02-09T06:52:33.070Z [INFO] server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: entering candidate state: node="Node at 2f7294cd-deaa-11d5-cabd-b4cfb810832f [Candidate]" term=5 -testLoggerAdapter verbose: server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: 2025-02-09T06:52:33.070Z [ERROR] server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: failed to make requestVote RPC: target="{Voter server-b85161ed-805a-30a5-72d2-168ba2464710 b85161ed-805a-30a5-72d2-168ba2464710}" error="failed to connect to peer: b85161ed-805a-30a5-72d2-168ba2464710" - testing.go:149: server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: 2025-02-09T06:52:33.070Z [ERROR] server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: failed to make requestVote RPC: target="{Voter server-b85161ed-805a-30a5-72d2-168ba2464710 b85161ed-805a-30a5-72d2-168ba2464710}" error="failed to connect to peer: b85161ed-805a-30a5-72d2-168ba2464710" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:33.078Z [INFO] stable state for Leader reached at 2025-02-09 06:52:32.973002961 +0000 UTC m=+13.951811735 (0 nodes), 91.69049ms from start of poll, 440.048743ms from cluster start. Timeout at 2025-02-09 06:52:33.07841625 +0000 UTC m=+14.057225014, 105.413279ms after stability - testing.go:149: cluster: 2025-02-09T06:52:33.078Z [INFO] stable state for Leader reached at 2025-02-09 06:52:32.973002961 +0000 UTC m=+13.951811735 (0 nodes), 91.69049ms from start of poll, 440.048743ms from cluster start. Timeout at 2025-02-09 06:52:33.07841625 +0000 UTC m=+14.057225014, 105.413279ms after stability -testLoggerAdapter verbose: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:33.127Z [WARN] server-b85161ed-805a-30a5-72d2-168ba2464710: Election timeout reached, restarting election - testing.go:149: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:33.127Z [WARN] server-b85161ed-805a-30a5-72d2-168ba2464710: Election timeout reached, restarting election -testLoggerAdapter verbose: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:33.127Z [INFO] server-b85161ed-805a-30a5-72d2-168ba2464710: entering candidate state: node="Node at b85161ed-805a-30a5-72d2-168ba2464710 [Candidate]" term=5 - testing.go:149: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:33.127Z [INFO] server-b85161ed-805a-30a5-72d2-168ba2464710: entering candidate state: node="Node at b85161ed-805a-30a5-72d2-168ba2464710 [Candidate]" term=5 -testLoggerAdapter verbose: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:33.127Z [ERROR] server-b85161ed-805a-30a5-72d2-168ba2464710: failed to make requestVote RPC: target="{Voter server-2f7294cd-deaa-11d5-cabd-b4cfb810832f 2f7294cd-deaa-11d5-cabd-b4cfb810832f}" error="failed to connect to peer: 2f7294cd-deaa-11d5-cabd-b4cfb810832f" - testing.go:149: server-b85161ed-805a-30a5-72d2-168ba2464710: 2025-02-09T06:52:33.127Z [ERROR] server-b85161ed-805a-30a5-72d2-168ba2464710: failed to make requestVote RPC: target="{Voter server-2f7294cd-deaa-11d5-cabd-b4cfb810832f 2f7294cd-deaa-11d5-cabd-b4cfb810832f}" error="failed to connect to peer: 2f7294cd-deaa-11d5-cabd-b4cfb810832f" -testLoggerAdapter verbose: server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: 2025-02-09T06:52:33.149Z [WARN] server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: Election timeout reached, restarting election - testing.go:149: server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: 2025-02-09T06:52:33.149Z [WARN] server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: Election timeout reached, restarting election -testLoggerAdapter verbose: server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: 2025-02-09T06:52:33.149Z [INFO] server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: entering candidate state: node="Node at 2f7294cd-deaa-11d5-cabd-b4cfb810832f [Candidate]" term=6 - testing.go:149: server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: 2025-02-09T06:52:33.149Z [INFO] server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: entering candidate state: node="Node at 2f7294cd-deaa-11d5-cabd-b4cfb810832f [Candidate]" term=6 -testLoggerAdapter verbose: server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: 2025-02-09T06:52:33.149Z [ERROR] server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: failed to make requestVote RPC: target="{Voter server-b85161ed-805a-30a5-72d2-168ba2464710 b85161ed-805a-30a5-72d2-168ba2464710}" error="failed to connect to peer: b85161ed-805a-30a5-72d2-168ba2464710" - testing.go:149: server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: 2025-02-09T06:52:33.149Z [ERROR] server-2f7294cd-deaa-11d5-cabd-b4cfb810832f: failed to make requestVote RPC: target="{Voter server-b85161ed-805a-30a5-72d2-168ba2464710 b85161ed-805a-30a5-72d2-168ba2464710}" error="failed to connect to peer: b85161ed-805a-30a5-72d2-168ba2464710" ---- PASS: TestRaft_LeaderLeaseExpire (0.65s) +testLoggerAdapter verbose: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:07.899Z [INFO] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9 Address:b0b383b6-dd43-0559-bb4f-8d2b4981f9f9} {Suffrage:Voter ID:server-1ab6f754-920c-98ef-c632-9c21a16b617a Address:1ab6f754-920c-98ef-c632-9c21a16b617a}]" + testing.go:149: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:07.899Z [INFO] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9 Address:b0b383b6-dd43-0559-bb4f-8d2b4981f9f9} {Suffrage:Voter ID:server-1ab6f754-920c-98ef-c632-9c21a16b617a Address:1ab6f754-920c-98ef-c632-9c21a16b617a}]" +testLoggerAdapter verbose: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:07.899Z [INFO] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: entering follower state: follower="Node at b0b383b6-dd43-0559-bb4f-8d2b4981f9f9 [Follower]" leader= + testing.go:149: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:07.899Z [INFO] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: entering follower state: follower="Node at b0b383b6-dd43-0559-bb4f-8d2b4981f9f9 [Follower]" leader= +testLoggerAdapter verbose: server-1ab6f754-920c-98ef-c632-9c21a16b617a: 2024-01-08T00:31:07.899Z [INFO] server-1ab6f754-920c-98ef-c632-9c21a16b617a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9 Address:b0b383b6-dd43-0559-bb4f-8d2b4981f9f9} {Suffrage:Voter ID:server-1ab6f754-920c-98ef-c632-9c21a16b617a Address:1ab6f754-920c-98ef-c632-9c21a16b617a}]" + testing.go:149: server-1ab6f754-920c-98ef-c632-9c21a16b617a: 2024-01-08T00:31:07.899Z [INFO] server-1ab6f754-920c-98ef-c632-9c21a16b617a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9 Address:b0b383b6-dd43-0559-bb4f-8d2b4981f9f9} {Suffrage:Voter ID:server-1ab6f754-920c-98ef-c632-9c21a16b617a Address:1ab6f754-920c-98ef-c632-9c21a16b617a}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:07.900Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:07.900Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-1ab6f754-920c-98ef-c632-9c21a16b617a: 2024-01-08T00:31:07.900Z [INFO] server-1ab6f754-920c-98ef-c632-9c21a16b617a: entering follower state: follower="Node at 1ab6f754-920c-98ef-c632-9c21a16b617a [Follower]" leader= + testing.go:149: server-1ab6f754-920c-98ef-c632-9c21a16b617a: 2024-01-08T00:31:07.900Z [INFO] server-1ab6f754-920c-98ef-c632-9c21a16b617a: entering follower state: follower="Node at 1ab6f754-920c-98ef-c632-9c21a16b617a [Follower]" leader= +testLoggerAdapter verbose: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:07.953Z [WARN] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:07.953Z [WARN] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:07.953Z [INFO] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: entering candidate state: node="Node at b0b383b6-dd43-0559-bb4f-8d2b4981f9f9 [Candidate]" term=2 + testing.go:149: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:07.953Z [INFO] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: entering candidate state: node="Node at b0b383b6-dd43-0559-bb4f-8d2b4981f9f9 [Candidate]" term=2 +testLoggerAdapter verbose: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:07.953Z [INFO] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: election won: tally=2 + testing.go:149: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:07.953Z [INFO] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: election won: tally=2 +testLoggerAdapter verbose: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:07.953Z [INFO] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: entering leader state: leader="Node at b0b383b6-dd43-0559-bb4f-8d2b4981f9f9 [Leader]" + testing.go:149: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:07.953Z [INFO] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: entering leader state: leader="Node at b0b383b6-dd43-0559-bb4f-8d2b4981f9f9 [Leader]" +testLoggerAdapter verbose: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:07.954Z [INFO] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: added peer, starting replication: peer=server-1ab6f754-920c-98ef-c632-9c21a16b617a + testing.go:149: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:07.954Z [INFO] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: added peer, starting replication: peer=server-1ab6f754-920c-98ef-c632-9c21a16b617a +testLoggerAdapter verbose: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:07.954Z [INFO] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: pipelining replication: peer="{Voter server-1ab6f754-920c-98ef-c632-9c21a16b617a 1ab6f754-920c-98ef-c632-9c21a16b617a}" + testing.go:149: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:07.954Z [INFO] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: pipelining replication: peer="{Voter server-1ab6f754-920c-98ef-c632-9c21a16b617a 1ab6f754-920c-98ef-c632-9c21a16b617a}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:08.060Z [INFO] stable state for Leader reached at 2024-01-08 00:31:07.953985543 +0000 UTC m=+14.000874588 (1 nodes), 53.897798ms from start of poll, 54.300972ms from cluster start. Timeout at 2024-01-08 00:31:08.060117103 +0000 UTC m=+14.107006146, 106.131558ms after stability + testing.go:149: cluster: 2024-01-08T00:31:08.060Z [INFO] stable state for Leader reached at 2024-01-08 00:31:07.953985543 +0000 UTC m=+14.000874588 (1 nodes), 53.897798ms from start of poll, 54.300972ms from cluster start. Timeout at 2024-01-08 00:31:08.060117103 +0000 UTC m=+14.107006146, 106.131558ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:08.065Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T00:31:08.065Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:08.171Z [INFO] stable state for Follower reached at 2024-01-08 00:31:08.065754205 +0000 UTC m=+14.112643256 (1 nodes), 1.74µs from start of poll, 166.06964ms from cluster start. Timeout at 2024-01-08 00:31:08.171824342 +0000 UTC m=+14.218713419, 106.070163ms after stability + testing.go:149: cluster: 2024-01-08T00:31:08.171Z [INFO] stable state for Follower reached at 2024-01-08 00:31:08.065754205 +0000 UTC m=+14.112643256 (1 nodes), 1.74µs from start of poll, 166.06964ms from cluster start. Timeout at 2024-01-08 00:31:08.171824342 +0000 UTC m=+14.218713419, 106.070163ms after stability + raft_test.go:1492: [INFO] Disconnecting Node at 1ab6f754-920c-98ef-c632-9c21a16b617a [Follower] +testLoggerAdapter verbose: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.177Z [ERROR] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: failed to heartbeat to: peer=1ab6f754-920c-98ef-c632-9c21a16b617a error="failed to connect to peer: 1ab6f754-920c-98ef-c632-9c21a16b617a" + testing.go:149: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.177Z [ERROR] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: failed to heartbeat to: peer=1ab6f754-920c-98ef-c632-9c21a16b617a error="failed to connect to peer: 1ab6f754-920c-98ef-c632-9c21a16b617a" +testLoggerAdapter verbose: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.180Z [ERROR] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: failed to pipeline appendEntries: peer="{Voter server-1ab6f754-920c-98ef-c632-9c21a16b617a 1ab6f754-920c-98ef-c632-9c21a16b617a}" error="append pipeline closed" + testing.go:149: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.180Z [ERROR] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: failed to pipeline appendEntries: peer="{Voter server-1ab6f754-920c-98ef-c632-9c21a16b617a 1ab6f754-920c-98ef-c632-9c21a16b617a}" error="append pipeline closed" +testLoggerAdapter verbose: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.180Z [INFO] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: aborting pipeline replication: peer="{Voter server-1ab6f754-920c-98ef-c632-9c21a16b617a 1ab6f754-920c-98ef-c632-9c21a16b617a}" + testing.go:149: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.180Z [INFO] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: aborting pipeline replication: peer="{Voter server-1ab6f754-920c-98ef-c632-9c21a16b617a 1ab6f754-920c-98ef-c632-9c21a16b617a}" +testLoggerAdapter verbose: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.189Z [ERROR] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: failed to appendEntries to: peer="{Voter server-1ab6f754-920c-98ef-c632-9c21a16b617a 1ab6f754-920c-98ef-c632-9c21a16b617a}" error="failed to connect to peer: 1ab6f754-920c-98ef-c632-9c21a16b617a" + testing.go:149: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.189Z [ERROR] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: failed to appendEntries to: peer="{Voter server-1ab6f754-920c-98ef-c632-9c21a16b617a 1ab6f754-920c-98ef-c632-9c21a16b617a}" error="failed to connect to peer: 1ab6f754-920c-98ef-c632-9c21a16b617a" +testLoggerAdapter verbose: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.196Z [ERROR] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: failed to heartbeat to: peer=1ab6f754-920c-98ef-c632-9c21a16b617a error="failed to connect to peer: 1ab6f754-920c-98ef-c632-9c21a16b617a" + testing.go:149: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.196Z [ERROR] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: failed to heartbeat to: peer=1ab6f754-920c-98ef-c632-9c21a16b617a error="failed to connect to peer: 1ab6f754-920c-98ef-c632-9c21a16b617a" +testLoggerAdapter verbose: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.208Z [ERROR] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: failed to appendEntries to: peer="{Voter server-1ab6f754-920c-98ef-c632-9c21a16b617a 1ab6f754-920c-98ef-c632-9c21a16b617a}" error="failed to connect to peer: 1ab6f754-920c-98ef-c632-9c21a16b617a" + testing.go:149: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.208Z [ERROR] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: failed to appendEntries to: peer="{Voter server-1ab6f754-920c-98ef-c632-9c21a16b617a 1ab6f754-920c-98ef-c632-9c21a16b617a}" error="failed to connect to peer: 1ab6f754-920c-98ef-c632-9c21a16b617a" +testLoggerAdapter verbose: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.215Z [ERROR] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: failed to heartbeat to: peer=1ab6f754-920c-98ef-c632-9c21a16b617a error="failed to connect to peer: 1ab6f754-920c-98ef-c632-9c21a16b617a" + testing.go:149: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.215Z [ERROR] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: failed to heartbeat to: peer=1ab6f754-920c-98ef-c632-9c21a16b617a error="failed to connect to peer: 1ab6f754-920c-98ef-c632-9c21a16b617a" +testLoggerAdapter verbose: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.221Z [WARN] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: failed to contact: server-id=server-1ab6f754-920c-98ef-c632-9c21a16b617a time=51.037614ms + testing.go:149: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.221Z [WARN] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: failed to contact: server-id=server-1ab6f754-920c-98ef-c632-9c21a16b617a time=51.037614ms +testLoggerAdapter verbose: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.221Z [WARN] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: failed to contact quorum of nodes, stepping down + testing.go:149: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.221Z [WARN] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: failed to contact quorum of nodes, stepping down +testLoggerAdapter verbose: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.221Z [INFO] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: entering follower state: follower="Node at b0b383b6-dd43-0559-bb4f-8d2b4981f9f9 [Follower]" leader= + testing.go:149: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.221Z [INFO] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: entering follower state: follower="Node at b0b383b6-dd43-0559-bb4f-8d2b4981f9f9 [Follower]" leader= +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:08.221Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:08.221Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.227Z [ERROR] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: failed to appendEntries to: peer="{Voter server-1ab6f754-920c-98ef-c632-9c21a16b617a 1ab6f754-920c-98ef-c632-9c21a16b617a}" error="failed to connect to peer: 1ab6f754-920c-98ef-c632-9c21a16b617a" + testing.go:149: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.227Z [ERROR] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: failed to appendEntries to: peer="{Voter server-1ab6f754-920c-98ef-c632-9c21a16b617a 1ab6f754-920c-98ef-c632-9c21a16b617a}" error="failed to connect to peer: 1ab6f754-920c-98ef-c632-9c21a16b617a" +testLoggerAdapter verbose: server-1ab6f754-920c-98ef-c632-9c21a16b617a: 2024-01-08T00:31:08.252Z [WARN] server-1ab6f754-920c-98ef-c632-9c21a16b617a: heartbeat timeout reached, starting election: last-leader=b0b383b6-dd43-0559-bb4f-8d2b4981f9f9 + testing.go:149: server-1ab6f754-920c-98ef-c632-9c21a16b617a: 2024-01-08T00:31:08.252Z [WARN] server-1ab6f754-920c-98ef-c632-9c21a16b617a: heartbeat timeout reached, starting election: last-leader=b0b383b6-dd43-0559-bb4f-8d2b4981f9f9 +testLoggerAdapter verbose: server-1ab6f754-920c-98ef-c632-9c21a16b617a: 2024-01-08T00:31:08.252Z [INFO] server-1ab6f754-920c-98ef-c632-9c21a16b617a: entering candidate state: node="Node at 1ab6f754-920c-98ef-c632-9c21a16b617a [Candidate]" term=3 + testing.go:149: server-1ab6f754-920c-98ef-c632-9c21a16b617a: 2024-01-08T00:31:08.252Z [INFO] server-1ab6f754-920c-98ef-c632-9c21a16b617a: entering candidate state: node="Node at 1ab6f754-920c-98ef-c632-9c21a16b617a [Candidate]" term=3 +testLoggerAdapter verbose: server-1ab6f754-920c-98ef-c632-9c21a16b617a: 2024-01-08T00:31:08.252Z [ERROR] server-1ab6f754-920c-98ef-c632-9c21a16b617a: failed to make requestVote RPC: target="{Voter server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9 b0b383b6-dd43-0559-bb4f-8d2b4981f9f9}" error="failed to connect to peer: b0b383b6-dd43-0559-bb4f-8d2b4981f9f9" + testing.go:149: server-1ab6f754-920c-98ef-c632-9c21a16b617a: 2024-01-08T00:31:08.252Z [ERROR] server-1ab6f754-920c-98ef-c632-9c21a16b617a: failed to make requestVote RPC: target="{Voter server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9 b0b383b6-dd43-0559-bb4f-8d2b4981f9f9}" error="failed to connect to peer: b0b383b6-dd43-0559-bb4f-8d2b4981f9f9" +testLoggerAdapter verbose: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.317Z [WARN] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.317Z [WARN] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.317Z [INFO] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: entering candidate state: node="Node at b0b383b6-dd43-0559-bb4f-8d2b4981f9f9 [Candidate]" term=3 + testing.go:149: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.317Z [INFO] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: entering candidate state: node="Node at b0b383b6-dd43-0559-bb4f-8d2b4981f9f9 [Candidate]" term=3 +testLoggerAdapter verbose: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.317Z [ERROR] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: failed to make requestVote RPC: target="{Voter server-1ab6f754-920c-98ef-c632-9c21a16b617a 1ab6f754-920c-98ef-c632-9c21a16b617a}" error="failed to connect to peer: 1ab6f754-920c-98ef-c632-9c21a16b617a" + testing.go:149: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.317Z [ERROR] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: failed to make requestVote RPC: target="{Voter server-1ab6f754-920c-98ef-c632-9c21a16b617a 1ab6f754-920c-98ef-c632-9c21a16b617a}" error="failed to connect to peer: 1ab6f754-920c-98ef-c632-9c21a16b617a" +testLoggerAdapter verbose: server-1ab6f754-920c-98ef-c632-9c21a16b617a: 2024-01-08T00:31:08.332Z [WARN] server-1ab6f754-920c-98ef-c632-9c21a16b617a: Election timeout reached, restarting election + testing.go:149: server-1ab6f754-920c-98ef-c632-9c21a16b617a: 2024-01-08T00:31:08.332Z [WARN] server-1ab6f754-920c-98ef-c632-9c21a16b617a: Election timeout reached, restarting election +testLoggerAdapter verbose: server-1ab6f754-920c-98ef-c632-9c21a16b617a: 2024-01-08T00:31:08.332Z [INFO] server-1ab6f754-920c-98ef-c632-9c21a16b617a: entering candidate state: node="Node at 1ab6f754-920c-98ef-c632-9c21a16b617a [Candidate]" term=4 + testing.go:149: server-1ab6f754-920c-98ef-c632-9c21a16b617a: 2024-01-08T00:31:08.332Z [INFO] server-1ab6f754-920c-98ef-c632-9c21a16b617a: entering candidate state: node="Node at 1ab6f754-920c-98ef-c632-9c21a16b617a [Candidate]" term=4 +testLoggerAdapter verbose: server-1ab6f754-920c-98ef-c632-9c21a16b617a: 2024-01-08T00:31:08.332Z [ERROR] server-1ab6f754-920c-98ef-c632-9c21a16b617a: failed to make requestVote RPC: target="{Voter server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9 b0b383b6-dd43-0559-bb4f-8d2b4981f9f9}" error="failed to connect to peer: b0b383b6-dd43-0559-bb4f-8d2b4981f9f9" + testing.go:149: server-1ab6f754-920c-98ef-c632-9c21a16b617a: 2024-01-08T00:31:08.332Z [ERROR] server-1ab6f754-920c-98ef-c632-9c21a16b617a: failed to make requestVote RPC: target="{Voter server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9 b0b383b6-dd43-0559-bb4f-8d2b4981f9f9}" error="failed to connect to peer: b0b383b6-dd43-0559-bb4f-8d2b4981f9f9" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:08.358Z [INFO] stable state for Leader reached at 2024-01-08 00:31:08.25224629 +0000 UTC m=+14.299135403 (0 nodes), 30.291665ms from start of poll, 352.561787ms from cluster start. Timeout at 2024-01-08 00:31:08.357952798 +0000 UTC m=+14.404841896, 105.706493ms after stability + testing.go:149: cluster: 2024-01-08T00:31:08.358Z [INFO] stable state for Leader reached at 2024-01-08 00:31:08.25224629 +0000 UTC m=+14.299135403 (0 nodes), 30.291665ms from start of poll, 352.561787ms from cluster start. Timeout at 2024-01-08 00:31:08.357952798 +0000 UTC m=+14.404841896, 105.706493ms after stability +testLoggerAdapter verbose: server-1ab6f754-920c-98ef-c632-9c21a16b617a: 2024-01-08T00:31:08.397Z [WARN] server-1ab6f754-920c-98ef-c632-9c21a16b617a: Election timeout reached, restarting election + testing.go:149: server-1ab6f754-920c-98ef-c632-9c21a16b617a: 2024-01-08T00:31:08.397Z [WARN] server-1ab6f754-920c-98ef-c632-9c21a16b617a: Election timeout reached, restarting election +testLoggerAdapter verbose: server-1ab6f754-920c-98ef-c632-9c21a16b617a: 2024-01-08T00:31:08.397Z [INFO] server-1ab6f754-920c-98ef-c632-9c21a16b617a: entering candidate state: node="Node at 1ab6f754-920c-98ef-c632-9c21a16b617a [Candidate]" term=5 + testing.go:149: server-1ab6f754-920c-98ef-c632-9c21a16b617a: 2024-01-08T00:31:08.397Z [INFO] server-1ab6f754-920c-98ef-c632-9c21a16b617a: entering candidate state: node="Node at 1ab6f754-920c-98ef-c632-9c21a16b617a [Candidate]" term=5 +testLoggerAdapter verbose: server-1ab6f754-920c-98ef-c632-9c21a16b617a: 2024-01-08T00:31:08.397Z [ERROR] server-1ab6f754-920c-98ef-c632-9c21a16b617a: failed to make requestVote RPC: target="{Voter server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9 b0b383b6-dd43-0559-bb4f-8d2b4981f9f9}" error="failed to connect to peer: b0b383b6-dd43-0559-bb4f-8d2b4981f9f9" + testing.go:149: server-1ab6f754-920c-98ef-c632-9c21a16b617a: 2024-01-08T00:31:08.397Z [ERROR] server-1ab6f754-920c-98ef-c632-9c21a16b617a: failed to make requestVote RPC: target="{Voter server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9 b0b383b6-dd43-0559-bb4f-8d2b4981f9f9}" error="failed to connect to peer: b0b383b6-dd43-0559-bb4f-8d2b4981f9f9" +testLoggerAdapter verbose: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.399Z [WARN] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: Election timeout reached, restarting election + testing.go:149: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.399Z [WARN] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: Election timeout reached, restarting election +testLoggerAdapter verbose: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.399Z [INFO] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: entering candidate state: node="Node at b0b383b6-dd43-0559-bb4f-8d2b4981f9f9 [Candidate]" term=4 + testing.go:149: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.399Z [INFO] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: entering candidate state: node="Node at b0b383b6-dd43-0559-bb4f-8d2b4981f9f9 [Candidate]" term=4 +testLoggerAdapter verbose: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.399Z [ERROR] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: failed to make requestVote RPC: target="{Voter server-1ab6f754-920c-98ef-c632-9c21a16b617a 1ab6f754-920c-98ef-c632-9c21a16b617a}" error="failed to connect to peer: 1ab6f754-920c-98ef-c632-9c21a16b617a" + testing.go:149: server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: 2024-01-08T00:31:08.399Z [ERROR] server-b0b383b6-dd43-0559-bb4f-8d2b4981f9f9: failed to make requestVote RPC: target="{Voter server-1ab6f754-920c-98ef-c632-9c21a16b617a 1ab6f754-920c-98ef-c632-9c21a16b617a}" error="failed to connect to peer: 1ab6f754-920c-98ef-c632-9c21a16b617a" +--- PASS: TestRaft_LeaderLeaseExpire (0.56s) === RUN TestRaft_Barrier -testLoggerAdapter verbose: server-93863f81-728f-8596-6af3-e3c555253058: 2025-02-09T06:52:33.184Z [INFO] server-93863f81-728f-8596-6af3-e3c555253058: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-93863f81-728f-8596-6af3-e3c555253058 Address:93863f81-728f-8596-6af3-e3c555253058} {Suffrage:Voter ID:server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608 Address:06fa92f0-2d9d-a0ea-46ee-01ba1714c608} {Suffrage:Voter ID:server-414f9659-6384-ab8e-b782-684e9b49be12 Address:414f9659-6384-ab8e-b782-684e9b49be12}]" - testing.go:149: server-93863f81-728f-8596-6af3-e3c555253058: 2025-02-09T06:52:33.184Z [INFO] server-93863f81-728f-8596-6af3-e3c555253058: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-93863f81-728f-8596-6af3-e3c555253058 Address:93863f81-728f-8596-6af3-e3c555253058} {Suffrage:Voter ID:server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608 Address:06fa92f0-2d9d-a0ea-46ee-01ba1714c608} {Suffrage:Voter ID:server-414f9659-6384-ab8e-b782-684e9b49be12 Address:414f9659-6384-ab8e-b782-684e9b49be12}]" -testLoggerAdapter verbose: server-93863f81-728f-8596-6af3-e3c555253058: 2025-02-09T06:52:33.184Z [INFO] server-93863f81-728f-8596-6af3-e3c555253058: entering follower state: follower="Node at 93863f81-728f-8596-6af3-e3c555253058 [Follower]" leader= - testing.go:149: server-93863f81-728f-8596-6af3-e3c555253058: 2025-02-09T06:52:33.184Z [INFO] server-93863f81-728f-8596-6af3-e3c555253058: entering follower state: follower="Node at 93863f81-728f-8596-6af3-e3c555253058 [Follower]" leader= -testLoggerAdapter verbose: server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: 2025-02-09T06:52:33.184Z [INFO] server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-93863f81-728f-8596-6af3-e3c555253058 Address:93863f81-728f-8596-6af3-e3c555253058} {Suffrage:Voter ID:server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608 Address:06fa92f0-2d9d-a0ea-46ee-01ba1714c608} {Suffrage:Voter ID:server-414f9659-6384-ab8e-b782-684e9b49be12 Address:414f9659-6384-ab8e-b782-684e9b49be12}]" - testing.go:149: server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: 2025-02-09T06:52:33.184Z [INFO] server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-93863f81-728f-8596-6af3-e3c555253058 Address:93863f81-728f-8596-6af3-e3c555253058} {Suffrage:Voter ID:server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608 Address:06fa92f0-2d9d-a0ea-46ee-01ba1714c608} {Suffrage:Voter ID:server-414f9659-6384-ab8e-b782-684e9b49be12 Address:414f9659-6384-ab8e-b782-684e9b49be12}]" -testLoggerAdapter verbose: server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: 2025-02-09T06:52:33.184Z [INFO] server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: entering follower state: follower="Node at 06fa92f0-2d9d-a0ea-46ee-01ba1714c608 [Follower]" leader= - testing.go:149: server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: 2025-02-09T06:52:33.184Z [INFO] server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: entering follower state: follower="Node at 06fa92f0-2d9d-a0ea-46ee-01ba1714c608 [Follower]" leader= -testLoggerAdapter verbose: server-414f9659-6384-ab8e-b782-684e9b49be12: 2025-02-09T06:52:33.184Z [INFO] server-414f9659-6384-ab8e-b782-684e9b49be12: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-93863f81-728f-8596-6af3-e3c555253058 Address:93863f81-728f-8596-6af3-e3c555253058} {Suffrage:Voter ID:server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608 Address:06fa92f0-2d9d-a0ea-46ee-01ba1714c608} {Suffrage:Voter ID:server-414f9659-6384-ab8e-b782-684e9b49be12 Address:414f9659-6384-ab8e-b782-684e9b49be12}]" - testing.go:149: server-414f9659-6384-ab8e-b782-684e9b49be12: 2025-02-09T06:52:33.184Z [INFO] server-414f9659-6384-ab8e-b782-684e9b49be12: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-93863f81-728f-8596-6af3-e3c555253058 Address:93863f81-728f-8596-6af3-e3c555253058} {Suffrage:Voter ID:server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608 Address:06fa92f0-2d9d-a0ea-46ee-01ba1714c608} {Suffrage:Voter ID:server-414f9659-6384-ab8e-b782-684e9b49be12 Address:414f9659-6384-ab8e-b782-684e9b49be12}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:33.184Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:33.184Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-414f9659-6384-ab8e-b782-684e9b49be12: 2025-02-09T06:52:33.184Z [INFO] server-414f9659-6384-ab8e-b782-684e9b49be12: entering follower state: follower="Node at 414f9659-6384-ab8e-b782-684e9b49be12 [Follower]" leader= - testing.go:149: server-414f9659-6384-ab8e-b782-684e9b49be12: 2025-02-09T06:52:33.184Z [INFO] server-414f9659-6384-ab8e-b782-684e9b49be12: entering follower state: follower="Node at 414f9659-6384-ab8e-b782-684e9b49be12 [Follower]" leader= -testLoggerAdapter verbose: server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: 2025-02-09T06:52:33.244Z [WARN] server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: 2025-02-09T06:52:33.244Z [WARN] server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: 2025-02-09T06:52:33.244Z [INFO] server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: entering candidate state: node="Node at 06fa92f0-2d9d-a0ea-46ee-01ba1714c608 [Candidate]" term=2 - testing.go:149: server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: 2025-02-09T06:52:33.244Z [INFO] server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: entering candidate state: node="Node at 06fa92f0-2d9d-a0ea-46ee-01ba1714c608 [Candidate]" term=2 -testLoggerAdapter verbose: server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: 2025-02-09T06:52:33.244Z [INFO] server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: election won: tally=2 - testing.go:149: server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: 2025-02-09T06:52:33.244Z [INFO] server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: election won: tally=2 -testLoggerAdapter verbose: server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: 2025-02-09T06:52:33.244Z [INFO] server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: entering leader state: leader="Node at 06fa92f0-2d9d-a0ea-46ee-01ba1714c608 [Leader]" - testing.go:149: server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: 2025-02-09T06:52:33.244Z [INFO] server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: entering leader state: leader="Node at 06fa92f0-2d9d-a0ea-46ee-01ba1714c608 [Leader]" -testLoggerAdapter verbose: server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: 2025-02-09T06:52:33.244Z [INFO] server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: added peer, starting replication: peer=server-93863f81-728f-8596-6af3-e3c555253058 - testing.go:149: server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: 2025-02-09T06:52:33.244Z [INFO] server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: added peer, starting replication: peer=server-93863f81-728f-8596-6af3-e3c555253058 -testLoggerAdapter verbose: server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: 2025-02-09T06:52:33.244Z [INFO] server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: added peer, starting replication: peer=server-414f9659-6384-ab8e-b782-684e9b49be12 - testing.go:149: server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: 2025-02-09T06:52:33.244Z [INFO] server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: added peer, starting replication: peer=server-414f9659-6384-ab8e-b782-684e9b49be12 -testLoggerAdapter verbose: server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: 2025-02-09T06:52:33.244Z [INFO] server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: pipelining replication: peer="{Voter server-93863f81-728f-8596-6af3-e3c555253058 93863f81-728f-8596-6af3-e3c555253058}" - testing.go:149: server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: 2025-02-09T06:52:33.244Z [INFO] server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: pipelining replication: peer="{Voter server-93863f81-728f-8596-6af3-e3c555253058 93863f81-728f-8596-6af3-e3c555253058}" -testLoggerAdapter verbose: server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: 2025-02-09T06:52:33.244Z [INFO] server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: pipelining replication: peer="{Voter server-414f9659-6384-ab8e-b782-684e9b49be12 414f9659-6384-ab8e-b782-684e9b49be12}" - testing.go:149: server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: 2025-02-09T06:52:33.244Z [INFO] server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: pipelining replication: peer="{Voter server-414f9659-6384-ab8e-b782-684e9b49be12 414f9659-6384-ab8e-b782-684e9b49be12}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:33.350Z [INFO] stable state for Leader reached at 2025-02-09 06:52:33.244565371 +0000 UTC m=+14.223374150 (1 nodes), 59.702197ms from start of poll, 60.23214ms from cluster start. Timeout at 2025-02-09 06:52:33.35020395 +0000 UTC m=+14.329012720, 105.63857ms after stability - testing.go:149: cluster: 2025-02-09T06:52:33.350Z [INFO] stable state for Leader reached at 2025-02-09 06:52:33.244565371 +0000 UTC m=+14.223374150 (1 nodes), 59.702197ms from start of poll, 60.23214ms from cluster start. Timeout at 2025-02-09 06:52:33.35020395 +0000 UTC m=+14.329012720, 105.63857ms after stability -testLoggerAdapter verbose: server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: 2025-02-09T06:52:33.363Z [INFO] server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: aborting pipeline replication: peer="{Voter server-414f9659-6384-ab8e-b782-684e9b49be12 414f9659-6384-ab8e-b782-684e9b49be12}" - testing.go:149: server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: 2025-02-09T06:52:33.363Z [INFO] server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: aborting pipeline replication: peer="{Voter server-414f9659-6384-ab8e-b782-684e9b49be12 414f9659-6384-ab8e-b782-684e9b49be12}" -testLoggerAdapter verbose: server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: 2025-02-09T06:52:33.363Z [INFO] server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: aborting pipeline replication: peer="{Voter server-93863f81-728f-8596-6af3-e3c555253058 93863f81-728f-8596-6af3-e3c555253058}" - testing.go:149: server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: 2025-02-09T06:52:33.363Z [INFO] server-06fa92f0-2d9d-a0ea-46ee-01ba1714c608: aborting pipeline replication: peer="{Voter server-93863f81-728f-8596-6af3-e3c555253058 93863f81-728f-8596-6af3-e3c555253058}" ---- PASS: TestRaft_Barrier (0.18s) +testLoggerAdapter verbose: server-7822456f-2ca2-2584-f424-e83c56de5b2e: 2024-01-08T00:31:08.464Z [INFO] server-7822456f-2ca2-2584-f424-e83c56de5b2e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7822456f-2ca2-2584-f424-e83c56de5b2e Address:7822456f-2ca2-2584-f424-e83c56de5b2e} {Suffrage:Voter ID:server-b877f982-a861-9ef1-b421-e46f5abdff30 Address:b877f982-a861-9ef1-b421-e46f5abdff30} {Suffrage:Voter ID:server-52e9f313-d0a2-1b7c-0541-bb792953c801 Address:52e9f313-d0a2-1b7c-0541-bb792953c801}]" + testing.go:149: server-7822456f-2ca2-2584-f424-e83c56de5b2e: 2024-01-08T00:31:08.464Z [INFO] server-7822456f-2ca2-2584-f424-e83c56de5b2e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7822456f-2ca2-2584-f424-e83c56de5b2e Address:7822456f-2ca2-2584-f424-e83c56de5b2e} {Suffrage:Voter ID:server-b877f982-a861-9ef1-b421-e46f5abdff30 Address:b877f982-a861-9ef1-b421-e46f5abdff30} {Suffrage:Voter ID:server-52e9f313-d0a2-1b7c-0541-bb792953c801 Address:52e9f313-d0a2-1b7c-0541-bb792953c801}]" +testLoggerAdapter verbose: server-b877f982-a861-9ef1-b421-e46f5abdff30: 2024-01-08T00:31:08.465Z [INFO] server-b877f982-a861-9ef1-b421-e46f5abdff30: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7822456f-2ca2-2584-f424-e83c56de5b2e Address:7822456f-2ca2-2584-f424-e83c56de5b2e} {Suffrage:Voter ID:server-b877f982-a861-9ef1-b421-e46f5abdff30 Address:b877f982-a861-9ef1-b421-e46f5abdff30} {Suffrage:Voter ID:server-52e9f313-d0a2-1b7c-0541-bb792953c801 Address:52e9f313-d0a2-1b7c-0541-bb792953c801}]" + testing.go:149: server-b877f982-a861-9ef1-b421-e46f5abdff30: 2024-01-08T00:31:08.465Z [INFO] server-b877f982-a861-9ef1-b421-e46f5abdff30: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7822456f-2ca2-2584-f424-e83c56de5b2e Address:7822456f-2ca2-2584-f424-e83c56de5b2e} {Suffrage:Voter ID:server-b877f982-a861-9ef1-b421-e46f5abdff30 Address:b877f982-a861-9ef1-b421-e46f5abdff30} {Suffrage:Voter ID:server-52e9f313-d0a2-1b7c-0541-bb792953c801 Address:52e9f313-d0a2-1b7c-0541-bb792953c801}]" +testLoggerAdapter verbose: server-7822456f-2ca2-2584-f424-e83c56de5b2e: 2024-01-08T00:31:08.464Z [INFO] server-7822456f-2ca2-2584-f424-e83c56de5b2e: entering follower state: follower="Node at 7822456f-2ca2-2584-f424-e83c56de5b2e [Follower]" leader= + testing.go:149: server-7822456f-2ca2-2584-f424-e83c56de5b2e: 2024-01-08T00:31:08.464Z [INFO] server-7822456f-2ca2-2584-f424-e83c56de5b2e: entering follower state: follower="Node at 7822456f-2ca2-2584-f424-e83c56de5b2e [Follower]" leader= +testLoggerAdapter verbose: server-b877f982-a861-9ef1-b421-e46f5abdff30: 2024-01-08T00:31:08.465Z [INFO] server-b877f982-a861-9ef1-b421-e46f5abdff30: entering follower state: follower="Node at b877f982-a861-9ef1-b421-e46f5abdff30 [Follower]" leader= +testLoggerAdapter verbose: server-52e9f313-d0a2-1b7c-0541-bb792953c801: 2024-01-08T00:31:08.465Z [INFO] server-52e9f313-d0a2-1b7c-0541-bb792953c801: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7822456f-2ca2-2584-f424-e83c56de5b2e Address:7822456f-2ca2-2584-f424-e83c56de5b2e} {Suffrage:Voter ID:server-b877f982-a861-9ef1-b421-e46f5abdff30 Address:b877f982-a861-9ef1-b421-e46f5abdff30} {Suffrage:Voter ID:server-52e9f313-d0a2-1b7c-0541-bb792953c801 Address:52e9f313-d0a2-1b7c-0541-bb792953c801}]" + testing.go:149: server-b877f982-a861-9ef1-b421-e46f5abdff30: 2024-01-08T00:31:08.465Z [INFO] server-b877f982-a861-9ef1-b421-e46f5abdff30: entering follower state: follower="Node at b877f982-a861-9ef1-b421-e46f5abdff30 [Follower]" leader= + testing.go:149: server-52e9f313-d0a2-1b7c-0541-bb792953c801: 2024-01-08T00:31:08.465Z [INFO] server-52e9f313-d0a2-1b7c-0541-bb792953c801: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7822456f-2ca2-2584-f424-e83c56de5b2e Address:7822456f-2ca2-2584-f424-e83c56de5b2e} {Suffrage:Voter ID:server-b877f982-a861-9ef1-b421-e46f5abdff30 Address:b877f982-a861-9ef1-b421-e46f5abdff30} {Suffrage:Voter ID:server-52e9f313-d0a2-1b7c-0541-bb792953c801 Address:52e9f313-d0a2-1b7c-0541-bb792953c801}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:08.465Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:08.465Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-52e9f313-d0a2-1b7c-0541-bb792953c801: 2024-01-08T00:31:08.465Z [INFO] server-52e9f313-d0a2-1b7c-0541-bb792953c801: entering follower state: follower="Node at 52e9f313-d0a2-1b7c-0541-bb792953c801 [Follower]" leader= + testing.go:149: server-52e9f313-d0a2-1b7c-0541-bb792953c801: 2024-01-08T00:31:08.465Z [INFO] server-52e9f313-d0a2-1b7c-0541-bb792953c801: entering follower state: follower="Node at 52e9f313-d0a2-1b7c-0541-bb792953c801 [Follower]" leader= +testLoggerAdapter verbose: server-52e9f313-d0a2-1b7c-0541-bb792953c801: 2024-01-08T00:31:08.554Z [WARN] server-52e9f313-d0a2-1b7c-0541-bb792953c801: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-52e9f313-d0a2-1b7c-0541-bb792953c801: 2024-01-08T00:31:08.554Z [WARN] server-52e9f313-d0a2-1b7c-0541-bb792953c801: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-52e9f313-d0a2-1b7c-0541-bb792953c801: 2024-01-08T00:31:08.554Z [INFO] server-52e9f313-d0a2-1b7c-0541-bb792953c801: entering candidate state: node="Node at 52e9f313-d0a2-1b7c-0541-bb792953c801 [Candidate]" term=2 + testing.go:149: server-52e9f313-d0a2-1b7c-0541-bb792953c801: 2024-01-08T00:31:08.554Z [INFO] server-52e9f313-d0a2-1b7c-0541-bb792953c801: entering candidate state: node="Node at 52e9f313-d0a2-1b7c-0541-bb792953c801 [Candidate]" term=2 +testLoggerAdapter verbose: server-52e9f313-d0a2-1b7c-0541-bb792953c801: 2024-01-08T00:31:08.554Z [INFO] server-52e9f313-d0a2-1b7c-0541-bb792953c801: election won: tally=2 + testing.go:149: server-52e9f313-d0a2-1b7c-0541-bb792953c801: 2024-01-08T00:31:08.554Z [INFO] server-52e9f313-d0a2-1b7c-0541-bb792953c801: election won: tally=2 +testLoggerAdapter verbose: server-52e9f313-d0a2-1b7c-0541-bb792953c801: 2024-01-08T00:31:08.554Z [INFO] server-52e9f313-d0a2-1b7c-0541-bb792953c801: entering leader state: leader="Node at 52e9f313-d0a2-1b7c-0541-bb792953c801 [Leader]" + testing.go:149: server-52e9f313-d0a2-1b7c-0541-bb792953c801: 2024-01-08T00:31:08.554Z [INFO] server-52e9f313-d0a2-1b7c-0541-bb792953c801: entering leader state: leader="Node at 52e9f313-d0a2-1b7c-0541-bb792953c801 [Leader]" +testLoggerAdapter verbose: server-52e9f313-d0a2-1b7c-0541-bb792953c801: 2024-01-08T00:31:08.554Z [INFO] server-52e9f313-d0a2-1b7c-0541-bb792953c801: added peer, starting replication: peer=server-7822456f-2ca2-2584-f424-e83c56de5b2e + testing.go:149: server-52e9f313-d0a2-1b7c-0541-bb792953c801: 2024-01-08T00:31:08.554Z [INFO] server-52e9f313-d0a2-1b7c-0541-bb792953c801: added peer, starting replication: peer=server-7822456f-2ca2-2584-f424-e83c56de5b2e +testLoggerAdapter verbose: server-52e9f313-d0a2-1b7c-0541-bb792953c801: 2024-01-08T00:31:08.554Z [INFO] server-52e9f313-d0a2-1b7c-0541-bb792953c801: added peer, starting replication: peer=server-b877f982-a861-9ef1-b421-e46f5abdff30 + testing.go:149: server-52e9f313-d0a2-1b7c-0541-bb792953c801: 2024-01-08T00:31:08.554Z [INFO] server-52e9f313-d0a2-1b7c-0541-bb792953c801: added peer, starting replication: peer=server-b877f982-a861-9ef1-b421-e46f5abdff30 +testLoggerAdapter verbose: server-52e9f313-d0a2-1b7c-0541-bb792953c801: 2024-01-08T00:31:08.554Z [INFO] server-52e9f313-d0a2-1b7c-0541-bb792953c801: pipelining replication: peer="{Voter server-7822456f-2ca2-2584-f424-e83c56de5b2e 7822456f-2ca2-2584-f424-e83c56de5b2e}" + testing.go:149: server-52e9f313-d0a2-1b7c-0541-bb792953c801: 2024-01-08T00:31:08.554Z [INFO] server-52e9f313-d0a2-1b7c-0541-bb792953c801: pipelining replication: peer="{Voter server-7822456f-2ca2-2584-f424-e83c56de5b2e 7822456f-2ca2-2584-f424-e83c56de5b2e}" +testLoggerAdapter verbose: server-52e9f313-d0a2-1b7c-0541-bb792953c801: 2024-01-08T00:31:08.554Z [INFO] server-52e9f313-d0a2-1b7c-0541-bb792953c801: pipelining replication: peer="{Voter server-b877f982-a861-9ef1-b421-e46f5abdff30 b877f982-a861-9ef1-b421-e46f5abdff30}" + testing.go:149: server-52e9f313-d0a2-1b7c-0541-bb792953c801: 2024-01-08T00:31:08.554Z [INFO] server-52e9f313-d0a2-1b7c-0541-bb792953c801: pipelining replication: peer="{Voter server-b877f982-a861-9ef1-b421-e46f5abdff30 b877f982-a861-9ef1-b421-e46f5abdff30}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:08.659Z [INFO] stable state for Leader reached at 2024-01-08 00:31:08.554337352 +0000 UTC m=+14.601226401 (1 nodes), 89.041993ms from start of poll, 89.61124ms from cluster start. Timeout at 2024-01-08 00:31:08.659486325 +0000 UTC m=+14.706375357, 105.148956ms after stability + testing.go:149: cluster: 2024-01-08T00:31:08.659Z [INFO] stable state for Leader reached at 2024-01-08 00:31:08.554337352 +0000 UTC m=+14.601226401 (1 nodes), 89.041993ms from start of poll, 89.61124ms from cluster start. Timeout at 2024-01-08 00:31:08.659486325 +0000 UTC m=+14.706375357, 105.148956ms after stability +testLoggerAdapter verbose: server-52e9f313-d0a2-1b7c-0541-bb792953c801: 2024-01-08T00:31:08.674Z [INFO] server-52e9f313-d0a2-1b7c-0541-bb792953c801: aborting pipeline replication: peer="{Voter server-b877f982-a861-9ef1-b421-e46f5abdff30 b877f982-a861-9ef1-b421-e46f5abdff30}" + testing.go:149: server-52e9f313-d0a2-1b7c-0541-bb792953c801: 2024-01-08T00:31:08.674Z [INFO] server-52e9f313-d0a2-1b7c-0541-bb792953c801: aborting pipeline replication: peer="{Voter server-b877f982-a861-9ef1-b421-e46f5abdff30 b877f982-a861-9ef1-b421-e46f5abdff30}" +testLoggerAdapter verbose: server-52e9f313-d0a2-1b7c-0541-bb792953c801: 2024-01-08T00:31:08.674Z [INFO] server-52e9f313-d0a2-1b7c-0541-bb792953c801: aborting pipeline replication: peer="{Voter server-7822456f-2ca2-2584-f424-e83c56de5b2e 7822456f-2ca2-2584-f424-e83c56de5b2e}" + testing.go:149: server-52e9f313-d0a2-1b7c-0541-bb792953c801: 2024-01-08T00:31:08.674Z [INFO] server-52e9f313-d0a2-1b7c-0541-bb792953c801: aborting pipeline replication: peer="{Voter server-7822456f-2ca2-2584-f424-e83c56de5b2e 7822456f-2ca2-2584-f424-e83c56de5b2e}" +--- PASS: TestRaft_Barrier (0.21s) === RUN TestRaft_VerifyLeader -testLoggerAdapter verbose: server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: 2025-02-09T06:52:33.364Z [INFO] server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b Address:7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b} {Suffrage:Voter ID:server-addd2fd6-4777-b1ca-3cfd-ebb7e113383d Address:addd2fd6-4777-b1ca-3cfd-ebb7e113383d} {Suffrage:Voter ID:server-f43b44e5-55d6-fdf6-eb31-2935691dbb35 Address:f43b44e5-55d6-fdf6-eb31-2935691dbb35}]" - testing.go:149: server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: 2025-02-09T06:52:33.364Z [INFO] server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b Address:7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b} {Suffrage:Voter ID:server-addd2fd6-4777-b1ca-3cfd-ebb7e113383d Address:addd2fd6-4777-b1ca-3cfd-ebb7e113383d} {Suffrage:Voter ID:server-f43b44e5-55d6-fdf6-eb31-2935691dbb35 Address:f43b44e5-55d6-fdf6-eb31-2935691dbb35}]" -testLoggerAdapter verbose: server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: 2025-02-09T06:52:33.364Z [INFO] server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: entering follower state: follower="Node at 7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b [Follower]" leader= -testLoggerAdapter verbose: server-addd2fd6-4777-b1ca-3cfd-ebb7e113383d: 2025-02-09T06:52:33.364Z [INFO] server-addd2fd6-4777-b1ca-3cfd-ebb7e113383d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b Address:7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b} {Suffrage:Voter ID:server-addd2fd6-4777-b1ca-3cfd-ebb7e113383d Address:addd2fd6-4777-b1ca-3cfd-ebb7e113383d} {Suffrage:Voter ID:server-f43b44e5-55d6-fdf6-eb31-2935691dbb35 Address:f43b44e5-55d6-fdf6-eb31-2935691dbb35}]" - testing.go:149: server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: 2025-02-09T06:52:33.364Z [INFO] server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: entering follower state: follower="Node at 7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b [Follower]" leader= - testing.go:149: server-addd2fd6-4777-b1ca-3cfd-ebb7e113383d: 2025-02-09T06:52:33.364Z [INFO] server-addd2fd6-4777-b1ca-3cfd-ebb7e113383d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b Address:7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b} {Suffrage:Voter ID:server-addd2fd6-4777-b1ca-3cfd-ebb7e113383d Address:addd2fd6-4777-b1ca-3cfd-ebb7e113383d} {Suffrage:Voter ID:server-f43b44e5-55d6-fdf6-eb31-2935691dbb35 Address:f43b44e5-55d6-fdf6-eb31-2935691dbb35}]" -testLoggerAdapter verbose: server-addd2fd6-4777-b1ca-3cfd-ebb7e113383d: 2025-02-09T06:52:33.365Z [INFO] server-addd2fd6-4777-b1ca-3cfd-ebb7e113383d: entering follower state: follower="Node at addd2fd6-4777-b1ca-3cfd-ebb7e113383d [Follower]" leader= - testing.go:149: server-addd2fd6-4777-b1ca-3cfd-ebb7e113383d: 2025-02-09T06:52:33.365Z [INFO] server-addd2fd6-4777-b1ca-3cfd-ebb7e113383d: entering follower state: follower="Node at addd2fd6-4777-b1ca-3cfd-ebb7e113383d [Follower]" leader= -testLoggerAdapter verbose: server-f43b44e5-55d6-fdf6-eb31-2935691dbb35: 2025-02-09T06:52:33.365Z [INFO] server-f43b44e5-55d6-fdf6-eb31-2935691dbb35: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b Address:7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b} {Suffrage:Voter ID:server-addd2fd6-4777-b1ca-3cfd-ebb7e113383d Address:addd2fd6-4777-b1ca-3cfd-ebb7e113383d} {Suffrage:Voter ID:server-f43b44e5-55d6-fdf6-eb31-2935691dbb35 Address:f43b44e5-55d6-fdf6-eb31-2935691dbb35}]" - testing.go:149: server-f43b44e5-55d6-fdf6-eb31-2935691dbb35: 2025-02-09T06:52:33.365Z [INFO] server-f43b44e5-55d6-fdf6-eb31-2935691dbb35: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b Address:7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b} {Suffrage:Voter ID:server-addd2fd6-4777-b1ca-3cfd-ebb7e113383d Address:addd2fd6-4777-b1ca-3cfd-ebb7e113383d} {Suffrage:Voter ID:server-f43b44e5-55d6-fdf6-eb31-2935691dbb35 Address:f43b44e5-55d6-fdf6-eb31-2935691dbb35}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:33.365Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:33.365Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-f43b44e5-55d6-fdf6-eb31-2935691dbb35: 2025-02-09T06:52:33.365Z [INFO] server-f43b44e5-55d6-fdf6-eb31-2935691dbb35: entering follower state: follower="Node at f43b44e5-55d6-fdf6-eb31-2935691dbb35 [Follower]" leader= - testing.go:149: server-f43b44e5-55d6-fdf6-eb31-2935691dbb35: 2025-02-09T06:52:33.365Z [INFO] server-f43b44e5-55d6-fdf6-eb31-2935691dbb35: entering follower state: follower="Node at f43b44e5-55d6-fdf6-eb31-2935691dbb35 [Follower]" leader= -testLoggerAdapter verbose: server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: 2025-02-09T06:52:33.421Z [WARN] server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: 2025-02-09T06:52:33.421Z [WARN] server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: 2025-02-09T06:52:33.421Z [INFO] server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: entering candidate state: node="Node at 7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b [Candidate]" term=2 - testing.go:149: server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: 2025-02-09T06:52:33.421Z [INFO] server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: entering candidate state: node="Node at 7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b [Candidate]" term=2 -testLoggerAdapter verbose: server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: 2025-02-09T06:52:33.421Z [INFO] server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: election won: tally=2 - testing.go:149: server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: 2025-02-09T06:52:33.421Z [INFO] server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: election won: tally=2 -testLoggerAdapter verbose: server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: 2025-02-09T06:52:33.421Z [INFO] server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: entering leader state: leader="Node at 7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b [Leader]" - testing.go:149: server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: 2025-02-09T06:52:33.421Z [INFO] server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: entering leader state: leader="Node at 7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b [Leader]" -testLoggerAdapter verbose: server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: 2025-02-09T06:52:33.421Z [INFO] server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: added peer, starting replication: peer=server-addd2fd6-4777-b1ca-3cfd-ebb7e113383d - testing.go:149: server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: 2025-02-09T06:52:33.421Z [INFO] server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: added peer, starting replication: peer=server-addd2fd6-4777-b1ca-3cfd-ebb7e113383d -testLoggerAdapter verbose: server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: 2025-02-09T06:52:33.421Z [INFO] server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: added peer, starting replication: peer=server-f43b44e5-55d6-fdf6-eb31-2935691dbb35 - testing.go:149: server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: 2025-02-09T06:52:33.421Z [INFO] server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: added peer, starting replication: peer=server-f43b44e5-55d6-fdf6-eb31-2935691dbb35 -testLoggerAdapter verbose: server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: 2025-02-09T06:52:33.421Z [INFO] server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: pipelining replication: peer="{Voter server-f43b44e5-55d6-fdf6-eb31-2935691dbb35 f43b44e5-55d6-fdf6-eb31-2935691dbb35}" - testing.go:149: server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: 2025-02-09T06:52:33.421Z [INFO] server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: pipelining replication: peer="{Voter server-f43b44e5-55d6-fdf6-eb31-2935691dbb35 f43b44e5-55d6-fdf6-eb31-2935691dbb35}" -testLoggerAdapter verbose: server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: 2025-02-09T06:52:33.422Z [INFO] server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: pipelining replication: peer="{Voter server-addd2fd6-4777-b1ca-3cfd-ebb7e113383d addd2fd6-4777-b1ca-3cfd-ebb7e113383d}" - testing.go:149: server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: 2025-02-09T06:52:33.422Z [INFO] server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: pipelining replication: peer="{Voter server-addd2fd6-4777-b1ca-3cfd-ebb7e113383d addd2fd6-4777-b1ca-3cfd-ebb7e113383d}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:33.527Z [INFO] stable state for Leader reached at 2025-02-09 06:52:33.421891365 +0000 UTC m=+14.400700130 (1 nodes), 56.738011ms from start of poll, 57.103172ms from cluster start. Timeout at 2025-02-09 06:52:33.52789275 +0000 UTC m=+14.506701520, 106.00139ms after stability - testing.go:149: cluster: 2025-02-09T06:52:33.527Z [INFO] stable state for Leader reached at 2025-02-09 06:52:33.421891365 +0000 UTC m=+14.400700130 (1 nodes), 56.738011ms from start of poll, 57.103172ms from cluster start. Timeout at 2025-02-09 06:52:33.52789275 +0000 UTC m=+14.506701520, 106.00139ms after stability -testLoggerAdapter verbose: server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: 2025-02-09T06:52:33.527Z [INFO] server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: aborting pipeline replication: peer="{Voter server-f43b44e5-55d6-fdf6-eb31-2935691dbb35 f43b44e5-55d6-fdf6-eb31-2935691dbb35}" - testing.go:149: server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: 2025-02-09T06:52:33.527Z [INFO] server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: aborting pipeline replication: peer="{Voter server-f43b44e5-55d6-fdf6-eb31-2935691dbb35 f43b44e5-55d6-fdf6-eb31-2935691dbb35}" -testLoggerAdapter verbose: server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: 2025-02-09T06:52:33.527Z [INFO] server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: aborting pipeline replication: peer="{Voter server-addd2fd6-4777-b1ca-3cfd-ebb7e113383d addd2fd6-4777-b1ca-3cfd-ebb7e113383d}" - testing.go:149: server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: 2025-02-09T06:52:33.527Z [INFO] server-7bf3ec9b-c077-5aa5-b3e7-004aa0f1520b: aborting pipeline replication: peer="{Voter server-addd2fd6-4777-b1ca-3cfd-ebb7e113383d addd2fd6-4777-b1ca-3cfd-ebb7e113383d}" ---- PASS: TestRaft_VerifyLeader (0.16s) +testLoggerAdapter verbose: server-188a475d-6dd8-002c-4d94-2cbadbb1b86c: 2024-01-08T00:31:08.675Z [INFO] server-188a475d-6dd8-002c-4d94-2cbadbb1b86c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-188a475d-6dd8-002c-4d94-2cbadbb1b86c Address:188a475d-6dd8-002c-4d94-2cbadbb1b86c} {Suffrage:Voter ID:server-f3e81a4e-2bee-4fd5-059a-5d2946712dc9 Address:f3e81a4e-2bee-4fd5-059a-5d2946712dc9} {Suffrage:Voter ID:server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff Address:f4e4e770-9a99-7a80-38b6-ff9dbe8637ff}]" + testing.go:149: server-188a475d-6dd8-002c-4d94-2cbadbb1b86c: 2024-01-08T00:31:08.675Z [INFO] server-188a475d-6dd8-002c-4d94-2cbadbb1b86c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-188a475d-6dd8-002c-4d94-2cbadbb1b86c Address:188a475d-6dd8-002c-4d94-2cbadbb1b86c} {Suffrage:Voter ID:server-f3e81a4e-2bee-4fd5-059a-5d2946712dc9 Address:f3e81a4e-2bee-4fd5-059a-5d2946712dc9} {Suffrage:Voter ID:server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff Address:f4e4e770-9a99-7a80-38b6-ff9dbe8637ff}]" +testLoggerAdapter verbose: server-f3e81a4e-2bee-4fd5-059a-5d2946712dc9: 2024-01-08T00:31:08.675Z [INFO] server-f3e81a4e-2bee-4fd5-059a-5d2946712dc9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-188a475d-6dd8-002c-4d94-2cbadbb1b86c Address:188a475d-6dd8-002c-4d94-2cbadbb1b86c} {Suffrage:Voter ID:server-f3e81a4e-2bee-4fd5-059a-5d2946712dc9 Address:f3e81a4e-2bee-4fd5-059a-5d2946712dc9} {Suffrage:Voter ID:server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff Address:f4e4e770-9a99-7a80-38b6-ff9dbe8637ff}]" + testing.go:149: server-f3e81a4e-2bee-4fd5-059a-5d2946712dc9: 2024-01-08T00:31:08.675Z [INFO] server-f3e81a4e-2bee-4fd5-059a-5d2946712dc9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-188a475d-6dd8-002c-4d94-2cbadbb1b86c Address:188a475d-6dd8-002c-4d94-2cbadbb1b86c} {Suffrage:Voter ID:server-f3e81a4e-2bee-4fd5-059a-5d2946712dc9 Address:f3e81a4e-2bee-4fd5-059a-5d2946712dc9} {Suffrage:Voter ID:server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff Address:f4e4e770-9a99-7a80-38b6-ff9dbe8637ff}]" +testLoggerAdapter verbose: server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: 2024-01-08T00:31:08.675Z [INFO] server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-188a475d-6dd8-002c-4d94-2cbadbb1b86c Address:188a475d-6dd8-002c-4d94-2cbadbb1b86c} {Suffrage:Voter ID:server-f3e81a4e-2bee-4fd5-059a-5d2946712dc9 Address:f3e81a4e-2bee-4fd5-059a-5d2946712dc9} {Suffrage:Voter ID:server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff Address:f4e4e770-9a99-7a80-38b6-ff9dbe8637ff}]" + testing.go:149: server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: 2024-01-08T00:31:08.675Z [INFO] server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-188a475d-6dd8-002c-4d94-2cbadbb1b86c Address:188a475d-6dd8-002c-4d94-2cbadbb1b86c} {Suffrage:Voter ID:server-f3e81a4e-2bee-4fd5-059a-5d2946712dc9 Address:f3e81a4e-2bee-4fd5-059a-5d2946712dc9} {Suffrage:Voter ID:server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff Address:f4e4e770-9a99-7a80-38b6-ff9dbe8637ff}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:08.675Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:08.675Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-188a475d-6dd8-002c-4d94-2cbadbb1b86c: 2024-01-08T00:31:08.675Z [INFO] server-188a475d-6dd8-002c-4d94-2cbadbb1b86c: entering follower state: follower="Node at 188a475d-6dd8-002c-4d94-2cbadbb1b86c [Follower]" leader= + testing.go:149: server-188a475d-6dd8-002c-4d94-2cbadbb1b86c: 2024-01-08T00:31:08.675Z [INFO] server-188a475d-6dd8-002c-4d94-2cbadbb1b86c: entering follower state: follower="Node at 188a475d-6dd8-002c-4d94-2cbadbb1b86c [Follower]" leader= +testLoggerAdapter verbose: server-f3e81a4e-2bee-4fd5-059a-5d2946712dc9: 2024-01-08T00:31:08.675Z [INFO] server-f3e81a4e-2bee-4fd5-059a-5d2946712dc9: entering follower state: follower="Node at f3e81a4e-2bee-4fd5-059a-5d2946712dc9 [Follower]" leader= + testing.go:149: server-f3e81a4e-2bee-4fd5-059a-5d2946712dc9: 2024-01-08T00:31:08.675Z [INFO] server-f3e81a4e-2bee-4fd5-059a-5d2946712dc9: entering follower state: follower="Node at f3e81a4e-2bee-4fd5-059a-5d2946712dc9 [Follower]" leader= +testLoggerAdapter verbose: server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: 2024-01-08T00:31:08.675Z [INFO] server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: entering follower state: follower="Node at f4e4e770-9a99-7a80-38b6-ff9dbe8637ff [Follower]" leader= + testing.go:149: server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: 2024-01-08T00:31:08.675Z [INFO] server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: entering follower state: follower="Node at f4e4e770-9a99-7a80-38b6-ff9dbe8637ff [Follower]" leader= +testLoggerAdapter verbose: server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: 2024-01-08T00:31:08.735Z [WARN] server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: 2024-01-08T00:31:08.735Z [WARN] server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: 2024-01-08T00:31:08.735Z [INFO] server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: entering candidate state: node="Node at f4e4e770-9a99-7a80-38b6-ff9dbe8637ff [Candidate]" term=2 + testing.go:149: server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: 2024-01-08T00:31:08.735Z [INFO] server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: entering candidate state: node="Node at f4e4e770-9a99-7a80-38b6-ff9dbe8637ff [Candidate]" term=2 +testLoggerAdapter verbose: server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: 2024-01-08T00:31:08.735Z [INFO] server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: election won: tally=2 + testing.go:149: server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: 2024-01-08T00:31:08.735Z [INFO] server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: election won: tally=2 +testLoggerAdapter verbose: server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: 2024-01-08T00:31:08.735Z [INFO] server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: entering leader state: leader="Node at f4e4e770-9a99-7a80-38b6-ff9dbe8637ff [Leader]" + testing.go:149: server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: 2024-01-08T00:31:08.735Z [INFO] server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: entering leader state: leader="Node at f4e4e770-9a99-7a80-38b6-ff9dbe8637ff [Leader]" +testLoggerAdapter verbose: server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: 2024-01-08T00:31:08.735Z [INFO] server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: added peer, starting replication: peer=server-188a475d-6dd8-002c-4d94-2cbadbb1b86c + testing.go:149: server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: 2024-01-08T00:31:08.735Z [INFO] server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: added peer, starting replication: peer=server-188a475d-6dd8-002c-4d94-2cbadbb1b86c +testLoggerAdapter verbose: server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: 2024-01-08T00:31:08.736Z [INFO] server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: added peer, starting replication: peer=server-f3e81a4e-2bee-4fd5-059a-5d2946712dc9 + testing.go:149: server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: 2024-01-08T00:31:08.736Z [INFO] server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: added peer, starting replication: peer=server-f3e81a4e-2bee-4fd5-059a-5d2946712dc9 +testLoggerAdapter verbose: server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: 2024-01-08T00:31:08.736Z [INFO] server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: pipelining replication: peer="{Voter server-f3e81a4e-2bee-4fd5-059a-5d2946712dc9 f3e81a4e-2bee-4fd5-059a-5d2946712dc9}" + testing.go:149: server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: 2024-01-08T00:31:08.736Z [INFO] server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: pipelining replication: peer="{Voter server-f3e81a4e-2bee-4fd5-059a-5d2946712dc9 f3e81a4e-2bee-4fd5-059a-5d2946712dc9}" +testLoggerAdapter verbose: server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: 2024-01-08T00:31:08.736Z [INFO] server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: pipelining replication: peer="{Voter server-188a475d-6dd8-002c-4d94-2cbadbb1b86c 188a475d-6dd8-002c-4d94-2cbadbb1b86c}" + testing.go:149: server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: 2024-01-08T00:31:08.736Z [INFO] server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: pipelining replication: peer="{Voter server-188a475d-6dd8-002c-4d94-2cbadbb1b86c 188a475d-6dd8-002c-4d94-2cbadbb1b86c}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:08.841Z [INFO] stable state for Leader reached at 2024-01-08 00:31:08.736276817 +0000 UTC m=+14.783165863 (1 nodes), 60.651818ms from start of poll, 61.035272ms from cluster start. Timeout at 2024-01-08 00:31:08.841319059 +0000 UTC m=+14.888208159, 105.042296ms after stability + testing.go:149: cluster: 2024-01-08T00:31:08.841Z [INFO] stable state for Leader reached at 2024-01-08 00:31:08.736276817 +0000 UTC m=+14.783165863 (1 nodes), 60.651818ms from start of poll, 61.035272ms from cluster start. Timeout at 2024-01-08 00:31:08.841319059 +0000 UTC m=+14.888208159, 105.042296ms after stability +testLoggerAdapter verbose: server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: 2024-01-08T00:31:08.841Z [INFO] server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: aborting pipeline replication: peer="{Voter server-188a475d-6dd8-002c-4d94-2cbadbb1b86c 188a475d-6dd8-002c-4d94-2cbadbb1b86c}" + testing.go:149: server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: 2024-01-08T00:31:08.841Z [INFO] server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: aborting pipeline replication: peer="{Voter server-188a475d-6dd8-002c-4d94-2cbadbb1b86c 188a475d-6dd8-002c-4d94-2cbadbb1b86c}" +testLoggerAdapter verbose: server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: 2024-01-08T00:31:08.841Z [INFO] server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: aborting pipeline replication: peer="{Voter server-f3e81a4e-2bee-4fd5-059a-5d2946712dc9 f3e81a4e-2bee-4fd5-059a-5d2946712dc9}" + testing.go:149: server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: 2024-01-08T00:31:08.841Z [INFO] server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: aborting pipeline replication: peer="{Voter server-f3e81a4e-2bee-4fd5-059a-5d2946712dc9 f3e81a4e-2bee-4fd5-059a-5d2946712dc9}" +testLoggerAdapter verbose: server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: 2024-01-08T00:31:08.892Z [ERROR] server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: failed to heartbeat to: peer=188a475d-6dd8-002c-4d94-2cbadbb1b86c error="command timed out" + testing.go:149: server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: 2024-01-08T00:31:08.892Z [ERROR] server-f4e4e770-9a99-7a80-38b6-ff9dbe8637ff: failed to heartbeat to: peer=188a475d-6dd8-002c-4d94-2cbadbb1b86c error="command timed out" +--- PASS: TestRaft_VerifyLeader (0.22s) === RUN TestRaft_VerifyLeader_Single -testLoggerAdapter verbose: server-6420c583-5e98-6da3-4606-3a28e7c27404: 2025-02-09T06:52:33.528Z [INFO] server-6420c583-5e98-6da3-4606-3a28e7c27404: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6420c583-5e98-6da3-4606-3a28e7c27404 Address:6420c583-5e98-6da3-4606-3a28e7c27404}]" - testing.go:149: server-6420c583-5e98-6da3-4606-3a28e7c27404: 2025-02-09T06:52:33.528Z [INFO] server-6420c583-5e98-6da3-4606-3a28e7c27404: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6420c583-5e98-6da3-4606-3a28e7c27404 Address:6420c583-5e98-6da3-4606-3a28e7c27404}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:33.528Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:33.528Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-6420c583-5e98-6da3-4606-3a28e7c27404: 2025-02-09T06:52:33.528Z [INFO] server-6420c583-5e98-6da3-4606-3a28e7c27404: entering follower state: follower="Node at 6420c583-5e98-6da3-4606-3a28e7c27404 [Follower]" leader= - testing.go:149: server-6420c583-5e98-6da3-4606-3a28e7c27404: 2025-02-09T06:52:33.528Z [INFO] server-6420c583-5e98-6da3-4606-3a28e7c27404: entering follower state: follower="Node at 6420c583-5e98-6da3-4606-3a28e7c27404 [Follower]" leader= -testLoggerAdapter verbose: server-6420c583-5e98-6da3-4606-3a28e7c27404: 2025-02-09T06:52:33.602Z [WARN] server-6420c583-5e98-6da3-4606-3a28e7c27404: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-6420c583-5e98-6da3-4606-3a28e7c27404: 2025-02-09T06:52:33.602Z [WARN] server-6420c583-5e98-6da3-4606-3a28e7c27404: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-6420c583-5e98-6da3-4606-3a28e7c27404: 2025-02-09T06:52:33.602Z [INFO] server-6420c583-5e98-6da3-4606-3a28e7c27404: entering candidate state: node="Node at 6420c583-5e98-6da3-4606-3a28e7c27404 [Candidate]" term=2 - testing.go:149: server-6420c583-5e98-6da3-4606-3a28e7c27404: 2025-02-09T06:52:33.602Z [INFO] server-6420c583-5e98-6da3-4606-3a28e7c27404: entering candidate state: node="Node at 6420c583-5e98-6da3-4606-3a28e7c27404 [Candidate]" term=2 -testLoggerAdapter verbose: server-6420c583-5e98-6da3-4606-3a28e7c27404: 2025-02-09T06:52:33.602Z [INFO] server-6420c583-5e98-6da3-4606-3a28e7c27404: election won: tally=1 - testing.go:149: server-6420c583-5e98-6da3-4606-3a28e7c27404: 2025-02-09T06:52:33.602Z [INFO] server-6420c583-5e98-6da3-4606-3a28e7c27404: election won: tally=1 -testLoggerAdapter verbose: server-6420c583-5e98-6da3-4606-3a28e7c27404: 2025-02-09T06:52:33.602Z [INFO] server-6420c583-5e98-6da3-4606-3a28e7c27404: entering leader state: leader="Node at 6420c583-5e98-6da3-4606-3a28e7c27404 [Leader]" - testing.go:149: server-6420c583-5e98-6da3-4606-3a28e7c27404: 2025-02-09T06:52:33.602Z [INFO] server-6420c583-5e98-6da3-4606-3a28e7c27404: entering leader state: leader="Node at 6420c583-5e98-6da3-4606-3a28e7c27404 [Leader]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:33.707Z [INFO] stable state for Leader reached at 2025-02-09 06:52:33.602260638 +0000 UTC m=+14.581069404 (1 nodes), 73.468308ms from start of poll, 73.613404ms from cluster start. Timeout at 2025-02-09 06:52:33.70748026 +0000 UTC m=+14.686289034, 105.21963ms after stability - testing.go:149: cluster: 2025-02-09T06:52:33.707Z [INFO] stable state for Leader reached at 2025-02-09 06:52:33.602260638 +0000 UTC m=+14.581069404 (1 nodes), 73.468308ms from start of poll, 73.613404ms from cluster start. Timeout at 2025-02-09 06:52:33.70748026 +0000 UTC m=+14.686289034, 105.21963ms after stability ---- PASS: TestRaft_VerifyLeader_Single (0.18s) +testLoggerAdapter verbose: server-6cca274f-8eff-4373-bae8-4a685a54ea06: 2024-01-08T00:31:08.893Z [INFO] server-6cca274f-8eff-4373-bae8-4a685a54ea06: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6cca274f-8eff-4373-bae8-4a685a54ea06 Address:6cca274f-8eff-4373-bae8-4a685a54ea06}]" + testing.go:149: server-6cca274f-8eff-4373-bae8-4a685a54ea06: 2024-01-08T00:31:08.893Z [INFO] server-6cca274f-8eff-4373-bae8-4a685a54ea06: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6cca274f-8eff-4373-bae8-4a685a54ea06 Address:6cca274f-8eff-4373-bae8-4a685a54ea06}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:08.893Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:08.893Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-6cca274f-8eff-4373-bae8-4a685a54ea06: 2024-01-08T00:31:08.893Z [INFO] server-6cca274f-8eff-4373-bae8-4a685a54ea06: entering follower state: follower="Node at 6cca274f-8eff-4373-bae8-4a685a54ea06 [Follower]" leader= + testing.go:149: server-6cca274f-8eff-4373-bae8-4a685a54ea06: 2024-01-08T00:31:08.893Z [INFO] server-6cca274f-8eff-4373-bae8-4a685a54ea06: entering follower state: follower="Node at 6cca274f-8eff-4373-bae8-4a685a54ea06 [Follower]" leader= +testLoggerAdapter verbose: server-6cca274f-8eff-4373-bae8-4a685a54ea06: 2024-01-08T00:31:08.990Z [WARN] server-6cca274f-8eff-4373-bae8-4a685a54ea06: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-6cca274f-8eff-4373-bae8-4a685a54ea06: 2024-01-08T00:31:08.990Z [WARN] server-6cca274f-8eff-4373-bae8-4a685a54ea06: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-6cca274f-8eff-4373-bae8-4a685a54ea06: 2024-01-08T00:31:08.990Z [INFO] server-6cca274f-8eff-4373-bae8-4a685a54ea06: entering candidate state: node="Node at 6cca274f-8eff-4373-bae8-4a685a54ea06 [Candidate]" term=2 + testing.go:149: server-6cca274f-8eff-4373-bae8-4a685a54ea06: 2024-01-08T00:31:08.990Z [INFO] server-6cca274f-8eff-4373-bae8-4a685a54ea06: entering candidate state: node="Node at 6cca274f-8eff-4373-bae8-4a685a54ea06 [Candidate]" term=2 +testLoggerAdapter verbose: server-6cca274f-8eff-4373-bae8-4a685a54ea06: 2024-01-08T00:31:08.990Z [INFO] server-6cca274f-8eff-4373-bae8-4a685a54ea06: election won: tally=1 + testing.go:149: server-6cca274f-8eff-4373-bae8-4a685a54ea06: 2024-01-08T00:31:08.990Z [INFO] server-6cca274f-8eff-4373-bae8-4a685a54ea06: election won: tally=1 +testLoggerAdapter verbose: server-6cca274f-8eff-4373-bae8-4a685a54ea06: 2024-01-08T00:31:08.990Z [INFO] server-6cca274f-8eff-4373-bae8-4a685a54ea06: entering leader state: leader="Node at 6cca274f-8eff-4373-bae8-4a685a54ea06 [Leader]" + testing.go:149: server-6cca274f-8eff-4373-bae8-4a685a54ea06: 2024-01-08T00:31:08.990Z [INFO] server-6cca274f-8eff-4373-bae8-4a685a54ea06: entering leader state: leader="Node at 6cca274f-8eff-4373-bae8-4a685a54ea06 [Leader]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:09.097Z [INFO] stable state for Leader reached at 2024-01-08 00:31:08.990954156 +0000 UTC m=+15.037843199 (1 nodes), 97.734594ms from start of poll, 97.943889ms from cluster start. Timeout at 2024-01-08 00:31:09.096958909 +0000 UTC m=+15.143848004, 106.004805ms after stability + testing.go:149: cluster: 2024-01-08T00:31:09.097Z [INFO] stable state for Leader reached at 2024-01-08 00:31:08.990954156 +0000 UTC m=+15.037843199 (1 nodes), 97.734594ms from start of poll, 97.943889ms from cluster start. Timeout at 2024-01-08 00:31:09.096958909 +0000 UTC m=+15.143848004, 106.004805ms after stability +--- PASS: TestRaft_VerifyLeader_Single (0.20s) === RUN TestRaft_VerifyLeader_Fail -testLoggerAdapter verbose: server-9e342e83-73bf-5e69-a63f-526682b23fce: 2025-02-09T06:52:33.708Z [INFO] server-9e342e83-73bf-5e69-a63f-526682b23fce: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9e342e83-73bf-5e69-a63f-526682b23fce Address:9e342e83-73bf-5e69-a63f-526682b23fce} {Suffrage:Voter ID:server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4 Address:f7c6d453-f2b6-2581-3bc1-7202fbec38c4}]" - testing.go:149: server-9e342e83-73bf-5e69-a63f-526682b23fce: 2025-02-09T06:52:33.708Z [INFO] server-9e342e83-73bf-5e69-a63f-526682b23fce: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9e342e83-73bf-5e69-a63f-526682b23fce Address:9e342e83-73bf-5e69-a63f-526682b23fce} {Suffrage:Voter ID:server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4 Address:f7c6d453-f2b6-2581-3bc1-7202fbec38c4}]" -testLoggerAdapter verbose: server-9e342e83-73bf-5e69-a63f-526682b23fce: 2025-02-09T06:52:33.708Z [INFO] server-9e342e83-73bf-5e69-a63f-526682b23fce: entering follower state: follower="Node at 9e342e83-73bf-5e69-a63f-526682b23fce [Follower]" leader= - testing.go:149: server-9e342e83-73bf-5e69-a63f-526682b23fce: 2025-02-09T06:52:33.708Z [INFO] server-9e342e83-73bf-5e69-a63f-526682b23fce: entering follower state: follower="Node at 9e342e83-73bf-5e69-a63f-526682b23fce [Follower]" leader= -testLoggerAdapter verbose: server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: 2025-02-09T06:52:33.708Z [INFO] server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9e342e83-73bf-5e69-a63f-526682b23fce Address:9e342e83-73bf-5e69-a63f-526682b23fce} {Suffrage:Voter ID:server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4 Address:f7c6d453-f2b6-2581-3bc1-7202fbec38c4}]" - testing.go:149: server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: 2025-02-09T06:52:33.708Z [INFO] server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9e342e83-73bf-5e69-a63f-526682b23fce Address:9e342e83-73bf-5e69-a63f-526682b23fce} {Suffrage:Voter ID:server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4 Address:f7c6d453-f2b6-2581-3bc1-7202fbec38c4}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:33.708Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:33.708Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: 2025-02-09T06:52:33.708Z [INFO] server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: entering follower state: follower="Node at f7c6d453-f2b6-2581-3bc1-7202fbec38c4 [Follower]" leader= - testing.go:149: server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: 2025-02-09T06:52:33.708Z [INFO] server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: entering follower state: follower="Node at f7c6d453-f2b6-2581-3bc1-7202fbec38c4 [Follower]" leader= -testLoggerAdapter verbose: server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: 2025-02-09T06:52:33.779Z [WARN] server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: 2025-02-09T06:52:33.779Z [WARN] server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: 2025-02-09T06:52:33.779Z [INFO] server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: entering candidate state: node="Node at f7c6d453-f2b6-2581-3bc1-7202fbec38c4 [Candidate]" term=2 - testing.go:149: server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: 2025-02-09T06:52:33.779Z [INFO] server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: entering candidate state: node="Node at f7c6d453-f2b6-2581-3bc1-7202fbec38c4 [Candidate]" term=2 -testLoggerAdapter verbose: server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: 2025-02-09T06:52:33.779Z [INFO] server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: election won: tally=2 - testing.go:149: server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: 2025-02-09T06:52:33.779Z [INFO] server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: election won: tally=2 -testLoggerAdapter verbose: server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: 2025-02-09T06:52:33.779Z [INFO] server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: entering leader state: leader="Node at f7c6d453-f2b6-2581-3bc1-7202fbec38c4 [Leader]" - testing.go:149: server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: 2025-02-09T06:52:33.779Z [INFO] server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: entering leader state: leader="Node at f7c6d453-f2b6-2581-3bc1-7202fbec38c4 [Leader]" -testLoggerAdapter verbose: server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: 2025-02-09T06:52:33.779Z [INFO] server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: added peer, starting replication: peer=server-9e342e83-73bf-5e69-a63f-526682b23fce - testing.go:149: server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: 2025-02-09T06:52:33.779Z [INFO] server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: added peer, starting replication: peer=server-9e342e83-73bf-5e69-a63f-526682b23fce -testLoggerAdapter verbose: server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: 2025-02-09T06:52:33.779Z [INFO] server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: pipelining replication: peer="{Voter server-9e342e83-73bf-5e69-a63f-526682b23fce 9e342e83-73bf-5e69-a63f-526682b23fce}" - testing.go:149: server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: 2025-02-09T06:52:33.779Z [INFO] server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: pipelining replication: peer="{Voter server-9e342e83-73bf-5e69-a63f-526682b23fce 9e342e83-73bf-5e69-a63f-526682b23fce}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:33.884Z [INFO] stable state for Leader reached at 2025-02-09 06:52:33.779450425 +0000 UTC m=+14.758259193 (1 nodes), 71.137839ms from start of poll, 71.385887ms from cluster start. Timeout at 2025-02-09 06:52:33.884900735 +0000 UTC m=+14.863709514, 105.450321ms after stability - testing.go:149: cluster: 2025-02-09T06:52:33.884Z [INFO] stable state for Leader reached at 2025-02-09 06:52:33.779450425 +0000 UTC m=+14.758259193 (1 nodes), 71.137839ms from start of poll, 71.385887ms from cluster start. Timeout at 2025-02-09 06:52:33.884900735 +0000 UTC m=+14.863709514, 105.450321ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:33.884Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T06:52:33.884Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:33.990Z [INFO] stable state for Follower reached at 2025-02-09 06:52:33.884982604 +0000 UTC m=+14.863791369 (1 nodes), 444ns from start of poll, 176.918063ms from cluster start. Timeout at 2025-02-09 06:52:33.990525314 +0000 UTC m=+14.969334072, 105.542703ms after stability - testing.go:149: cluster: 2025-02-09T06:52:33.990Z [INFO] stable state for Follower reached at 2025-02-09 06:52:33.884982604 +0000 UTC m=+14.863791369 (1 nodes), 444ns from start of poll, 176.918063ms from cluster start. Timeout at 2025-02-09 06:52:33.990525314 +0000 UTC m=+14.969334072, 105.542703ms after stability -testLoggerAdapter verbose: server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: 2025-02-09T06:52:33.990Z [WARN] server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: new leader elected, stepping down - testing.go:149: server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: 2025-02-09T06:52:33.990Z [WARN] server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: new leader elected, stepping down -testLoggerAdapter verbose: server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: 2025-02-09T06:52:33.990Z [INFO] server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: entering follower state: follower="Node at f7c6d453-f2b6-2581-3bc1-7202fbec38c4 [Follower]" leader= - testing.go:149: server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: 2025-02-09T06:52:33.990Z [INFO] server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: entering follower state: follower="Node at f7c6d453-f2b6-2581-3bc1-7202fbec38c4 [Follower]" leader= -testLoggerAdapter verbose: server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: 2025-02-09T06:52:33.990Z [INFO] server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: aborting pipeline replication: peer="{Voter server-9e342e83-73bf-5e69-a63f-526682b23fce 9e342e83-73bf-5e69-a63f-526682b23fce}" - testing.go:149: server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: 2025-02-09T06:52:33.990Z [INFO] server-f7c6d453-f2b6-2581-3bc1-7202fbec38c4: aborting pipeline replication: peer="{Voter server-9e342e83-73bf-5e69-a63f-526682b23fce 9e342e83-73bf-5e69-a63f-526682b23fce}" ---- PASS: TestRaft_VerifyLeader_Fail (0.28s) +testLoggerAdapter verbose: server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: 2024-01-08T00:31:09.097Z [INFO] server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1 Address:03f1d90c-c4d6-545b-5b55-dee0fcac31b1} {Suffrage:Voter ID:server-06abc411-bf4b-1321-e69e-5851116249c5 Address:06abc411-bf4b-1321-e69e-5851116249c5}]" + testing.go:149: server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: 2024-01-08T00:31:09.097Z [INFO] server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1 Address:03f1d90c-c4d6-545b-5b55-dee0fcac31b1} {Suffrage:Voter ID:server-06abc411-bf4b-1321-e69e-5851116249c5 Address:06abc411-bf4b-1321-e69e-5851116249c5}]" +testLoggerAdapter verbose: server-06abc411-bf4b-1321-e69e-5851116249c5: 2024-01-08T00:31:09.097Z [INFO] server-06abc411-bf4b-1321-e69e-5851116249c5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1 Address:03f1d90c-c4d6-545b-5b55-dee0fcac31b1} {Suffrage:Voter ID:server-06abc411-bf4b-1321-e69e-5851116249c5 Address:06abc411-bf4b-1321-e69e-5851116249c5}]" + testing.go:149: server-06abc411-bf4b-1321-e69e-5851116249c5: 2024-01-08T00:31:09.097Z [INFO] server-06abc411-bf4b-1321-e69e-5851116249c5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1 Address:03f1d90c-c4d6-545b-5b55-dee0fcac31b1} {Suffrage:Voter ID:server-06abc411-bf4b-1321-e69e-5851116249c5 Address:06abc411-bf4b-1321-e69e-5851116249c5}]" +testLoggerAdapter verbose: server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: 2024-01-08T00:31:09.097Z [INFO] server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: entering follower state: follower="Node at 03f1d90c-c4d6-545b-5b55-dee0fcac31b1 [Follower]" leader= +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:09.098Z [INFO] starting stability test: raft-state=Leader + testing.go:149: server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: 2024-01-08T00:31:09.097Z [INFO] server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: entering follower state: follower="Node at 03f1d90c-c4d6-545b-5b55-dee0fcac31b1 [Follower]" leader= +testLoggerAdapter verbose: server-06abc411-bf4b-1321-e69e-5851116249c5: 2024-01-08T00:31:09.098Z [INFO] server-06abc411-bf4b-1321-e69e-5851116249c5: entering follower state: follower="Node at 06abc411-bf4b-1321-e69e-5851116249c5 [Follower]" leader= + testing.go:149: server-06abc411-bf4b-1321-e69e-5851116249c5: 2024-01-08T00:31:09.098Z [INFO] server-06abc411-bf4b-1321-e69e-5851116249c5: entering follower state: follower="Node at 06abc411-bf4b-1321-e69e-5851116249c5 [Follower]" leader= + testing.go:149: cluster: 2024-01-08T00:31:09.098Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: 2024-01-08T00:31:09.153Z [WARN] server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: 2024-01-08T00:31:09.153Z [WARN] server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: 2024-01-08T00:31:09.153Z [INFO] server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: entering candidate state: node="Node at 03f1d90c-c4d6-545b-5b55-dee0fcac31b1 [Candidate]" term=2 + testing.go:149: server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: 2024-01-08T00:31:09.153Z [INFO] server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: entering candidate state: node="Node at 03f1d90c-c4d6-545b-5b55-dee0fcac31b1 [Candidate]" term=2 +testLoggerAdapter verbose: server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: 2024-01-08T00:31:09.154Z [INFO] server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: election won: tally=2 + testing.go:149: server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: 2024-01-08T00:31:09.154Z [INFO] server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: election won: tally=2 +testLoggerAdapter verbose: server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: 2024-01-08T00:31:09.154Z [INFO] server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: entering leader state: leader="Node at 03f1d90c-c4d6-545b-5b55-dee0fcac31b1 [Leader]" + testing.go:149: server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: 2024-01-08T00:31:09.154Z [INFO] server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: entering leader state: leader="Node at 03f1d90c-c4d6-545b-5b55-dee0fcac31b1 [Leader]" +testLoggerAdapter verbose: server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: 2024-01-08T00:31:09.154Z [INFO] server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: added peer, starting replication: peer=server-06abc411-bf4b-1321-e69e-5851116249c5 + testing.go:149: server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: 2024-01-08T00:31:09.154Z [INFO] server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: added peer, starting replication: peer=server-06abc411-bf4b-1321-e69e-5851116249c5 +testLoggerAdapter verbose: server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: 2024-01-08T00:31:09.154Z [INFO] server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: pipelining replication: peer="{Voter server-06abc411-bf4b-1321-e69e-5851116249c5 06abc411-bf4b-1321-e69e-5851116249c5}" + testing.go:149: server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: 2024-01-08T00:31:09.154Z [INFO] server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: pipelining replication: peer="{Voter server-06abc411-bf4b-1321-e69e-5851116249c5 06abc411-bf4b-1321-e69e-5851116249c5}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:09.259Z [INFO] stable state for Leader reached at 2024-01-08 00:31:09.154274902 +0000 UTC m=+15.201163949 (1 nodes), 56.176291ms from start of poll, 56.550823ms from cluster start. Timeout at 2024-01-08 00:31:09.259765366 +0000 UTC m=+15.306654414, 105.490465ms after stability + testing.go:149: cluster: 2024-01-08T00:31:09.259Z [INFO] stable state for Leader reached at 2024-01-08 00:31:09.154274902 +0000 UTC m=+15.201163949 (1 nodes), 56.176291ms from start of poll, 56.550823ms from cluster start. Timeout at 2024-01-08 00:31:09.259765366 +0000 UTC m=+15.306654414, 105.490465ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:09.259Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T00:31:09.259Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:09.366Z [INFO] stable state for Follower reached at 2024-01-08 00:31:09.259905116 +0000 UTC m=+15.306794159 (1 nodes), 1.84µs from start of poll, 162.181033ms from cluster start. Timeout at 2024-01-08 00:31:09.365980386 +0000 UTC m=+15.412869751, 106.075592ms after stability + testing.go:149: cluster: 2024-01-08T00:31:09.366Z [INFO] stable state for Follower reached at 2024-01-08 00:31:09.259905116 +0000 UTC m=+15.306794159 (1 nodes), 1.84µs from start of poll, 162.181033ms from cluster start. Timeout at 2024-01-08 00:31:09.365980386 +0000 UTC m=+15.412869751, 106.075592ms after stability +testLoggerAdapter verbose: server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: 2024-01-08T00:31:09.366Z [WARN] server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: new leader elected, stepping down + testing.go:149: server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: 2024-01-08T00:31:09.366Z [WARN] server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: new leader elected, stepping down +testLoggerAdapter verbose: server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: 2024-01-08T00:31:09.366Z [INFO] server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: entering follower state: follower="Node at 03f1d90c-c4d6-545b-5b55-dee0fcac31b1 [Follower]" leader= + testing.go:149: server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: 2024-01-08T00:31:09.366Z [INFO] server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: entering follower state: follower="Node at 03f1d90c-c4d6-545b-5b55-dee0fcac31b1 [Follower]" leader= +testLoggerAdapter verbose: server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: 2024-01-08T00:31:09.366Z [INFO] server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: aborting pipeline replication: peer="{Voter server-06abc411-bf4b-1321-e69e-5851116249c5 06abc411-bf4b-1321-e69e-5851116249c5}" + testing.go:149: server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: 2024-01-08T00:31:09.366Z [INFO] server-03f1d90c-c4d6-545b-5b55-dee0fcac31b1: aborting pipeline replication: peer="{Voter server-06abc411-bf4b-1321-e69e-5851116249c5 06abc411-bf4b-1321-e69e-5851116249c5}" +--- PASS: TestRaft_VerifyLeader_Fail (0.27s) === RUN TestRaft_VerifyLeader_PartialConnect -testLoggerAdapter verbose: server-9d1599f6-da3c-0851-04df-2cc5787ef026: 2025-02-09T06:52:33.991Z [INFO] server-9d1599f6-da3c-0851-04df-2cc5787ef026: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9d1599f6-da3c-0851-04df-2cc5787ef026 Address:9d1599f6-da3c-0851-04df-2cc5787ef026} {Suffrage:Voter ID:server-407cc3b4-66c0-c34b-5a67-82fda2c709c1 Address:407cc3b4-66c0-c34b-5a67-82fda2c709c1} {Suffrage:Voter ID:server-df9f2c72-97eb-acff-c8ad-75ebb50a00ba Address:df9f2c72-97eb-acff-c8ad-75ebb50a00ba}]" - testing.go:149: server-9d1599f6-da3c-0851-04df-2cc5787ef026: 2025-02-09T06:52:33.991Z [INFO] server-9d1599f6-da3c-0851-04df-2cc5787ef026: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9d1599f6-da3c-0851-04df-2cc5787ef026 Address:9d1599f6-da3c-0851-04df-2cc5787ef026} {Suffrage:Voter ID:server-407cc3b4-66c0-c34b-5a67-82fda2c709c1 Address:407cc3b4-66c0-c34b-5a67-82fda2c709c1} {Suffrage:Voter ID:server-df9f2c72-97eb-acff-c8ad-75ebb50a00ba Address:df9f2c72-97eb-acff-c8ad-75ebb50a00ba}]" -testLoggerAdapter verbose: server-9d1599f6-da3c-0851-04df-2cc5787ef026: 2025-02-09T06:52:33.991Z [INFO] server-9d1599f6-da3c-0851-04df-2cc5787ef026: entering follower state: follower="Node at 9d1599f6-da3c-0851-04df-2cc5787ef026 [Follower]" leader= - testing.go:149: server-9d1599f6-da3c-0851-04df-2cc5787ef026: 2025-02-09T06:52:33.991Z [INFO] server-9d1599f6-da3c-0851-04df-2cc5787ef026: entering follower state: follower="Node at 9d1599f6-da3c-0851-04df-2cc5787ef026 [Follower]" leader= -testLoggerAdapter verbose: server-407cc3b4-66c0-c34b-5a67-82fda2c709c1: 2025-02-09T06:52:33.991Z [INFO] server-407cc3b4-66c0-c34b-5a67-82fda2c709c1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9d1599f6-da3c-0851-04df-2cc5787ef026 Address:9d1599f6-da3c-0851-04df-2cc5787ef026} {Suffrage:Voter ID:server-407cc3b4-66c0-c34b-5a67-82fda2c709c1 Address:407cc3b4-66c0-c34b-5a67-82fda2c709c1} {Suffrage:Voter ID:server-df9f2c72-97eb-acff-c8ad-75ebb50a00ba Address:df9f2c72-97eb-acff-c8ad-75ebb50a00ba}]" - testing.go:149: server-407cc3b4-66c0-c34b-5a67-82fda2c709c1: 2025-02-09T06:52:33.991Z [INFO] server-407cc3b4-66c0-c34b-5a67-82fda2c709c1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9d1599f6-da3c-0851-04df-2cc5787ef026 Address:9d1599f6-da3c-0851-04df-2cc5787ef026} {Suffrage:Voter ID:server-407cc3b4-66c0-c34b-5a67-82fda2c709c1 Address:407cc3b4-66c0-c34b-5a67-82fda2c709c1} {Suffrage:Voter ID:server-df9f2c72-97eb-acff-c8ad-75ebb50a00ba Address:df9f2c72-97eb-acff-c8ad-75ebb50a00ba}]" -testLoggerAdapter verbose: server-407cc3b4-66c0-c34b-5a67-82fda2c709c1: 2025-02-09T06:52:33.991Z [INFO] server-407cc3b4-66c0-c34b-5a67-82fda2c709c1: entering follower state: follower="Node at 407cc3b4-66c0-c34b-5a67-82fda2c709c1 [Follower]" leader= -testLoggerAdapter verbose: server-df9f2c72-97eb-acff-c8ad-75ebb50a00ba: 2025-02-09T06:52:33.991Z [INFO] server-df9f2c72-97eb-acff-c8ad-75ebb50a00ba: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9d1599f6-da3c-0851-04df-2cc5787ef026 Address:9d1599f6-da3c-0851-04df-2cc5787ef026} {Suffrage:Voter ID:server-407cc3b4-66c0-c34b-5a67-82fda2c709c1 Address:407cc3b4-66c0-c34b-5a67-82fda2c709c1} {Suffrage:Voter ID:server-df9f2c72-97eb-acff-c8ad-75ebb50a00ba Address:df9f2c72-97eb-acff-c8ad-75ebb50a00ba}]" - testing.go:149: server-407cc3b4-66c0-c34b-5a67-82fda2c709c1: 2025-02-09T06:52:33.991Z [INFO] server-407cc3b4-66c0-c34b-5a67-82fda2c709c1: entering follower state: follower="Node at 407cc3b4-66c0-c34b-5a67-82fda2c709c1 [Follower]" leader= - testing.go:149: server-df9f2c72-97eb-acff-c8ad-75ebb50a00ba: 2025-02-09T06:52:33.991Z [INFO] server-df9f2c72-97eb-acff-c8ad-75ebb50a00ba: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9d1599f6-da3c-0851-04df-2cc5787ef026 Address:9d1599f6-da3c-0851-04df-2cc5787ef026} {Suffrage:Voter ID:server-407cc3b4-66c0-c34b-5a67-82fda2c709c1 Address:407cc3b4-66c0-c34b-5a67-82fda2c709c1} {Suffrage:Voter ID:server-df9f2c72-97eb-acff-c8ad-75ebb50a00ba Address:df9f2c72-97eb-acff-c8ad-75ebb50a00ba}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:33.991Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:33.991Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-df9f2c72-97eb-acff-c8ad-75ebb50a00ba: 2025-02-09T06:52:33.991Z [INFO] server-df9f2c72-97eb-acff-c8ad-75ebb50a00ba: entering follower state: follower="Node at df9f2c72-97eb-acff-c8ad-75ebb50a00ba [Follower]" leader= - testing.go:149: server-df9f2c72-97eb-acff-c8ad-75ebb50a00ba: 2025-02-09T06:52:33.991Z [INFO] server-df9f2c72-97eb-acff-c8ad-75ebb50a00ba: entering follower state: follower="Node at df9f2c72-97eb-acff-c8ad-75ebb50a00ba [Follower]" leader= -testLoggerAdapter verbose: server-9d1599f6-da3c-0851-04df-2cc5787ef026: 2025-02-09T06:52:34.049Z [WARN] server-9d1599f6-da3c-0851-04df-2cc5787ef026: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-9d1599f6-da3c-0851-04df-2cc5787ef026: 2025-02-09T06:52:34.049Z [WARN] server-9d1599f6-da3c-0851-04df-2cc5787ef026: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-9d1599f6-da3c-0851-04df-2cc5787ef026: 2025-02-09T06:52:34.049Z [INFO] server-9d1599f6-da3c-0851-04df-2cc5787ef026: entering candidate state: node="Node at 9d1599f6-da3c-0851-04df-2cc5787ef026 [Candidate]" term=2 - testing.go:149: server-9d1599f6-da3c-0851-04df-2cc5787ef026: 2025-02-09T06:52:34.049Z [INFO] server-9d1599f6-da3c-0851-04df-2cc5787ef026: entering candidate state: node="Node at 9d1599f6-da3c-0851-04df-2cc5787ef026 [Candidate]" term=2 -testLoggerAdapter verbose: server-9d1599f6-da3c-0851-04df-2cc5787ef026: 2025-02-09T06:52:34.049Z [INFO] server-9d1599f6-da3c-0851-04df-2cc5787ef026: election won: tally=2 - testing.go:149: server-9d1599f6-da3c-0851-04df-2cc5787ef026: 2025-02-09T06:52:34.049Z [INFO] server-9d1599f6-da3c-0851-04df-2cc5787ef026: election won: tally=2 -testLoggerAdapter verbose: server-9d1599f6-da3c-0851-04df-2cc5787ef026: 2025-02-09T06:52:34.049Z [INFO] server-9d1599f6-da3c-0851-04df-2cc5787ef026: entering leader state: leader="Node at 9d1599f6-da3c-0851-04df-2cc5787ef026 [Leader]" - testing.go:149: server-9d1599f6-da3c-0851-04df-2cc5787ef026: 2025-02-09T06:52:34.049Z [INFO] server-9d1599f6-da3c-0851-04df-2cc5787ef026: entering leader state: leader="Node at 9d1599f6-da3c-0851-04df-2cc5787ef026 [Leader]" -testLoggerAdapter verbose: server-9d1599f6-da3c-0851-04df-2cc5787ef026: 2025-02-09T06:52:34.050Z [INFO] server-9d1599f6-da3c-0851-04df-2cc5787ef026: added peer, starting replication: peer=server-407cc3b4-66c0-c34b-5a67-82fda2c709c1 - testing.go:149: server-9d1599f6-da3c-0851-04df-2cc5787ef026: 2025-02-09T06:52:34.050Z [INFO] server-9d1599f6-da3c-0851-04df-2cc5787ef026: added peer, starting replication: peer=server-407cc3b4-66c0-c34b-5a67-82fda2c709c1 -testLoggerAdapter verbose: server-9d1599f6-da3c-0851-04df-2cc5787ef026: 2025-02-09T06:52:34.050Z [INFO] server-9d1599f6-da3c-0851-04df-2cc5787ef026: added peer, starting replication: peer=server-df9f2c72-97eb-acff-c8ad-75ebb50a00ba - testing.go:149: server-9d1599f6-da3c-0851-04df-2cc5787ef026: 2025-02-09T06:52:34.050Z [INFO] server-9d1599f6-da3c-0851-04df-2cc5787ef026: added peer, starting replication: peer=server-df9f2c72-97eb-acff-c8ad-75ebb50a00ba -testLoggerAdapter verbose: server-9d1599f6-da3c-0851-04df-2cc5787ef026: 2025-02-09T06:52:34.050Z [INFO] server-9d1599f6-da3c-0851-04df-2cc5787ef026: pipelining replication: peer="{Voter server-df9f2c72-97eb-acff-c8ad-75ebb50a00ba df9f2c72-97eb-acff-c8ad-75ebb50a00ba}" - testing.go:149: server-9d1599f6-da3c-0851-04df-2cc5787ef026: 2025-02-09T06:52:34.050Z [INFO] server-9d1599f6-da3c-0851-04df-2cc5787ef026: pipelining replication: peer="{Voter server-df9f2c72-97eb-acff-c8ad-75ebb50a00ba df9f2c72-97eb-acff-c8ad-75ebb50a00ba}" -testLoggerAdapter verbose: server-9d1599f6-da3c-0851-04df-2cc5787ef026: 2025-02-09T06:52:34.050Z [INFO] server-9d1599f6-da3c-0851-04df-2cc5787ef026: pipelining replication: peer="{Voter server-407cc3b4-66c0-c34b-5a67-82fda2c709c1 407cc3b4-66c0-c34b-5a67-82fda2c709c1}" - testing.go:149: server-9d1599f6-da3c-0851-04df-2cc5787ef026: 2025-02-09T06:52:34.050Z [INFO] server-9d1599f6-da3c-0851-04df-2cc5787ef026: pipelining replication: peer="{Voter server-407cc3b4-66c0-c34b-5a67-82fda2c709c1 407cc3b4-66c0-c34b-5a67-82fda2c709c1}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:34.155Z [INFO] stable state for Leader reached at 2025-02-09 06:52:34.050034059 +0000 UTC m=+15.028842824 (1 nodes), 58.233422ms from start of poll, 58.621104ms from cluster start. Timeout at 2025-02-09 06:52:34.155589992 +0000 UTC m=+15.134398751, 105.555927ms after stability - testing.go:149: cluster: 2025-02-09T06:52:34.155Z [INFO] stable state for Leader reached at 2025-02-09 06:52:34.050034059 +0000 UTC m=+15.028842824 (1 nodes), 58.233422ms from start of poll, 58.621104ms from cluster start. Timeout at 2025-02-09 06:52:34.155589992 +0000 UTC m=+15.134398751, 105.555927ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:34.161Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T06:52:34.161Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:34.266Z [INFO] stable state for Follower reached at 2025-02-09 06:52:34.161116239 +0000 UTC m=+15.139925004 (2 nodes), 773ns from start of poll, 169.703284ms from cluster start. Timeout at 2025-02-09 06:52:34.266946128 +0000 UTC m=+15.245754896, 105.829892ms after stability - testing.go:149: cluster: 2025-02-09T06:52:34.266Z [INFO] stable state for Follower reached at 2025-02-09 06:52:34.161116239 +0000 UTC m=+15.139925004 (2 nodes), 773ns from start of poll, 169.703284ms from cluster start. Timeout at 2025-02-09 06:52:34.266946128 +0000 UTC m=+15.245754896, 105.829892ms after stability - raft_test.go:1647: [INFO] Disconnecting Node at 407cc3b4-66c0-c34b-5a67-82fda2c709c1 [Follower] -testLoggerAdapter verbose: server-9d1599f6-da3c-0851-04df-2cc5787ef026: 2025-02-09T06:52:34.267Z [INFO] server-9d1599f6-da3c-0851-04df-2cc5787ef026: aborting pipeline replication: peer="{Voter server-df9f2c72-97eb-acff-c8ad-75ebb50a00ba df9f2c72-97eb-acff-c8ad-75ebb50a00ba}" - testing.go:149: server-9d1599f6-da3c-0851-04df-2cc5787ef026: 2025-02-09T06:52:34.267Z [INFO] server-9d1599f6-da3c-0851-04df-2cc5787ef026: aborting pipeline replication: peer="{Voter server-df9f2c72-97eb-acff-c8ad-75ebb50a00ba df9f2c72-97eb-acff-c8ad-75ebb50a00ba}" -testLoggerAdapter verbose: server-9d1599f6-da3c-0851-04df-2cc5787ef026: 2025-02-09T06:52:34.267Z [INFO] server-9d1599f6-da3c-0851-04df-2cc5787ef026: aborting pipeline replication: peer="{Voter server-407cc3b4-66c0-c34b-5a67-82fda2c709c1 407cc3b4-66c0-c34b-5a67-82fda2c709c1}" - testing.go:149: server-9d1599f6-da3c-0851-04df-2cc5787ef026: 2025-02-09T06:52:34.267Z [INFO] server-9d1599f6-da3c-0851-04df-2cc5787ef026: aborting pipeline replication: peer="{Voter server-407cc3b4-66c0-c34b-5a67-82fda2c709c1 407cc3b4-66c0-c34b-5a67-82fda2c709c1}" -testLoggerAdapter verbose: server-9d1599f6-da3c-0851-04df-2cc5787ef026: 2025-02-09T06:52:34.267Z [ERROR] server-9d1599f6-da3c-0851-04df-2cc5787ef026: failed to heartbeat to: peer=407cc3b4-66c0-c34b-5a67-82fda2c709c1 error="failed to connect to peer: 407cc3b4-66c0-c34b-5a67-82fda2c709c1" - testing.go:149: server-9d1599f6-da3c-0851-04df-2cc5787ef026: 2025-02-09T06:52:34.267Z [ERROR] server-9d1599f6-da3c-0851-04df-2cc5787ef026: failed to heartbeat to: peer=407cc3b4-66c0-c34b-5a67-82fda2c709c1 error="failed to connect to peer: 407cc3b4-66c0-c34b-5a67-82fda2c709c1" ---- PASS: TestRaft_VerifyLeader_PartialConnect (0.28s) +testLoggerAdapter verbose: server-6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a: 2024-01-08T00:31:09.367Z [INFO] server-6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a Address:6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a} {Suffrage:Voter ID:server-ffd8c2cb-99b3-4123-382d-5c3643aed726 Address:ffd8c2cb-99b3-4123-382d-5c3643aed726} {Suffrage:Voter ID:server-d15f7885-77e5-b765-8bfa-0d47db2d2caa Address:d15f7885-77e5-b765-8bfa-0d47db2d2caa}]" + testing.go:149: server-6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a: 2024-01-08T00:31:09.367Z [INFO] server-6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a Address:6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a} {Suffrage:Voter ID:server-ffd8c2cb-99b3-4123-382d-5c3643aed726 Address:ffd8c2cb-99b3-4123-382d-5c3643aed726} {Suffrage:Voter ID:server-d15f7885-77e5-b765-8bfa-0d47db2d2caa Address:d15f7885-77e5-b765-8bfa-0d47db2d2caa}]" +testLoggerAdapter verbose: server-ffd8c2cb-99b3-4123-382d-5c3643aed726: 2024-01-08T00:31:09.367Z [INFO] server-ffd8c2cb-99b3-4123-382d-5c3643aed726: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a Address:6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a} {Suffrage:Voter ID:server-ffd8c2cb-99b3-4123-382d-5c3643aed726 Address:ffd8c2cb-99b3-4123-382d-5c3643aed726} {Suffrage:Voter ID:server-d15f7885-77e5-b765-8bfa-0d47db2d2caa Address:d15f7885-77e5-b765-8bfa-0d47db2d2caa}]" + testing.go:149: server-ffd8c2cb-99b3-4123-382d-5c3643aed726: 2024-01-08T00:31:09.367Z [INFO] server-ffd8c2cb-99b3-4123-382d-5c3643aed726: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a Address:6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a} {Suffrage:Voter ID:server-ffd8c2cb-99b3-4123-382d-5c3643aed726 Address:ffd8c2cb-99b3-4123-382d-5c3643aed726} {Suffrage:Voter ID:server-d15f7885-77e5-b765-8bfa-0d47db2d2caa Address:d15f7885-77e5-b765-8bfa-0d47db2d2caa}]" +testLoggerAdapter verbose: server-d15f7885-77e5-b765-8bfa-0d47db2d2caa: 2024-01-08T00:31:09.367Z [INFO] server-d15f7885-77e5-b765-8bfa-0d47db2d2caa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a Address:6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a} {Suffrage:Voter ID:server-ffd8c2cb-99b3-4123-382d-5c3643aed726 Address:ffd8c2cb-99b3-4123-382d-5c3643aed726} {Suffrage:Voter ID:server-d15f7885-77e5-b765-8bfa-0d47db2d2caa Address:d15f7885-77e5-b765-8bfa-0d47db2d2caa}]" + testing.go:149: server-d15f7885-77e5-b765-8bfa-0d47db2d2caa: 2024-01-08T00:31:09.367Z [INFO] server-d15f7885-77e5-b765-8bfa-0d47db2d2caa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a Address:6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a} {Suffrage:Voter ID:server-ffd8c2cb-99b3-4123-382d-5c3643aed726 Address:ffd8c2cb-99b3-4123-382d-5c3643aed726} {Suffrage:Voter ID:server-d15f7885-77e5-b765-8bfa-0d47db2d2caa Address:d15f7885-77e5-b765-8bfa-0d47db2d2caa}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:09.367Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:09.367Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a: 2024-01-08T00:31:09.367Z [INFO] server-6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a: entering follower state: follower="Node at 6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a [Follower]" leader= + testing.go:149: server-6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a: 2024-01-08T00:31:09.367Z [INFO] server-6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a: entering follower state: follower="Node at 6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a [Follower]" leader= +testLoggerAdapter verbose: server-ffd8c2cb-99b3-4123-382d-5c3643aed726: 2024-01-08T00:31:09.367Z [INFO] server-ffd8c2cb-99b3-4123-382d-5c3643aed726: entering follower state: follower="Node at ffd8c2cb-99b3-4123-382d-5c3643aed726 [Follower]" leader= + testing.go:149: server-ffd8c2cb-99b3-4123-382d-5c3643aed726: 2024-01-08T00:31:09.367Z [INFO] server-ffd8c2cb-99b3-4123-382d-5c3643aed726: entering follower state: follower="Node at ffd8c2cb-99b3-4123-382d-5c3643aed726 [Follower]" leader= +testLoggerAdapter verbose: server-d15f7885-77e5-b765-8bfa-0d47db2d2caa: 2024-01-08T00:31:09.367Z [INFO] server-d15f7885-77e5-b765-8bfa-0d47db2d2caa: entering follower state: follower="Node at d15f7885-77e5-b765-8bfa-0d47db2d2caa [Follower]" leader= + testing.go:149: server-d15f7885-77e5-b765-8bfa-0d47db2d2caa: 2024-01-08T00:31:09.367Z [INFO] server-d15f7885-77e5-b765-8bfa-0d47db2d2caa: entering follower state: follower="Node at d15f7885-77e5-b765-8bfa-0d47db2d2caa [Follower]" leader= +testLoggerAdapter verbose: server-ffd8c2cb-99b3-4123-382d-5c3643aed726: 2024-01-08T00:31:09.420Z [WARN] server-ffd8c2cb-99b3-4123-382d-5c3643aed726: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-ffd8c2cb-99b3-4123-382d-5c3643aed726: 2024-01-08T00:31:09.420Z [WARN] server-ffd8c2cb-99b3-4123-382d-5c3643aed726: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-ffd8c2cb-99b3-4123-382d-5c3643aed726: 2024-01-08T00:31:09.420Z [INFO] server-ffd8c2cb-99b3-4123-382d-5c3643aed726: entering candidate state: node="Node at ffd8c2cb-99b3-4123-382d-5c3643aed726 [Candidate]" term=2 + testing.go:149: server-ffd8c2cb-99b3-4123-382d-5c3643aed726: 2024-01-08T00:31:09.420Z [INFO] server-ffd8c2cb-99b3-4123-382d-5c3643aed726: entering candidate state: node="Node at ffd8c2cb-99b3-4123-382d-5c3643aed726 [Candidate]" term=2 +testLoggerAdapter verbose: server-ffd8c2cb-99b3-4123-382d-5c3643aed726: 2024-01-08T00:31:09.421Z [INFO] server-ffd8c2cb-99b3-4123-382d-5c3643aed726: election won: tally=2 + testing.go:149: server-ffd8c2cb-99b3-4123-382d-5c3643aed726: 2024-01-08T00:31:09.421Z [INFO] server-ffd8c2cb-99b3-4123-382d-5c3643aed726: election won: tally=2 +testLoggerAdapter verbose: server-ffd8c2cb-99b3-4123-382d-5c3643aed726: 2024-01-08T00:31:09.421Z [INFO] server-ffd8c2cb-99b3-4123-382d-5c3643aed726: entering leader state: leader="Node at ffd8c2cb-99b3-4123-382d-5c3643aed726 [Leader]" + testing.go:149: server-ffd8c2cb-99b3-4123-382d-5c3643aed726: 2024-01-08T00:31:09.421Z [INFO] server-ffd8c2cb-99b3-4123-382d-5c3643aed726: entering leader state: leader="Node at ffd8c2cb-99b3-4123-382d-5c3643aed726 [Leader]" +testLoggerAdapter verbose: server-ffd8c2cb-99b3-4123-382d-5c3643aed726: 2024-01-08T00:31:09.421Z [INFO] server-ffd8c2cb-99b3-4123-382d-5c3643aed726: added peer, starting replication: peer=server-6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a + testing.go:149: server-ffd8c2cb-99b3-4123-382d-5c3643aed726: 2024-01-08T00:31:09.421Z [INFO] server-ffd8c2cb-99b3-4123-382d-5c3643aed726: added peer, starting replication: peer=server-6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a +testLoggerAdapter verbose: server-ffd8c2cb-99b3-4123-382d-5c3643aed726: 2024-01-08T00:31:09.421Z [INFO] server-ffd8c2cb-99b3-4123-382d-5c3643aed726: added peer, starting replication: peer=server-d15f7885-77e5-b765-8bfa-0d47db2d2caa + testing.go:149: server-ffd8c2cb-99b3-4123-382d-5c3643aed726: 2024-01-08T00:31:09.421Z [INFO] server-ffd8c2cb-99b3-4123-382d-5c3643aed726: added peer, starting replication: peer=server-d15f7885-77e5-b765-8bfa-0d47db2d2caa +testLoggerAdapter verbose: server-ffd8c2cb-99b3-4123-382d-5c3643aed726: 2024-01-08T00:31:09.421Z [INFO] server-ffd8c2cb-99b3-4123-382d-5c3643aed726: pipelining replication: peer="{Voter server-6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a 6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a}" + testing.go:149: server-ffd8c2cb-99b3-4123-382d-5c3643aed726: 2024-01-08T00:31:09.421Z [INFO] server-ffd8c2cb-99b3-4123-382d-5c3643aed726: pipelining replication: peer="{Voter server-6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a 6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a}" +testLoggerAdapter verbose: server-ffd8c2cb-99b3-4123-382d-5c3643aed726: 2024-01-08T00:31:09.421Z [INFO] server-ffd8c2cb-99b3-4123-382d-5c3643aed726: pipelining replication: peer="{Voter server-d15f7885-77e5-b765-8bfa-0d47db2d2caa d15f7885-77e5-b765-8bfa-0d47db2d2caa}" + testing.go:149: server-ffd8c2cb-99b3-4123-382d-5c3643aed726: 2024-01-08T00:31:09.421Z [INFO] server-ffd8c2cb-99b3-4123-382d-5c3643aed726: pipelining replication: peer="{Voter server-d15f7885-77e5-b765-8bfa-0d47db2d2caa d15f7885-77e5-b765-8bfa-0d47db2d2caa}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:09.527Z [INFO] stable state for Leader reached at 2024-01-08 00:31:09.421131217 +0000 UTC m=+15.468020365 (1 nodes), 53.248254ms from start of poll, 53.730583ms from cluster start. Timeout at 2024-01-08 00:31:09.527127422 +0000 UTC m=+15.574016577, 105.996212ms after stability + testing.go:149: cluster: 2024-01-08T00:31:09.527Z [INFO] stable state for Leader reached at 2024-01-08 00:31:09.421131217 +0000 UTC m=+15.468020365 (1 nodes), 53.248254ms from start of poll, 53.730583ms from cluster start. Timeout at 2024-01-08 00:31:09.527127422 +0000 UTC m=+15.574016577, 105.996212ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:09.532Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T00:31:09.532Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:09.638Z [INFO] stable state for Follower reached at 2024-01-08 00:31:09.532928484 +0000 UTC m=+15.579817630 (2 nodes), 2.52µs from start of poll, 165.527848ms from cluster start. Timeout at 2024-01-08 00:31:09.638814989 +0000 UTC m=+15.685704150, 105.88652ms after stability + testing.go:149: cluster: 2024-01-08T00:31:09.638Z [INFO] stable state for Follower reached at 2024-01-08 00:31:09.532928484 +0000 UTC m=+15.579817630 (2 nodes), 2.52µs from start of poll, 165.527848ms from cluster start. Timeout at 2024-01-08 00:31:09.638814989 +0000 UTC m=+15.685704150, 105.88652ms after stability + raft_test.go:1647: [INFO] Disconnecting Node at 6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a [Follower] +testLoggerAdapter verbose: server-ffd8c2cb-99b3-4123-382d-5c3643aed726: 2024-01-08T00:31:09.639Z [ERROR] server-ffd8c2cb-99b3-4123-382d-5c3643aed726: failed to heartbeat to: peer=6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a error="failed to connect to peer: 6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a" + testing.go:149: server-ffd8c2cb-99b3-4123-382d-5c3643aed726: 2024-01-08T00:31:09.639Z [ERROR] server-ffd8c2cb-99b3-4123-382d-5c3643aed726: failed to heartbeat to: peer=6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a error="failed to connect to peer: 6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a" +testLoggerAdapter verbose: server-ffd8c2cb-99b3-4123-382d-5c3643aed726: 2024-01-08T00:31:09.639Z [INFO] server-ffd8c2cb-99b3-4123-382d-5c3643aed726: aborting pipeline replication: peer="{Voter server-d15f7885-77e5-b765-8bfa-0d47db2d2caa d15f7885-77e5-b765-8bfa-0d47db2d2caa}" + testing.go:149: server-ffd8c2cb-99b3-4123-382d-5c3643aed726: 2024-01-08T00:31:09.639Z [INFO] server-ffd8c2cb-99b3-4123-382d-5c3643aed726: aborting pipeline replication: peer="{Voter server-d15f7885-77e5-b765-8bfa-0d47db2d2caa d15f7885-77e5-b765-8bfa-0d47db2d2caa}" +testLoggerAdapter verbose: server-ffd8c2cb-99b3-4123-382d-5c3643aed726: 2024-01-08T00:31:09.639Z [INFO] server-ffd8c2cb-99b3-4123-382d-5c3643aed726: aborting pipeline replication: peer="{Voter server-6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a 6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a}" + testing.go:149: server-ffd8c2cb-99b3-4123-382d-5c3643aed726: 2024-01-08T00:31:09.639Z [INFO] server-ffd8c2cb-99b3-4123-382d-5c3643aed726: aborting pipeline replication: peer="{Voter server-6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a 6f4e87c3-88bf-b8c0-c46b-bd2b0b39475a}" +--- PASS: TestRaft_VerifyLeader_PartialConnect (0.27s) === RUN TestRaft_StartAsLeader -testLoggerAdapter verbose: server-ec3f17ef-df76-8d61-fd8b-fe45d9bd7cd0: 2025-02-09T06:52:34.267Z [INFO] server-ec3f17ef-df76-8d61-fd8b-fe45d9bd7cd0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ec3f17ef-df76-8d61-fd8b-fe45d9bd7cd0 Address:ec3f17ef-df76-8d61-fd8b-fe45d9bd7cd0}]" - testing.go:149: server-ec3f17ef-df76-8d61-fd8b-fe45d9bd7cd0: 2025-02-09T06:52:34.267Z [INFO] server-ec3f17ef-df76-8d61-fd8b-fe45d9bd7cd0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ec3f17ef-df76-8d61-fd8b-fe45d9bd7cd0 Address:ec3f17ef-df76-8d61-fd8b-fe45d9bd7cd0}]" -testLoggerAdapter verbose: server-ec3f17ef-df76-8d61-fd8b-fe45d9bd7cd0: 2025-02-09T06:52:34.267Z [INFO] server-ec3f17ef-df76-8d61-fd8b-fe45d9bd7cd0: entering leader state: leader="Node at ec3f17ef-df76-8d61-fd8b-fe45d9bd7cd0 [Leader]" - testing.go:149: server-ec3f17ef-df76-8d61-fd8b-fe45d9bd7cd0: 2025-02-09T06:52:34.267Z [INFO] server-ec3f17ef-df76-8d61-fd8b-fe45d9bd7cd0: entering leader state: leader="Node at ec3f17ef-df76-8d61-fd8b-fe45d9bd7cd0 [Leader]" +testLoggerAdapter verbose: server-efe4450a-1b1c-9bc6-d9d8-4215e6db3736: 2024-01-08T00:31:09.640Z [INFO] server-efe4450a-1b1c-9bc6-d9d8-4215e6db3736: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-efe4450a-1b1c-9bc6-d9d8-4215e6db3736 Address:efe4450a-1b1c-9bc6-d9d8-4215e6db3736}]" + testing.go:149: server-efe4450a-1b1c-9bc6-d9d8-4215e6db3736: 2024-01-08T00:31:09.640Z [INFO] server-efe4450a-1b1c-9bc6-d9d8-4215e6db3736: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-efe4450a-1b1c-9bc6-d9d8-4215e6db3736 Address:efe4450a-1b1c-9bc6-d9d8-4215e6db3736}]" +testLoggerAdapter verbose: server-efe4450a-1b1c-9bc6-d9d8-4215e6db3736: 2024-01-08T00:31:09.640Z [INFO] server-efe4450a-1b1c-9bc6-d9d8-4215e6db3736: entering leader state: leader="Node at efe4450a-1b1c-9bc6-d9d8-4215e6db3736 [Leader]" + testing.go:149: server-efe4450a-1b1c-9bc6-d9d8-4215e6db3736: 2024-01-08T00:31:09.640Z [INFO] server-efe4450a-1b1c-9bc6-d9d8-4215e6db3736: entering leader state: leader="Node at efe4450a-1b1c-9bc6-d9d8-4215e6db3736 [Leader]" --- PASS: TestRaft_StartAsLeader (0.00s) === RUN TestRaft_NotifyCh -testLoggerAdapter verbose: server-0e510a56-65ae-c993-f684-401f9345e797: 2025-02-09T06:52:34.268Z [INFO] server-0e510a56-65ae-c993-f684-401f9345e797: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0e510a56-65ae-c993-f684-401f9345e797 Address:0e510a56-65ae-c993-f684-401f9345e797}]" - testing.go:149: server-0e510a56-65ae-c993-f684-401f9345e797: 2025-02-09T06:52:34.268Z [INFO] server-0e510a56-65ae-c993-f684-401f9345e797: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0e510a56-65ae-c993-f684-401f9345e797 Address:0e510a56-65ae-c993-f684-401f9345e797}]" -testLoggerAdapter verbose: server-0e510a56-65ae-c993-f684-401f9345e797: 2025-02-09T06:52:34.268Z [INFO] server-0e510a56-65ae-c993-f684-401f9345e797: entering follower state: follower="Node at 0e510a56-65ae-c993-f684-401f9345e797 [Follower]" leader= - testing.go:149: server-0e510a56-65ae-c993-f684-401f9345e797: 2025-02-09T06:52:34.268Z [INFO] server-0e510a56-65ae-c993-f684-401f9345e797: entering follower state: follower="Node at 0e510a56-65ae-c993-f684-401f9345e797 [Follower]" leader= -testLoggerAdapter verbose: server-0e510a56-65ae-c993-f684-401f9345e797: 2025-02-09T06:52:34.359Z [WARN] server-0e510a56-65ae-c993-f684-401f9345e797: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-0e510a56-65ae-c993-f684-401f9345e797: 2025-02-09T06:52:34.359Z [WARN] server-0e510a56-65ae-c993-f684-401f9345e797: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-0e510a56-65ae-c993-f684-401f9345e797: 2025-02-09T06:52:34.359Z [INFO] server-0e510a56-65ae-c993-f684-401f9345e797: entering candidate state: node="Node at 0e510a56-65ae-c993-f684-401f9345e797 [Candidate]" term=2 - testing.go:149: server-0e510a56-65ae-c993-f684-401f9345e797: 2025-02-09T06:52:34.359Z [INFO] server-0e510a56-65ae-c993-f684-401f9345e797: entering candidate state: node="Node at 0e510a56-65ae-c993-f684-401f9345e797 [Candidate]" term=2 -testLoggerAdapter verbose: server-0e510a56-65ae-c993-f684-401f9345e797: 2025-02-09T06:52:34.359Z [INFO] server-0e510a56-65ae-c993-f684-401f9345e797: election won: tally=1 - testing.go:149: server-0e510a56-65ae-c993-f684-401f9345e797: 2025-02-09T06:52:34.359Z [INFO] server-0e510a56-65ae-c993-f684-401f9345e797: election won: tally=1 -testLoggerAdapter verbose: server-0e510a56-65ae-c993-f684-401f9345e797: 2025-02-09T06:52:34.359Z [INFO] server-0e510a56-65ae-c993-f684-401f9345e797: entering leader state: leader="Node at 0e510a56-65ae-c993-f684-401f9345e797 [Leader]" - testing.go:149: server-0e510a56-65ae-c993-f684-401f9345e797: 2025-02-09T06:52:34.359Z [INFO] server-0e510a56-65ae-c993-f684-401f9345e797: entering leader state: leader="Node at 0e510a56-65ae-c993-f684-401f9345e797 [Leader]" ---- PASS: TestRaft_NotifyCh (0.09s) +testLoggerAdapter verbose: server-34e697e9-b19b-e0f5-b353-004d99039ae9: 2024-01-08T00:31:09.640Z [INFO] server-34e697e9-b19b-e0f5-b353-004d99039ae9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-34e697e9-b19b-e0f5-b353-004d99039ae9 Address:34e697e9-b19b-e0f5-b353-004d99039ae9}]" + testing.go:149: server-34e697e9-b19b-e0f5-b353-004d99039ae9: 2024-01-08T00:31:09.640Z [INFO] server-34e697e9-b19b-e0f5-b353-004d99039ae9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-34e697e9-b19b-e0f5-b353-004d99039ae9 Address:34e697e9-b19b-e0f5-b353-004d99039ae9}]" +testLoggerAdapter verbose: server-34e697e9-b19b-e0f5-b353-004d99039ae9: 2024-01-08T00:31:09.640Z [INFO] server-34e697e9-b19b-e0f5-b353-004d99039ae9: entering follower state: follower="Node at 34e697e9-b19b-e0f5-b353-004d99039ae9 [Follower]" leader= + testing.go:149: server-34e697e9-b19b-e0f5-b353-004d99039ae9: 2024-01-08T00:31:09.640Z [INFO] server-34e697e9-b19b-e0f5-b353-004d99039ae9: entering follower state: follower="Node at 34e697e9-b19b-e0f5-b353-004d99039ae9 [Follower]" leader= +testLoggerAdapter verbose: server-34e697e9-b19b-e0f5-b353-004d99039ae9: 2024-01-08T00:31:09.721Z [WARN] server-34e697e9-b19b-e0f5-b353-004d99039ae9: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-34e697e9-b19b-e0f5-b353-004d99039ae9: 2024-01-08T00:31:09.721Z [WARN] server-34e697e9-b19b-e0f5-b353-004d99039ae9: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-34e697e9-b19b-e0f5-b353-004d99039ae9: 2024-01-08T00:31:09.721Z [INFO] server-34e697e9-b19b-e0f5-b353-004d99039ae9: entering candidate state: node="Node at 34e697e9-b19b-e0f5-b353-004d99039ae9 [Candidate]" term=2 + testing.go:149: server-34e697e9-b19b-e0f5-b353-004d99039ae9: 2024-01-08T00:31:09.721Z [INFO] server-34e697e9-b19b-e0f5-b353-004d99039ae9: entering candidate state: node="Node at 34e697e9-b19b-e0f5-b353-004d99039ae9 [Candidate]" term=2 +testLoggerAdapter verbose: server-34e697e9-b19b-e0f5-b353-004d99039ae9: 2024-01-08T00:31:09.721Z [INFO] server-34e697e9-b19b-e0f5-b353-004d99039ae9: election won: tally=1 + testing.go:149: server-34e697e9-b19b-e0f5-b353-004d99039ae9: 2024-01-08T00:31:09.721Z [INFO] server-34e697e9-b19b-e0f5-b353-004d99039ae9: election won: tally=1 +testLoggerAdapter verbose: server-34e697e9-b19b-e0f5-b353-004d99039ae9: 2024-01-08T00:31:09.721Z [INFO] server-34e697e9-b19b-e0f5-b353-004d99039ae9: entering leader state: leader="Node at 34e697e9-b19b-e0f5-b353-004d99039ae9 [Leader]" + testing.go:149: server-34e697e9-b19b-e0f5-b353-004d99039ae9: 2024-01-08T00:31:09.721Z [INFO] server-34e697e9-b19b-e0f5-b353-004d99039ae9: entering leader state: leader="Node at 34e697e9-b19b-e0f5-b353-004d99039ae9 [Leader]" +--- PASS: TestRaft_NotifyCh (0.08s) === RUN TestRaft_Voting -testLoggerAdapter verbose: server-6d608e61-8c13-b9ca-8ce4-87cfcc4de59c: 2025-02-09T06:52:34.360Z [INFO] server-6d608e61-8c13-b9ca-8ce4-87cfcc4de59c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6d608e61-8c13-b9ca-8ce4-87cfcc4de59c Address:6d608e61-8c13-b9ca-8ce4-87cfcc4de59c} {Suffrage:Voter ID:server-495f2503-a9c7-727c-2520-1b572acf664c Address:495f2503-a9c7-727c-2520-1b572acf664c} {Suffrage:Voter ID:server-283496e2-c19e-0c9b-c344-d497adf8e685 Address:283496e2-c19e-0c9b-c344-d497adf8e685}]" - testing.go:149: server-6d608e61-8c13-b9ca-8ce4-87cfcc4de59c: 2025-02-09T06:52:34.360Z [INFO] server-6d608e61-8c13-b9ca-8ce4-87cfcc4de59c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6d608e61-8c13-b9ca-8ce4-87cfcc4de59c Address:6d608e61-8c13-b9ca-8ce4-87cfcc4de59c} {Suffrage:Voter ID:server-495f2503-a9c7-727c-2520-1b572acf664c Address:495f2503-a9c7-727c-2520-1b572acf664c} {Suffrage:Voter ID:server-283496e2-c19e-0c9b-c344-d497adf8e685 Address:283496e2-c19e-0c9b-c344-d497adf8e685}]" -testLoggerAdapter verbose: server-6d608e61-8c13-b9ca-8ce4-87cfcc4de59c: 2025-02-09T06:52:34.360Z [INFO] server-6d608e61-8c13-b9ca-8ce4-87cfcc4de59c: entering follower state: follower="Node at 6d608e61-8c13-b9ca-8ce4-87cfcc4de59c [Follower]" leader= - testing.go:149: server-6d608e61-8c13-b9ca-8ce4-87cfcc4de59c: 2025-02-09T06:52:34.360Z [INFO] server-6d608e61-8c13-b9ca-8ce4-87cfcc4de59c: entering follower state: follower="Node at 6d608e61-8c13-b9ca-8ce4-87cfcc4de59c [Follower]" leader= -testLoggerAdapter verbose: server-495f2503-a9c7-727c-2520-1b572acf664c: 2025-02-09T06:52:34.360Z [INFO] server-495f2503-a9c7-727c-2520-1b572acf664c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6d608e61-8c13-b9ca-8ce4-87cfcc4de59c Address:6d608e61-8c13-b9ca-8ce4-87cfcc4de59c} {Suffrage:Voter ID:server-495f2503-a9c7-727c-2520-1b572acf664c Address:495f2503-a9c7-727c-2520-1b572acf664c} {Suffrage:Voter ID:server-283496e2-c19e-0c9b-c344-d497adf8e685 Address:283496e2-c19e-0c9b-c344-d497adf8e685}]" - testing.go:149: server-495f2503-a9c7-727c-2520-1b572acf664c: 2025-02-09T06:52:34.360Z [INFO] server-495f2503-a9c7-727c-2520-1b572acf664c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6d608e61-8c13-b9ca-8ce4-87cfcc4de59c Address:6d608e61-8c13-b9ca-8ce4-87cfcc4de59c} {Suffrage:Voter ID:server-495f2503-a9c7-727c-2520-1b572acf664c Address:495f2503-a9c7-727c-2520-1b572acf664c} {Suffrage:Voter ID:server-283496e2-c19e-0c9b-c344-d497adf8e685 Address:283496e2-c19e-0c9b-c344-d497adf8e685}]" -testLoggerAdapter verbose: server-495f2503-a9c7-727c-2520-1b572acf664c: 2025-02-09T06:52:34.360Z [INFO] server-495f2503-a9c7-727c-2520-1b572acf664c: entering follower state: follower="Node at 495f2503-a9c7-727c-2520-1b572acf664c [Follower]" leader= - testing.go:149: server-495f2503-a9c7-727c-2520-1b572acf664c: 2025-02-09T06:52:34.360Z [INFO] server-495f2503-a9c7-727c-2520-1b572acf664c: entering follower state: follower="Node at 495f2503-a9c7-727c-2520-1b572acf664c [Follower]" leader= -testLoggerAdapter verbose: server-283496e2-c19e-0c9b-c344-d497adf8e685: 2025-02-09T06:52:34.360Z [INFO] server-283496e2-c19e-0c9b-c344-d497adf8e685: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6d608e61-8c13-b9ca-8ce4-87cfcc4de59c Address:6d608e61-8c13-b9ca-8ce4-87cfcc4de59c} {Suffrage:Voter ID:server-495f2503-a9c7-727c-2520-1b572acf664c Address:495f2503-a9c7-727c-2520-1b572acf664c} {Suffrage:Voter ID:server-283496e2-c19e-0c9b-c344-d497adf8e685 Address:283496e2-c19e-0c9b-c344-d497adf8e685}]" - testing.go:149: server-283496e2-c19e-0c9b-c344-d497adf8e685: 2025-02-09T06:52:34.360Z [INFO] server-283496e2-c19e-0c9b-c344-d497adf8e685: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6d608e61-8c13-b9ca-8ce4-87cfcc4de59c Address:6d608e61-8c13-b9ca-8ce4-87cfcc4de59c} {Suffrage:Voter ID:server-495f2503-a9c7-727c-2520-1b572acf664c Address:495f2503-a9c7-727c-2520-1b572acf664c} {Suffrage:Voter ID:server-283496e2-c19e-0c9b-c344-d497adf8e685 Address:283496e2-c19e-0c9b-c344-d497adf8e685}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:34.360Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T06:52:34.360Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-283496e2-c19e-0c9b-c344-d497adf8e685: 2025-02-09T06:52:34.360Z [INFO] server-283496e2-c19e-0c9b-c344-d497adf8e685: entering follower state: follower="Node at 283496e2-c19e-0c9b-c344-d497adf8e685 [Follower]" leader= - testing.go:149: server-283496e2-c19e-0c9b-c344-d497adf8e685: 2025-02-09T06:52:34.360Z [INFO] server-283496e2-c19e-0c9b-c344-d497adf8e685: entering follower state: follower="Node at 283496e2-c19e-0c9b-c344-d497adf8e685 [Follower]" leader= -testLoggerAdapter verbose: server-283496e2-c19e-0c9b-c344-d497adf8e685: 2025-02-09T06:52:34.430Z [WARN] server-283496e2-c19e-0c9b-c344-d497adf8e685: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-283496e2-c19e-0c9b-c344-d497adf8e685: 2025-02-09T06:52:34.430Z [WARN] server-283496e2-c19e-0c9b-c344-d497adf8e685: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-283496e2-c19e-0c9b-c344-d497adf8e685: 2025-02-09T06:52:34.430Z [INFO] server-283496e2-c19e-0c9b-c344-d497adf8e685: entering candidate state: node="Node at 283496e2-c19e-0c9b-c344-d497adf8e685 [Candidate]" term=2 - testing.go:149: server-283496e2-c19e-0c9b-c344-d497adf8e685: 2025-02-09T06:52:34.430Z [INFO] server-283496e2-c19e-0c9b-c344-d497adf8e685: entering candidate state: node="Node at 283496e2-c19e-0c9b-c344-d497adf8e685 [Candidate]" term=2 -testLoggerAdapter verbose: server-283496e2-c19e-0c9b-c344-d497adf8e685: 2025-02-09T06:52:34.430Z [INFO] server-283496e2-c19e-0c9b-c344-d497adf8e685: election won: tally=2 - testing.go:149: server-283496e2-c19e-0c9b-c344-d497adf8e685: 2025-02-09T06:52:34.430Z [INFO] server-283496e2-c19e-0c9b-c344-d497adf8e685: election won: tally=2 -testLoggerAdapter verbose: server-283496e2-c19e-0c9b-c344-d497adf8e685: 2025-02-09T06:52:34.430Z [INFO] server-283496e2-c19e-0c9b-c344-d497adf8e685: entering leader state: leader="Node at 283496e2-c19e-0c9b-c344-d497adf8e685 [Leader]" - testing.go:149: server-283496e2-c19e-0c9b-c344-d497adf8e685: 2025-02-09T06:52:34.430Z [INFO] server-283496e2-c19e-0c9b-c344-d497adf8e685: entering leader state: leader="Node at 283496e2-c19e-0c9b-c344-d497adf8e685 [Leader]" -testLoggerAdapter verbose: server-283496e2-c19e-0c9b-c344-d497adf8e685: 2025-02-09T06:52:34.430Z [INFO] server-283496e2-c19e-0c9b-c344-d497adf8e685: added peer, starting replication: peer=server-6d608e61-8c13-b9ca-8ce4-87cfcc4de59c - testing.go:149: server-283496e2-c19e-0c9b-c344-d497adf8e685: 2025-02-09T06:52:34.430Z [INFO] server-283496e2-c19e-0c9b-c344-d497adf8e685: added peer, starting replication: peer=server-6d608e61-8c13-b9ca-8ce4-87cfcc4de59c -testLoggerAdapter verbose: server-283496e2-c19e-0c9b-c344-d497adf8e685: 2025-02-09T06:52:34.430Z [INFO] server-283496e2-c19e-0c9b-c344-d497adf8e685: added peer, starting replication: peer=server-495f2503-a9c7-727c-2520-1b572acf664c - testing.go:149: server-283496e2-c19e-0c9b-c344-d497adf8e685: 2025-02-09T06:52:34.430Z [INFO] server-283496e2-c19e-0c9b-c344-d497adf8e685: added peer, starting replication: peer=server-495f2503-a9c7-727c-2520-1b572acf664c -testLoggerAdapter verbose: server-283496e2-c19e-0c9b-c344-d497adf8e685: 2025-02-09T06:52:34.430Z [INFO] server-283496e2-c19e-0c9b-c344-d497adf8e685: pipelining replication: peer="{Voter server-495f2503-a9c7-727c-2520-1b572acf664c 495f2503-a9c7-727c-2520-1b572acf664c}" - testing.go:149: server-283496e2-c19e-0c9b-c344-d497adf8e685: 2025-02-09T06:52:34.430Z [INFO] server-283496e2-c19e-0c9b-c344-d497adf8e685: pipelining replication: peer="{Voter server-495f2503-a9c7-727c-2520-1b572acf664c 495f2503-a9c7-727c-2520-1b572acf664c}" -testLoggerAdapter verbose: server-283496e2-c19e-0c9b-c344-d497adf8e685: 2025-02-09T06:52:34.430Z [INFO] server-283496e2-c19e-0c9b-c344-d497adf8e685: pipelining replication: peer="{Voter server-6d608e61-8c13-b9ca-8ce4-87cfcc4de59c 6d608e61-8c13-b9ca-8ce4-87cfcc4de59c}" - testing.go:149: server-283496e2-c19e-0c9b-c344-d497adf8e685: 2025-02-09T06:52:34.430Z [INFO] server-283496e2-c19e-0c9b-c344-d497adf8e685: pipelining replication: peer="{Voter server-6d608e61-8c13-b9ca-8ce4-87cfcc4de59c 6d608e61-8c13-b9ca-8ce4-87cfcc4de59c}" -testLoggerAdapter verbose: server-6d608e61-8c13-b9ca-8ce4-87cfcc4de59c: 2025-02-09T06:52:34.430Z [WARN] server-6d608e61-8c13-b9ca-8ce4-87cfcc4de59c: rejecting vote request since our last term is greater: candidate=283496e2-c19e-0c9b-c344-d497adf8e685 last-term=2 last-candidate-term=1 - testing.go:149: server-6d608e61-8c13-b9ca-8ce4-87cfcc4de59c: 2025-02-09T06:52:34.430Z [WARN] server-6d608e61-8c13-b9ca-8ce4-87cfcc4de59c: rejecting vote request since our last term is greater: candidate=283496e2-c19e-0c9b-c344-d497adf8e685 last-term=2 last-candidate-term=1 -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:34.536Z [INFO] stable state for Follower reached at 2025-02-09 06:52:34.430802537 +0000 UTC m=+15.409611300 (2 nodes), 69.934801ms from start of poll, 70.31543ms from cluster start. Timeout at 2025-02-09 06:52:34.536531212 +0000 UTC m=+15.515339980, 105.72868ms after stability - testing.go:149: cluster: 2025-02-09T06:52:34.536Z [INFO] stable state for Follower reached at 2025-02-09 06:52:34.430802537 +0000 UTC m=+15.409611300 (2 nodes), 69.934801ms from start of poll, 70.31543ms from cluster start. Timeout at 2025-02-09 06:52:34.536531212 +0000 UTC m=+15.515339980, 105.72868ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:34.536Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:34.536Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:34.642Z [INFO] stable state for Leader reached at 2025-02-09 06:52:34.536606742 +0000 UTC m=+15.515415506 (1 nodes), 477ns from start of poll, 176.119636ms from cluster start. Timeout at 2025-02-09 06:52:34.64236387 +0000 UTC m=+15.621172643, 105.757137ms after stability - testing.go:149: cluster: 2025-02-09T06:52:34.642Z [INFO] stable state for Leader reached at 2025-02-09 06:52:34.536606742 +0000 UTC m=+15.515415506 (1 nodes), 477ns from start of poll, 176.119636ms from cluster start. Timeout at 2025-02-09 06:52:34.64236387 +0000 UTC m=+15.621172643, 105.757137ms after stability -testLoggerAdapter verbose: server-495f2503-a9c7-727c-2520-1b572acf664c: 2025-02-09T06:52:34.642Z [WARN] server-495f2503-a9c7-727c-2520-1b572acf664c: rejecting vote request since we have a leader: from=6d608e61-8c13-b9ca-8ce4-87cfcc4de59c leader=283496e2-c19e-0c9b-c344-d497adf8e685 - testing.go:149: server-495f2503-a9c7-727c-2520-1b572acf664c: 2025-02-09T06:52:34.642Z [WARN] server-495f2503-a9c7-727c-2520-1b572acf664c: rejecting vote request since we have a leader: from=6d608e61-8c13-b9ca-8ce4-87cfcc4de59c leader=283496e2-c19e-0c9b-c344-d497adf8e685 -testLoggerAdapter verbose: server-283496e2-c19e-0c9b-c344-d497adf8e685: 2025-02-09T06:52:34.642Z [INFO] server-283496e2-c19e-0c9b-c344-d497adf8e685: aborting pipeline replication: peer="{Voter server-495f2503-a9c7-727c-2520-1b572acf664c 495f2503-a9c7-727c-2520-1b572acf664c}" - testing.go:149: server-283496e2-c19e-0c9b-c344-d497adf8e685: 2025-02-09T06:52:34.642Z [INFO] server-283496e2-c19e-0c9b-c344-d497adf8e685: aborting pipeline replication: peer="{Voter server-495f2503-a9c7-727c-2520-1b572acf664c 495f2503-a9c7-727c-2520-1b572acf664c}" -testLoggerAdapter verbose: server-283496e2-c19e-0c9b-c344-d497adf8e685: 2025-02-09T06:52:34.642Z [INFO] server-283496e2-c19e-0c9b-c344-d497adf8e685: aborting pipeline replication: peer="{Voter server-6d608e61-8c13-b9ca-8ce4-87cfcc4de59c 6d608e61-8c13-b9ca-8ce4-87cfcc4de59c}" - testing.go:149: server-283496e2-c19e-0c9b-c344-d497adf8e685: 2025-02-09T06:52:34.642Z [INFO] server-283496e2-c19e-0c9b-c344-d497adf8e685: aborting pipeline replication: peer="{Voter server-6d608e61-8c13-b9ca-8ce4-87cfcc4de59c 6d608e61-8c13-b9ca-8ce4-87cfcc4de59c}" ---- PASS: TestRaft_Voting (0.28s) +testLoggerAdapter verbose: server-f1529f8c-c92a-34e8-eab1-fd91c6ec97b8: 2024-01-08T00:31:09.722Z [INFO] server-f1529f8c-c92a-34e8-eab1-fd91c6ec97b8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f1529f8c-c92a-34e8-eab1-fd91c6ec97b8 Address:f1529f8c-c92a-34e8-eab1-fd91c6ec97b8} {Suffrage:Voter ID:server-c10dc359-caf1-e611-5c61-3ef4559792d0 Address:c10dc359-caf1-e611-5c61-3ef4559792d0} {Suffrage:Voter ID:server-329dec1e-6988-8a18-7460-fa0ada9e7bcd Address:329dec1e-6988-8a18-7460-fa0ada9e7bcd}]" + testing.go:149: server-f1529f8c-c92a-34e8-eab1-fd91c6ec97b8: 2024-01-08T00:31:09.722Z [INFO] server-f1529f8c-c92a-34e8-eab1-fd91c6ec97b8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f1529f8c-c92a-34e8-eab1-fd91c6ec97b8 Address:f1529f8c-c92a-34e8-eab1-fd91c6ec97b8} {Suffrage:Voter ID:server-c10dc359-caf1-e611-5c61-3ef4559792d0 Address:c10dc359-caf1-e611-5c61-3ef4559792d0} {Suffrage:Voter ID:server-329dec1e-6988-8a18-7460-fa0ada9e7bcd Address:329dec1e-6988-8a18-7460-fa0ada9e7bcd}]" +testLoggerAdapter verbose: server-c10dc359-caf1-e611-5c61-3ef4559792d0: 2024-01-08T00:31:09.722Z [INFO] server-c10dc359-caf1-e611-5c61-3ef4559792d0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f1529f8c-c92a-34e8-eab1-fd91c6ec97b8 Address:f1529f8c-c92a-34e8-eab1-fd91c6ec97b8} {Suffrage:Voter ID:server-c10dc359-caf1-e611-5c61-3ef4559792d0 Address:c10dc359-caf1-e611-5c61-3ef4559792d0} {Suffrage:Voter ID:server-329dec1e-6988-8a18-7460-fa0ada9e7bcd Address:329dec1e-6988-8a18-7460-fa0ada9e7bcd}]" + testing.go:149: server-c10dc359-caf1-e611-5c61-3ef4559792d0: 2024-01-08T00:31:09.722Z [INFO] server-c10dc359-caf1-e611-5c61-3ef4559792d0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f1529f8c-c92a-34e8-eab1-fd91c6ec97b8 Address:f1529f8c-c92a-34e8-eab1-fd91c6ec97b8} {Suffrage:Voter ID:server-c10dc359-caf1-e611-5c61-3ef4559792d0 Address:c10dc359-caf1-e611-5c61-3ef4559792d0} {Suffrage:Voter ID:server-329dec1e-6988-8a18-7460-fa0ada9e7bcd Address:329dec1e-6988-8a18-7460-fa0ada9e7bcd}]" +testLoggerAdapter verbose: server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: 2024-01-08T00:31:09.723Z [INFO] server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f1529f8c-c92a-34e8-eab1-fd91c6ec97b8 Address:f1529f8c-c92a-34e8-eab1-fd91c6ec97b8} {Suffrage:Voter ID:server-c10dc359-caf1-e611-5c61-3ef4559792d0 Address:c10dc359-caf1-e611-5c61-3ef4559792d0} {Suffrage:Voter ID:server-329dec1e-6988-8a18-7460-fa0ada9e7bcd Address:329dec1e-6988-8a18-7460-fa0ada9e7bcd}]" + testing.go:149: server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: 2024-01-08T00:31:09.723Z [INFO] server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f1529f8c-c92a-34e8-eab1-fd91c6ec97b8 Address:f1529f8c-c92a-34e8-eab1-fd91c6ec97b8} {Suffrage:Voter ID:server-c10dc359-caf1-e611-5c61-3ef4559792d0 Address:c10dc359-caf1-e611-5c61-3ef4559792d0} {Suffrage:Voter ID:server-329dec1e-6988-8a18-7460-fa0ada9e7bcd Address:329dec1e-6988-8a18-7460-fa0ada9e7bcd}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:09.723Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T00:31:09.723Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: server-f1529f8c-c92a-34e8-eab1-fd91c6ec97b8: 2024-01-08T00:31:09.723Z [INFO] server-f1529f8c-c92a-34e8-eab1-fd91c6ec97b8: entering follower state: follower="Node at f1529f8c-c92a-34e8-eab1-fd91c6ec97b8 [Follower]" leader= + testing.go:149: server-f1529f8c-c92a-34e8-eab1-fd91c6ec97b8: 2024-01-08T00:31:09.723Z [INFO] server-f1529f8c-c92a-34e8-eab1-fd91c6ec97b8: entering follower state: follower="Node at f1529f8c-c92a-34e8-eab1-fd91c6ec97b8 [Follower]" leader= +testLoggerAdapter verbose: server-c10dc359-caf1-e611-5c61-3ef4559792d0: 2024-01-08T00:31:09.723Z [INFO] server-c10dc359-caf1-e611-5c61-3ef4559792d0: entering follower state: follower="Node at c10dc359-caf1-e611-5c61-3ef4559792d0 [Follower]" leader= + testing.go:149: server-c10dc359-caf1-e611-5c61-3ef4559792d0: 2024-01-08T00:31:09.723Z [INFO] server-c10dc359-caf1-e611-5c61-3ef4559792d0: entering follower state: follower="Node at c10dc359-caf1-e611-5c61-3ef4559792d0 [Follower]" leader= +testLoggerAdapter verbose: server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: 2024-01-08T00:31:09.723Z [INFO] server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: entering follower state: follower="Node at 329dec1e-6988-8a18-7460-fa0ada9e7bcd [Follower]" leader= + testing.go:149: server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: 2024-01-08T00:31:09.723Z [INFO] server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: entering follower state: follower="Node at 329dec1e-6988-8a18-7460-fa0ada9e7bcd [Follower]" leader= +testLoggerAdapter verbose: server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: 2024-01-08T00:31:09.802Z [WARN] server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: 2024-01-08T00:31:09.802Z [WARN] server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: 2024-01-08T00:31:09.802Z [INFO] server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: entering candidate state: node="Node at 329dec1e-6988-8a18-7460-fa0ada9e7bcd [Candidate]" term=2 + testing.go:149: server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: 2024-01-08T00:31:09.802Z [INFO] server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: entering candidate state: node="Node at 329dec1e-6988-8a18-7460-fa0ada9e7bcd [Candidate]" term=2 +testLoggerAdapter verbose: server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: 2024-01-08T00:31:09.803Z [INFO] server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: election won: tally=2 + testing.go:149: server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: 2024-01-08T00:31:09.803Z [INFO] server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: election won: tally=2 +testLoggerAdapter verbose: server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: 2024-01-08T00:31:09.803Z [INFO] server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: entering leader state: leader="Node at 329dec1e-6988-8a18-7460-fa0ada9e7bcd [Leader]" + testing.go:149: server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: 2024-01-08T00:31:09.803Z [INFO] server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: entering leader state: leader="Node at 329dec1e-6988-8a18-7460-fa0ada9e7bcd [Leader]" +testLoggerAdapter verbose: server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: 2024-01-08T00:31:09.803Z [INFO] server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: added peer, starting replication: peer=server-f1529f8c-c92a-34e8-eab1-fd91c6ec97b8 + testing.go:149: server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: 2024-01-08T00:31:09.803Z [INFO] server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: added peer, starting replication: peer=server-f1529f8c-c92a-34e8-eab1-fd91c6ec97b8 +testLoggerAdapter verbose: server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: 2024-01-08T00:31:09.803Z [INFO] server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: added peer, starting replication: peer=server-c10dc359-caf1-e611-5c61-3ef4559792d0 + testing.go:149: server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: 2024-01-08T00:31:09.803Z [INFO] server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: added peer, starting replication: peer=server-c10dc359-caf1-e611-5c61-3ef4559792d0 +testLoggerAdapter verbose: server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: 2024-01-08T00:31:09.803Z [INFO] server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: pipelining replication: peer="{Voter server-c10dc359-caf1-e611-5c61-3ef4559792d0 c10dc359-caf1-e611-5c61-3ef4559792d0}" + testing.go:149: server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: 2024-01-08T00:31:09.803Z [INFO] server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: pipelining replication: peer="{Voter server-c10dc359-caf1-e611-5c61-3ef4559792d0 c10dc359-caf1-e611-5c61-3ef4559792d0}" +testLoggerAdapter verbose: server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: 2024-01-08T00:31:09.803Z [INFO] server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: pipelining replication: peer="{Voter server-f1529f8c-c92a-34e8-eab1-fd91c6ec97b8 f1529f8c-c92a-34e8-eab1-fd91c6ec97b8}" + testing.go:149: server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: 2024-01-08T00:31:09.803Z [INFO] server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: pipelining replication: peer="{Voter server-f1529f8c-c92a-34e8-eab1-fd91c6ec97b8 f1529f8c-c92a-34e8-eab1-fd91c6ec97b8}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:09.909Z [INFO] stable state for Follower reached at 2024-01-08 00:31:09.803439447 +0000 UTC m=+15.850328499 (2 nodes), 80.283947ms from start of poll, 80.652183ms from cluster start. Timeout at 2024-01-08 00:31:09.909083411 +0000 UTC m=+15.955972557, 105.644058ms after stability + testing.go:149: cluster: 2024-01-08T00:31:09.909Z [INFO] stable state for Follower reached at 2024-01-08 00:31:09.803439447 +0000 UTC m=+15.850328499 (2 nodes), 80.283947ms from start of poll, 80.652183ms from cluster start. Timeout at 2024-01-08 00:31:09.909083411 +0000 UTC m=+15.955972557, 105.644058ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:09.909Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:09.909Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:10.014Z [INFO] stable state for Leader reached at 2024-01-08 00:31:09.909228198 +0000 UTC m=+15.956117339 (1 nodes), 2.273µs from start of poll, 186.441023ms from cluster start. Timeout at 2024-01-08 00:31:10.014647857 +0000 UTC m=+16.061536899, 105.41956ms after stability + testing.go:149: cluster: 2024-01-08T00:31:10.014Z [INFO] stable state for Leader reached at 2024-01-08 00:31:09.909228198 +0000 UTC m=+15.956117339 (1 nodes), 2.273µs from start of poll, 186.441023ms from cluster start. Timeout at 2024-01-08 00:31:10.014647857 +0000 UTC m=+16.061536899, 105.41956ms after stability +testLoggerAdapter verbose: server-c10dc359-caf1-e611-5c61-3ef4559792d0: 2024-01-08T00:31:10.014Z [WARN] server-c10dc359-caf1-e611-5c61-3ef4559792d0: rejecting vote request since we have a leader: from=f1529f8c-c92a-34e8-eab1-fd91c6ec97b8 leader=329dec1e-6988-8a18-7460-fa0ada9e7bcd + testing.go:149: server-c10dc359-caf1-e611-5c61-3ef4559792d0: 2024-01-08T00:31:10.014Z [WARN] server-c10dc359-caf1-e611-5c61-3ef4559792d0: rejecting vote request since we have a leader: from=f1529f8c-c92a-34e8-eab1-fd91c6ec97b8 leader=329dec1e-6988-8a18-7460-fa0ada9e7bcd +testLoggerAdapter verbose: server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: 2024-01-08T00:31:10.014Z [INFO] server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: aborting pipeline replication: peer="{Voter server-c10dc359-caf1-e611-5c61-3ef4559792d0 c10dc359-caf1-e611-5c61-3ef4559792d0}" + testing.go:149: server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: 2024-01-08T00:31:10.014Z [INFO] server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: aborting pipeline replication: peer="{Voter server-c10dc359-caf1-e611-5c61-3ef4559792d0 c10dc359-caf1-e611-5c61-3ef4559792d0}" +testLoggerAdapter verbose: server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: 2024-01-08T00:31:10.014Z [INFO] server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: aborting pipeline replication: peer="{Voter server-f1529f8c-c92a-34e8-eab1-fd91c6ec97b8 f1529f8c-c92a-34e8-eab1-fd91c6ec97b8}" + testing.go:149: server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: 2024-01-08T00:31:10.014Z [INFO] server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: aborting pipeline replication: peer="{Voter server-f1529f8c-c92a-34e8-eab1-fd91c6ec97b8 f1529f8c-c92a-34e8-eab1-fd91c6ec97b8}" +testLoggerAdapter verbose: server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: 2024-01-08T00:31:10.065Z [ERROR] server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: failed to heartbeat to: peer=c10dc359-caf1-e611-5c61-3ef4559792d0 error="command timed out" + testing.go:149: server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: 2024-01-08T00:31:10.065Z [ERROR] server-329dec1e-6988-8a18-7460-fa0ada9e7bcd: failed to heartbeat to: peer=c10dc359-caf1-e611-5c61-3ef4559792d0 error="command timed out" +--- PASS: TestRaft_Voting (0.34s) === RUN TestRaft_ProtocolVersion_RejectRPC -testLoggerAdapter verbose: server-61297dd4-04b3-b836-70c7-64ffffa840fb: 2025-02-09T06:52:34.643Z [INFO] server-61297dd4-04b3-b836-70c7-64ffffa840fb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-61297dd4-04b3-b836-70c7-64ffffa840fb Address:61297dd4-04b3-b836-70c7-64ffffa840fb} {Suffrage:Voter ID:server-47382fa7-c90c-cefa-ea0f-586190dfd445 Address:47382fa7-c90c-cefa-ea0f-586190dfd445} {Suffrage:Voter ID:server-35de255a-c7a2-94c4-8446-aeabccc7ae3f Address:35de255a-c7a2-94c4-8446-aeabccc7ae3f}]" - testing.go:149: server-61297dd4-04b3-b836-70c7-64ffffa840fb: 2025-02-09T06:52:34.643Z [INFO] server-61297dd4-04b3-b836-70c7-64ffffa840fb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-61297dd4-04b3-b836-70c7-64ffffa840fb Address:61297dd4-04b3-b836-70c7-64ffffa840fb} {Suffrage:Voter ID:server-47382fa7-c90c-cefa-ea0f-586190dfd445 Address:47382fa7-c90c-cefa-ea0f-586190dfd445} {Suffrage:Voter ID:server-35de255a-c7a2-94c4-8446-aeabccc7ae3f Address:35de255a-c7a2-94c4-8446-aeabccc7ae3f}]" -testLoggerAdapter verbose: server-61297dd4-04b3-b836-70c7-64ffffa840fb: 2025-02-09T06:52:34.643Z [INFO] server-61297dd4-04b3-b836-70c7-64ffffa840fb: entering follower state: follower="Node at 61297dd4-04b3-b836-70c7-64ffffa840fb [Follower]" leader= - testing.go:149: server-61297dd4-04b3-b836-70c7-64ffffa840fb: 2025-02-09T06:52:34.643Z [INFO] server-61297dd4-04b3-b836-70c7-64ffffa840fb: entering follower state: follower="Node at 61297dd4-04b3-b836-70c7-64ffffa840fb [Follower]" leader= -testLoggerAdapter verbose: server-47382fa7-c90c-cefa-ea0f-586190dfd445: 2025-02-09T06:52:34.643Z [INFO] server-47382fa7-c90c-cefa-ea0f-586190dfd445: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-61297dd4-04b3-b836-70c7-64ffffa840fb Address:61297dd4-04b3-b836-70c7-64ffffa840fb} {Suffrage:Voter ID:server-47382fa7-c90c-cefa-ea0f-586190dfd445 Address:47382fa7-c90c-cefa-ea0f-586190dfd445} {Suffrage:Voter ID:server-35de255a-c7a2-94c4-8446-aeabccc7ae3f Address:35de255a-c7a2-94c4-8446-aeabccc7ae3f}]" - testing.go:149: server-47382fa7-c90c-cefa-ea0f-586190dfd445: 2025-02-09T06:52:34.643Z [INFO] server-47382fa7-c90c-cefa-ea0f-586190dfd445: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-61297dd4-04b3-b836-70c7-64ffffa840fb Address:61297dd4-04b3-b836-70c7-64ffffa840fb} {Suffrage:Voter ID:server-47382fa7-c90c-cefa-ea0f-586190dfd445 Address:47382fa7-c90c-cefa-ea0f-586190dfd445} {Suffrage:Voter ID:server-35de255a-c7a2-94c4-8446-aeabccc7ae3f Address:35de255a-c7a2-94c4-8446-aeabccc7ae3f}]" -testLoggerAdapter verbose: server-47382fa7-c90c-cefa-ea0f-586190dfd445: 2025-02-09T06:52:34.643Z [INFO] server-47382fa7-c90c-cefa-ea0f-586190dfd445: entering follower state: follower="Node at 47382fa7-c90c-cefa-ea0f-586190dfd445 [Follower]" leader= - testing.go:149: server-47382fa7-c90c-cefa-ea0f-586190dfd445: 2025-02-09T06:52:34.643Z [INFO] server-47382fa7-c90c-cefa-ea0f-586190dfd445: entering follower state: follower="Node at 47382fa7-c90c-cefa-ea0f-586190dfd445 [Follower]" leader= -testLoggerAdapter verbose: server-35de255a-c7a2-94c4-8446-aeabccc7ae3f: 2025-02-09T06:52:34.643Z [INFO] server-35de255a-c7a2-94c4-8446-aeabccc7ae3f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-61297dd4-04b3-b836-70c7-64ffffa840fb Address:61297dd4-04b3-b836-70c7-64ffffa840fb} {Suffrage:Voter ID:server-47382fa7-c90c-cefa-ea0f-586190dfd445 Address:47382fa7-c90c-cefa-ea0f-586190dfd445} {Suffrage:Voter ID:server-35de255a-c7a2-94c4-8446-aeabccc7ae3f Address:35de255a-c7a2-94c4-8446-aeabccc7ae3f}]" - testing.go:149: server-35de255a-c7a2-94c4-8446-aeabccc7ae3f: 2025-02-09T06:52:34.643Z [INFO] server-35de255a-c7a2-94c4-8446-aeabccc7ae3f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-61297dd4-04b3-b836-70c7-64ffffa840fb Address:61297dd4-04b3-b836-70c7-64ffffa840fb} {Suffrage:Voter ID:server-47382fa7-c90c-cefa-ea0f-586190dfd445 Address:47382fa7-c90c-cefa-ea0f-586190dfd445} {Suffrage:Voter ID:server-35de255a-c7a2-94c4-8446-aeabccc7ae3f Address:35de255a-c7a2-94c4-8446-aeabccc7ae3f}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:34.643Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T06:52:34.643Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-35de255a-c7a2-94c4-8446-aeabccc7ae3f: 2025-02-09T06:52:34.643Z [INFO] server-35de255a-c7a2-94c4-8446-aeabccc7ae3f: entering follower state: follower="Node at 35de255a-c7a2-94c4-8446-aeabccc7ae3f [Follower]" leader= - testing.go:149: server-35de255a-c7a2-94c4-8446-aeabccc7ae3f: 2025-02-09T06:52:34.643Z [INFO] server-35de255a-c7a2-94c4-8446-aeabccc7ae3f: entering follower state: follower="Node at 35de255a-c7a2-94c4-8446-aeabccc7ae3f [Follower]" leader= -testLoggerAdapter verbose: server-47382fa7-c90c-cefa-ea0f-586190dfd445: 2025-02-09T06:52:34.696Z [WARN] server-47382fa7-c90c-cefa-ea0f-586190dfd445: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-47382fa7-c90c-cefa-ea0f-586190dfd445: 2025-02-09T06:52:34.696Z [WARN] server-47382fa7-c90c-cefa-ea0f-586190dfd445: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-47382fa7-c90c-cefa-ea0f-586190dfd445: 2025-02-09T06:52:34.696Z [INFO] server-47382fa7-c90c-cefa-ea0f-586190dfd445: entering candidate state: node="Node at 47382fa7-c90c-cefa-ea0f-586190dfd445 [Candidate]" term=2 - testing.go:149: server-47382fa7-c90c-cefa-ea0f-586190dfd445: 2025-02-09T06:52:34.696Z [INFO] server-47382fa7-c90c-cefa-ea0f-586190dfd445: entering candidate state: node="Node at 47382fa7-c90c-cefa-ea0f-586190dfd445 [Candidate]" term=2 -testLoggerAdapter verbose: server-47382fa7-c90c-cefa-ea0f-586190dfd445: 2025-02-09T06:52:34.696Z [INFO] server-47382fa7-c90c-cefa-ea0f-586190dfd445: election won: tally=2 - testing.go:149: server-47382fa7-c90c-cefa-ea0f-586190dfd445: 2025-02-09T06:52:34.696Z [INFO] server-47382fa7-c90c-cefa-ea0f-586190dfd445: election won: tally=2 -testLoggerAdapter verbose: server-47382fa7-c90c-cefa-ea0f-586190dfd445: 2025-02-09T06:52:34.696Z [INFO] server-47382fa7-c90c-cefa-ea0f-586190dfd445: entering leader state: leader="Node at 47382fa7-c90c-cefa-ea0f-586190dfd445 [Leader]" - testing.go:149: server-47382fa7-c90c-cefa-ea0f-586190dfd445: 2025-02-09T06:52:34.696Z [INFO] server-47382fa7-c90c-cefa-ea0f-586190dfd445: entering leader state: leader="Node at 47382fa7-c90c-cefa-ea0f-586190dfd445 [Leader]" -testLoggerAdapter verbose: server-47382fa7-c90c-cefa-ea0f-586190dfd445: 2025-02-09T06:52:34.696Z [INFO] server-47382fa7-c90c-cefa-ea0f-586190dfd445: added peer, starting replication: peer=server-61297dd4-04b3-b836-70c7-64ffffa840fb - testing.go:149: server-47382fa7-c90c-cefa-ea0f-586190dfd445: 2025-02-09T06:52:34.696Z [INFO] server-47382fa7-c90c-cefa-ea0f-586190dfd445: added peer, starting replication: peer=server-61297dd4-04b3-b836-70c7-64ffffa840fb -testLoggerAdapter verbose: server-47382fa7-c90c-cefa-ea0f-586190dfd445: 2025-02-09T06:52:34.696Z [INFO] server-47382fa7-c90c-cefa-ea0f-586190dfd445: added peer, starting replication: peer=server-35de255a-c7a2-94c4-8446-aeabccc7ae3f - testing.go:149: server-47382fa7-c90c-cefa-ea0f-586190dfd445: 2025-02-09T06:52:34.696Z [INFO] server-47382fa7-c90c-cefa-ea0f-586190dfd445: added peer, starting replication: peer=server-35de255a-c7a2-94c4-8446-aeabccc7ae3f -testLoggerAdapter verbose: server-47382fa7-c90c-cefa-ea0f-586190dfd445: 2025-02-09T06:52:34.696Z [INFO] server-47382fa7-c90c-cefa-ea0f-586190dfd445: pipelining replication: peer="{Voter server-61297dd4-04b3-b836-70c7-64ffffa840fb 61297dd4-04b3-b836-70c7-64ffffa840fb}" - testing.go:149: server-47382fa7-c90c-cefa-ea0f-586190dfd445: 2025-02-09T06:52:34.696Z [INFO] server-47382fa7-c90c-cefa-ea0f-586190dfd445: pipelining replication: peer="{Voter server-61297dd4-04b3-b836-70c7-64ffffa840fb 61297dd4-04b3-b836-70c7-64ffffa840fb}" -testLoggerAdapter verbose: server-47382fa7-c90c-cefa-ea0f-586190dfd445: 2025-02-09T06:52:34.696Z [INFO] server-47382fa7-c90c-cefa-ea0f-586190dfd445: pipelining replication: peer="{Voter server-35de255a-c7a2-94c4-8446-aeabccc7ae3f 35de255a-c7a2-94c4-8446-aeabccc7ae3f}" - testing.go:149: server-47382fa7-c90c-cefa-ea0f-586190dfd445: 2025-02-09T06:52:34.696Z [INFO] server-47382fa7-c90c-cefa-ea0f-586190dfd445: pipelining replication: peer="{Voter server-35de255a-c7a2-94c4-8446-aeabccc7ae3f 35de255a-c7a2-94c4-8446-aeabccc7ae3f}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:34.802Z [INFO] stable state for Follower reached at 2025-02-09 06:52:34.696560386 +0000 UTC m=+15.675369161 (2 nodes), 52.910329ms from start of poll, 53.264005ms from cluster start. Timeout at 2025-02-09 06:52:34.802054063 +0000 UTC m=+15.780862835, 105.493674ms after stability - testing.go:149: cluster: 2025-02-09T06:52:34.802Z [INFO] stable state for Follower reached at 2025-02-09 06:52:34.696560386 +0000 UTC m=+15.675369161 (2 nodes), 52.910329ms from start of poll, 53.264005ms from cluster start. Timeout at 2025-02-09 06:52:34.802054063 +0000 UTC m=+15.780862835, 105.493674ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:34.802Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:34.802Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:34.907Z [INFO] stable state for Leader reached at 2025-02-09 06:52:34.802115152 +0000 UTC m=+15.780923920 (1 nodes), 447ns from start of poll, 158.818764ms from cluster start. Timeout at 2025-02-09 06:52:34.907778965 +0000 UTC m=+15.886587725, 105.663805ms after stability - testing.go:149: cluster: 2025-02-09T06:52:34.907Z [INFO] stable state for Leader reached at 2025-02-09 06:52:34.802115152 +0000 UTC m=+15.780923920 (1 nodes), 447ns from start of poll, 158.818764ms from cluster start. Timeout at 2025-02-09 06:52:34.907778965 +0000 UTC m=+15.886587725, 105.663805ms after stability -testLoggerAdapter verbose: server-47382fa7-c90c-cefa-ea0f-586190dfd445: 2025-02-09T06:52:34.907Z [INFO] server-47382fa7-c90c-cefa-ea0f-586190dfd445: aborting pipeline replication: peer="{Voter server-35de255a-c7a2-94c4-8446-aeabccc7ae3f 35de255a-c7a2-94c4-8446-aeabccc7ae3f}" - testing.go:149: server-47382fa7-c90c-cefa-ea0f-586190dfd445: 2025-02-09T06:52:34.907Z [INFO] server-47382fa7-c90c-cefa-ea0f-586190dfd445: aborting pipeline replication: peer="{Voter server-35de255a-c7a2-94c4-8446-aeabccc7ae3f 35de255a-c7a2-94c4-8446-aeabccc7ae3f}" -testLoggerAdapter verbose: server-47382fa7-c90c-cefa-ea0f-586190dfd445: 2025-02-09T06:52:34.907Z [INFO] server-47382fa7-c90c-cefa-ea0f-586190dfd445: aborting pipeline replication: peer="{Voter server-61297dd4-04b3-b836-70c7-64ffffa840fb 61297dd4-04b3-b836-70c7-64ffffa840fb}" - testing.go:149: server-47382fa7-c90c-cefa-ea0f-586190dfd445: 2025-02-09T06:52:34.907Z [INFO] server-47382fa7-c90c-cefa-ea0f-586190dfd445: aborting pipeline replication: peer="{Voter server-61297dd4-04b3-b836-70c7-64ffffa840fb 61297dd4-04b3-b836-70c7-64ffffa840fb}" ---- PASS: TestRaft_ProtocolVersion_RejectRPC (0.27s) +testLoggerAdapter verbose: server-8bc78e41-10f1-c37a-9251-fc6d2aeb45c0: 2024-01-08T00:31:10.066Z [INFO] server-8bc78e41-10f1-c37a-9251-fc6d2aeb45c0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8bc78e41-10f1-c37a-9251-fc6d2aeb45c0 Address:8bc78e41-10f1-c37a-9251-fc6d2aeb45c0} {Suffrage:Voter ID:server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122 Address:50419f4d-0d2e-80ad-c6de-5f74e0f3b122} {Suffrage:Voter ID:server-8236ebae-d6f0-77f0-856c-ceaecf9995a9 Address:8236ebae-d6f0-77f0-856c-ceaecf9995a9}]" + testing.go:149: server-8bc78e41-10f1-c37a-9251-fc6d2aeb45c0: 2024-01-08T00:31:10.066Z [INFO] server-8bc78e41-10f1-c37a-9251-fc6d2aeb45c0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8bc78e41-10f1-c37a-9251-fc6d2aeb45c0 Address:8bc78e41-10f1-c37a-9251-fc6d2aeb45c0} {Suffrage:Voter ID:server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122 Address:50419f4d-0d2e-80ad-c6de-5f74e0f3b122} {Suffrage:Voter ID:server-8236ebae-d6f0-77f0-856c-ceaecf9995a9 Address:8236ebae-d6f0-77f0-856c-ceaecf9995a9}]" +testLoggerAdapter verbose: server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: 2024-01-08T00:31:10.066Z [INFO] server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8bc78e41-10f1-c37a-9251-fc6d2aeb45c0 Address:8bc78e41-10f1-c37a-9251-fc6d2aeb45c0} {Suffrage:Voter ID:server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122 Address:50419f4d-0d2e-80ad-c6de-5f74e0f3b122} {Suffrage:Voter ID:server-8236ebae-d6f0-77f0-856c-ceaecf9995a9 Address:8236ebae-d6f0-77f0-856c-ceaecf9995a9}]" + testing.go:149: server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: 2024-01-08T00:31:10.066Z [INFO] server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8bc78e41-10f1-c37a-9251-fc6d2aeb45c0 Address:8bc78e41-10f1-c37a-9251-fc6d2aeb45c0} {Suffrage:Voter ID:server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122 Address:50419f4d-0d2e-80ad-c6de-5f74e0f3b122} {Suffrage:Voter ID:server-8236ebae-d6f0-77f0-856c-ceaecf9995a9 Address:8236ebae-d6f0-77f0-856c-ceaecf9995a9}]" +testLoggerAdapter verbose: server-8236ebae-d6f0-77f0-856c-ceaecf9995a9: 2024-01-08T00:31:10.066Z [INFO] server-8236ebae-d6f0-77f0-856c-ceaecf9995a9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8bc78e41-10f1-c37a-9251-fc6d2aeb45c0 Address:8bc78e41-10f1-c37a-9251-fc6d2aeb45c0} {Suffrage:Voter ID:server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122 Address:50419f4d-0d2e-80ad-c6de-5f74e0f3b122} {Suffrage:Voter ID:server-8236ebae-d6f0-77f0-856c-ceaecf9995a9 Address:8236ebae-d6f0-77f0-856c-ceaecf9995a9}]" + testing.go:149: server-8236ebae-d6f0-77f0-856c-ceaecf9995a9: 2024-01-08T00:31:10.066Z [INFO] server-8236ebae-d6f0-77f0-856c-ceaecf9995a9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8bc78e41-10f1-c37a-9251-fc6d2aeb45c0 Address:8bc78e41-10f1-c37a-9251-fc6d2aeb45c0} {Suffrage:Voter ID:server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122 Address:50419f4d-0d2e-80ad-c6de-5f74e0f3b122} {Suffrage:Voter ID:server-8236ebae-d6f0-77f0-856c-ceaecf9995a9 Address:8236ebae-d6f0-77f0-856c-ceaecf9995a9}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:10.066Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T00:31:10.066Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: server-8bc78e41-10f1-c37a-9251-fc6d2aeb45c0: 2024-01-08T00:31:10.066Z [INFO] server-8bc78e41-10f1-c37a-9251-fc6d2aeb45c0: entering follower state: follower="Node at 8bc78e41-10f1-c37a-9251-fc6d2aeb45c0 [Follower]" leader= + testing.go:149: server-8bc78e41-10f1-c37a-9251-fc6d2aeb45c0: 2024-01-08T00:31:10.066Z [INFO] server-8bc78e41-10f1-c37a-9251-fc6d2aeb45c0: entering follower state: follower="Node at 8bc78e41-10f1-c37a-9251-fc6d2aeb45c0 [Follower]" leader= +testLoggerAdapter verbose: server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: 2024-01-08T00:31:10.066Z [INFO] server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: entering follower state: follower="Node at 50419f4d-0d2e-80ad-c6de-5f74e0f3b122 [Follower]" leader= + testing.go:149: server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: 2024-01-08T00:31:10.066Z [INFO] server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: entering follower state: follower="Node at 50419f4d-0d2e-80ad-c6de-5f74e0f3b122 [Follower]" leader= +testLoggerAdapter verbose: server-8236ebae-d6f0-77f0-856c-ceaecf9995a9: 2024-01-08T00:31:10.066Z [INFO] server-8236ebae-d6f0-77f0-856c-ceaecf9995a9: entering follower state: follower="Node at 8236ebae-d6f0-77f0-856c-ceaecf9995a9 [Follower]" leader= + testing.go:149: server-8236ebae-d6f0-77f0-856c-ceaecf9995a9: 2024-01-08T00:31:10.066Z [INFO] server-8236ebae-d6f0-77f0-856c-ceaecf9995a9: entering follower state: follower="Node at 8236ebae-d6f0-77f0-856c-ceaecf9995a9 [Follower]" leader= +testLoggerAdapter verbose: server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: 2024-01-08T00:31:10.129Z [WARN] server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: 2024-01-08T00:31:10.129Z [WARN] server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: 2024-01-08T00:31:10.130Z [INFO] server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: entering candidate state: node="Node at 50419f4d-0d2e-80ad-c6de-5f74e0f3b122 [Candidate]" term=2 + testing.go:149: server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: 2024-01-08T00:31:10.130Z [INFO] server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: entering candidate state: node="Node at 50419f4d-0d2e-80ad-c6de-5f74e0f3b122 [Candidate]" term=2 +testLoggerAdapter verbose: server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: 2024-01-08T00:31:10.130Z [INFO] server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: election won: tally=2 + testing.go:149: server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: 2024-01-08T00:31:10.130Z [INFO] server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: election won: tally=2 +testLoggerAdapter verbose: server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: 2024-01-08T00:31:10.130Z [INFO] server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: entering leader state: leader="Node at 50419f4d-0d2e-80ad-c6de-5f74e0f3b122 [Leader]" + testing.go:149: server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: 2024-01-08T00:31:10.130Z [INFO] server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: entering leader state: leader="Node at 50419f4d-0d2e-80ad-c6de-5f74e0f3b122 [Leader]" +testLoggerAdapter verbose: server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: 2024-01-08T00:31:10.130Z [INFO] server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: added peer, starting replication: peer=server-8bc78e41-10f1-c37a-9251-fc6d2aeb45c0 + testing.go:149: server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: 2024-01-08T00:31:10.130Z [INFO] server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: added peer, starting replication: peer=server-8bc78e41-10f1-c37a-9251-fc6d2aeb45c0 +testLoggerAdapter verbose: server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: 2024-01-08T00:31:10.130Z [INFO] server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: added peer, starting replication: peer=server-8236ebae-d6f0-77f0-856c-ceaecf9995a9 + testing.go:149: server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: 2024-01-08T00:31:10.130Z [INFO] server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: added peer, starting replication: peer=server-8236ebae-d6f0-77f0-856c-ceaecf9995a9 +testLoggerAdapter verbose: server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: 2024-01-08T00:31:10.130Z [INFO] server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: pipelining replication: peer="{Voter server-8236ebae-d6f0-77f0-856c-ceaecf9995a9 8236ebae-d6f0-77f0-856c-ceaecf9995a9}" + testing.go:149: server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: 2024-01-08T00:31:10.130Z [INFO] server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: pipelining replication: peer="{Voter server-8236ebae-d6f0-77f0-856c-ceaecf9995a9 8236ebae-d6f0-77f0-856c-ceaecf9995a9}" +testLoggerAdapter verbose: server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: 2024-01-08T00:31:10.130Z [INFO] server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: pipelining replication: peer="{Voter server-8bc78e41-10f1-c37a-9251-fc6d2aeb45c0 8bc78e41-10f1-c37a-9251-fc6d2aeb45c0}" + testing.go:149: server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: 2024-01-08T00:31:10.130Z [INFO] server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: pipelining replication: peer="{Voter server-8bc78e41-10f1-c37a-9251-fc6d2aeb45c0 8bc78e41-10f1-c37a-9251-fc6d2aeb45c0}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:10.235Z [INFO] stable state for Follower reached at 2024-01-08 00:31:10.130442448 +0000 UTC m=+16.177331502 (2 nodes), 63.598614ms from start of poll, 64.064199ms from cluster start. Timeout at 2024-01-08 00:31:10.235870757 +0000 UTC m=+16.282759847, 105.428345ms after stability + testing.go:149: cluster: 2024-01-08T00:31:10.235Z [INFO] stable state for Follower reached at 2024-01-08 00:31:10.130442448 +0000 UTC m=+16.177331502 (2 nodes), 63.598614ms from start of poll, 64.064199ms from cluster start. Timeout at 2024-01-08 00:31:10.235870757 +0000 UTC m=+16.282759847, 105.428345ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:10.235Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:10.235Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:10.341Z [INFO] stable state for Leader reached at 2024-01-08 00:31:10.236015264 +0000 UTC m=+16.282904320 (1 nodes), 1.97µs from start of poll, 169.637017ms from cluster start. Timeout at 2024-01-08 00:31:10.341323374 +0000 UTC m=+16.388212552, 105.308232ms after stability + testing.go:149: cluster: 2024-01-08T00:31:10.341Z [INFO] stable state for Leader reached at 2024-01-08 00:31:10.236015264 +0000 UTC m=+16.282904320 (1 nodes), 1.97µs from start of poll, 169.637017ms from cluster start. Timeout at 2024-01-08 00:31:10.341323374 +0000 UTC m=+16.388212552, 105.308232ms after stability +testLoggerAdapter verbose: server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: 2024-01-08T00:31:10.341Z [INFO] server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: aborting pipeline replication: peer="{Voter server-8bc78e41-10f1-c37a-9251-fc6d2aeb45c0 8bc78e41-10f1-c37a-9251-fc6d2aeb45c0}" + testing.go:149: server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: 2024-01-08T00:31:10.341Z [INFO] server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: aborting pipeline replication: peer="{Voter server-8bc78e41-10f1-c37a-9251-fc6d2aeb45c0 8bc78e41-10f1-c37a-9251-fc6d2aeb45c0}" +testLoggerAdapter verbose: server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: 2024-01-08T00:31:10.341Z [INFO] server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: aborting pipeline replication: peer="{Voter server-8236ebae-d6f0-77f0-856c-ceaecf9995a9 8236ebae-d6f0-77f0-856c-ceaecf9995a9}" + testing.go:149: server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: 2024-01-08T00:31:10.341Z [INFO] server-50419f4d-0d2e-80ad-c6de-5f74e0f3b122: aborting pipeline replication: peer="{Voter server-8236ebae-d6f0-77f0-856c-ceaecf9995a9 8236ebae-d6f0-77f0-856c-ceaecf9995a9}" +--- PASS: TestRaft_ProtocolVersion_RejectRPC (0.28s) === RUN TestRaft_ProtocolVersion_Upgrade_1_2 -testLoggerAdapter verbose: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:34.908Z [INFO] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: initial configuration: index=1 servers="[{Suffrage:Voter ID:e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d Address:e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d} {Suffrage:Voter ID:d0462cea-3ad4-806d-b04c-dd014ba238ba Address:d0462cea-3ad4-806d-b04c-dd014ba238ba}]" - testing.go:149: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:34.908Z [INFO] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: initial configuration: index=1 servers="[{Suffrage:Voter ID:e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d Address:e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d} {Suffrage:Voter ID:d0462cea-3ad4-806d-b04c-dd014ba238ba Address:d0462cea-3ad4-806d-b04c-dd014ba238ba}]" -testLoggerAdapter verbose: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:34.908Z [INFO] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: entering follower state: follower="Node at e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d [Follower]" leader= - testing.go:149: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:34.908Z [INFO] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: entering follower state: follower="Node at e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d [Follower]" leader= -testLoggerAdapter verbose: d0462cea-3ad4-806d-b04c-dd014ba238ba: 2025-02-09T06:52:34.908Z [INFO] d0462cea-3ad4-806d-b04c-dd014ba238ba: initial configuration: index=1 servers="[{Suffrage:Voter ID:e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d Address:e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d} {Suffrage:Voter ID:d0462cea-3ad4-806d-b04c-dd014ba238ba Address:d0462cea-3ad4-806d-b04c-dd014ba238ba}]" - testing.go:149: d0462cea-3ad4-806d-b04c-dd014ba238ba: 2025-02-09T06:52:34.908Z [INFO] d0462cea-3ad4-806d-b04c-dd014ba238ba: initial configuration: index=1 servers="[{Suffrage:Voter ID:e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d Address:e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d} {Suffrage:Voter ID:d0462cea-3ad4-806d-b04c-dd014ba238ba Address:d0462cea-3ad4-806d-b04c-dd014ba238ba}]" -testLoggerAdapter verbose: d0462cea-3ad4-806d-b04c-dd014ba238ba: 2025-02-09T06:52:34.909Z [INFO] d0462cea-3ad4-806d-b04c-dd014ba238ba: entering follower state: follower="Node at d0462cea-3ad4-806d-b04c-dd014ba238ba [Follower]" leader= - testing.go:149: d0462cea-3ad4-806d-b04c-dd014ba238ba: 2025-02-09T06:52:34.909Z [INFO] d0462cea-3ad4-806d-b04c-dd014ba238ba: entering follower state: follower="Node at d0462cea-3ad4-806d-b04c-dd014ba238ba [Follower]" leader= -testLoggerAdapter verbose: 339b6473-3f55-1511-ad33-e1b79d113b2e: 2025-02-09T06:52:34.909Z [INFO] 339b6473-3f55-1511-ad33-e1b79d113b2e: initial configuration: index=0 servers=[] - testing.go:149: 339b6473-3f55-1511-ad33-e1b79d113b2e: 2025-02-09T06:52:34.909Z [INFO] 339b6473-3f55-1511-ad33-e1b79d113b2e: initial configuration: index=0 servers=[] -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:34.909Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:34.909Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: 339b6473-3f55-1511-ad33-e1b79d113b2e: 2025-02-09T06:52:34.909Z [INFO] 339b6473-3f55-1511-ad33-e1b79d113b2e: entering follower state: follower="Node at 339b6473-3f55-1511-ad33-e1b79d113b2e [Follower]" leader= - testing.go:149: 339b6473-3f55-1511-ad33-e1b79d113b2e: 2025-02-09T06:52:34.909Z [INFO] 339b6473-3f55-1511-ad33-e1b79d113b2e: entering follower state: follower="Node at 339b6473-3f55-1511-ad33-e1b79d113b2e [Follower]" leader= -testLoggerAdapter verbose: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:34.962Z [WARN] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: heartbeat timeout reached, starting election: last-leader= - testing.go:149: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:34.962Z [WARN] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:34.962Z [INFO] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: entering candidate state: node="Node at e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d [Candidate]" term=2 - testing.go:149: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:34.962Z [INFO] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: entering candidate state: node="Node at e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d [Candidate]" term=2 -testLoggerAdapter verbose: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:34.962Z [INFO] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: election won: tally=2 - testing.go:149: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:34.962Z [INFO] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: election won: tally=2 -testLoggerAdapter verbose: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:34.962Z [INFO] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: entering leader state: leader="Node at e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d [Leader]" - testing.go:149: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:34.962Z [INFO] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: entering leader state: leader="Node at e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d [Leader]" -testLoggerAdapter verbose: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:34.962Z [INFO] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: added peer, starting replication: peer=d0462cea-3ad4-806d-b04c-dd014ba238ba - testing.go:149: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:34.962Z [INFO] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: added peer, starting replication: peer=d0462cea-3ad4-806d-b04c-dd014ba238ba -testLoggerAdapter verbose: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:34.962Z [INFO] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: pipelining replication: peer="{Voter d0462cea-3ad4-806d-b04c-dd014ba238ba d0462cea-3ad4-806d-b04c-dd014ba238ba}" - testing.go:149: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:34.962Z [INFO] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: pipelining replication: peer="{Voter d0462cea-3ad4-806d-b04c-dd014ba238ba d0462cea-3ad4-806d-b04c-dd014ba238ba}" -testLoggerAdapter verbose: 339b6473-3f55-1511-ad33-e1b79d113b2e: 2025-02-09T06:52:34.970Z [WARN] 339b6473-3f55-1511-ad33-e1b79d113b2e: no known peers, aborting election - testing.go:149: 339b6473-3f55-1511-ad33-e1b79d113b2e: 2025-02-09T06:52:34.970Z [WARN] 339b6473-3f55-1511-ad33-e1b79d113b2e: no known peers, aborting election -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:35.067Z [INFO] stable state for Leader reached at 2025-02-09 06:52:34.962565532 +0000 UTC m=+15.941374307 (1 nodes), 53.250734ms from start of poll, 53.791882ms from cluster start. Timeout at 2025-02-09 06:52:35.067881532 +0000 UTC m=+16.046690290, 105.315983ms after stability - testing.go:149: cluster: 2025-02-09T06:52:35.067Z [INFO] stable state for Leader reached at 2025-02-09 06:52:34.962565532 +0000 UTC m=+15.941374307 (1 nodes), 53.250734ms from start of poll, 53.791882ms from cluster start. Timeout at 2025-02-09 06:52:35.067881532 +0000 UTC m=+16.046690290, 105.315983ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:35.067Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:35.067Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:35.173Z [INFO] stable state for Leader reached at 2025-02-09 06:52:35.067943708 +0000 UTC m=+16.046752474 (1 nodes), 488ns from start of poll, 159.170049ms from cluster start. Timeout at 2025-02-09 06:52:35.173468354 +0000 UTC m=+16.152277108, 105.524634ms after stability - testing.go:149: cluster: 2025-02-09T06:52:35.173Z [INFO] stable state for Leader reached at 2025-02-09 06:52:35.067943708 +0000 UTC m=+16.046752474 (1 nodes), 488ns from start of poll, 159.170049ms from cluster start. Timeout at 2025-02-09 06:52:35.173468354 +0000 UTC m=+16.152277108, 105.524634ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:35.173Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:35.173Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:35.279Z [INFO] stable state for Leader reached at 2025-02-09 06:52:35.173540947 +0000 UTC m=+16.152349712 (1 nodes), 458ns from start of poll, 264.767287ms from cluster start. Timeout at 2025-02-09 06:52:35.279053353 +0000 UTC m=+16.257862121, 105.512409ms after stability - testing.go:149: cluster: 2025-02-09T06:52:35.279Z [INFO] stable state for Leader reached at 2025-02-09 06:52:35.173540947 +0000 UTC m=+16.152349712 (1 nodes), 458ns from start of poll, 264.767287ms from cluster start. Timeout at 2025-02-09 06:52:35.279053353 +0000 UTC m=+16.257862121, 105.512409ms after stability -testLoggerAdapter verbose: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:35.279Z [INFO] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: updating configuration: command=AddStaging server-id=339b6473-3f55-1511-ad33-e1b79d113b2e server-addr=339b6473-3f55-1511-ad33-e1b79d113b2e servers="[{Suffrage:Voter ID:e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d Address:e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d} {Suffrage:Voter ID:d0462cea-3ad4-806d-b04c-dd014ba238ba Address:d0462cea-3ad4-806d-b04c-dd014ba238ba} {Suffrage:Voter ID:339b6473-3f55-1511-ad33-e1b79d113b2e Address:339b6473-3f55-1511-ad33-e1b79d113b2e}]" - testing.go:149: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:35.279Z [INFO] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: updating configuration: command=AddStaging server-id=339b6473-3f55-1511-ad33-e1b79d113b2e server-addr=339b6473-3f55-1511-ad33-e1b79d113b2e servers="[{Suffrage:Voter ID:e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d Address:e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d} {Suffrage:Voter ID:d0462cea-3ad4-806d-b04c-dd014ba238ba Address:d0462cea-3ad4-806d-b04c-dd014ba238ba} {Suffrage:Voter ID:339b6473-3f55-1511-ad33-e1b79d113b2e Address:339b6473-3f55-1511-ad33-e1b79d113b2e}]" -testLoggerAdapter verbose: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:35.279Z [INFO] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: added peer, starting replication: peer=339b6473-3f55-1511-ad33-e1b79d113b2e - testing.go:149: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:35.279Z [INFO] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: added peer, starting replication: peer=339b6473-3f55-1511-ad33-e1b79d113b2e -testLoggerAdapter verbose: 339b6473-3f55-1511-ad33-e1b79d113b2e: 2025-02-09T06:52:35.279Z [WARN] 339b6473-3f55-1511-ad33-e1b79d113b2e: failed to get previous log: previous-index=3 last-index=0 error="log not found" - testing.go:149: 339b6473-3f55-1511-ad33-e1b79d113b2e: 2025-02-09T06:52:35.279Z [WARN] 339b6473-3f55-1511-ad33-e1b79d113b2e: failed to get previous log: previous-index=3 last-index=0 error="log not found" -testLoggerAdapter verbose: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:35.279Z [WARN] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: appendEntries rejected, sending older logs: peer="{Voter 339b6473-3f55-1511-ad33-e1b79d113b2e 339b6473-3f55-1511-ad33-e1b79d113b2e}" next=1 - testing.go:149: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:35.279Z [WARN] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: appendEntries rejected, sending older logs: peer="{Voter 339b6473-3f55-1511-ad33-e1b79d113b2e 339b6473-3f55-1511-ad33-e1b79d113b2e}" next=1 -testLoggerAdapter verbose: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:35.279Z [INFO] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: pipelining replication: peer="{Voter 339b6473-3f55-1511-ad33-e1b79d113b2e 339b6473-3f55-1511-ad33-e1b79d113b2e}" - testing.go:149: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:35.279Z [INFO] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: pipelining replication: peer="{Voter 339b6473-3f55-1511-ad33-e1b79d113b2e 339b6473-3f55-1511-ad33-e1b79d113b2e}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:35.279Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:35.279Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:35.385Z [INFO] stable state for Leader reached at 2025-02-09 06:52:35.279407196 +0000 UTC m=+16.258215960 (1 nodes), 504ns from start of poll, 370.633535ms from cluster start. Timeout at 2025-02-09 06:52:35.385082758 +0000 UTC m=+16.363891517, 105.675557ms after stability - testing.go:149: cluster: 2025-02-09T06:52:35.385Z [INFO] stable state for Leader reached at 2025-02-09 06:52:35.279407196 +0000 UTC m=+16.258215960 (1 nodes), 504ns from start of poll, 370.633535ms from cluster start. Timeout at 2025-02-09 06:52:35.385082758 +0000 UTC m=+16.363891517, 105.675557ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:35.385Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:35.385Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:35.490Z [INFO] stable state for Leader reached at 2025-02-09 06:52:35.385181518 +0000 UTC m=+16.363990283 (1 nodes), 466ns from start of poll, 476.407858ms from cluster start. Timeout at 2025-02-09 06:52:35.490796279 +0000 UTC m=+16.469605048, 105.614765ms after stability - testing.go:149: cluster: 2025-02-09T06:52:35.490Z [INFO] stable state for Leader reached at 2025-02-09 06:52:35.385181518 +0000 UTC m=+16.363990283 (1 nodes), 466ns from start of poll, 476.407858ms from cluster start. Timeout at 2025-02-09 06:52:35.490796279 +0000 UTC m=+16.469605048, 105.614765ms after stability -testLoggerAdapter verbose: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:35.490Z [INFO] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: updating configuration: command=RemoveServer server-id=339b6473-3f55-1511-ad33-e1b79d113b2e server-addr= servers="[{Suffrage:Voter ID:e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d Address:e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d} {Suffrage:Voter ID:d0462cea-3ad4-806d-b04c-dd014ba238ba Address:d0462cea-3ad4-806d-b04c-dd014ba238ba}]" - testing.go:149: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:35.490Z [INFO] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: updating configuration: command=RemoveServer server-id=339b6473-3f55-1511-ad33-e1b79d113b2e server-addr= servers="[{Suffrage:Voter ID:e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d Address:e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d} {Suffrage:Voter ID:d0462cea-3ad4-806d-b04c-dd014ba238ba Address:d0462cea-3ad4-806d-b04c-dd014ba238ba}]" -testLoggerAdapter verbose: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:35.490Z [INFO] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: removed peer, stopping replication: peer=339b6473-3f55-1511-ad33-e1b79d113b2e last-index=4 - testing.go:149: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:35.490Z [INFO] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: removed peer, stopping replication: peer=339b6473-3f55-1511-ad33-e1b79d113b2e last-index=4 -testLoggerAdapter verbose: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:35.491Z [INFO] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: aborting pipeline replication: peer="{Voter 339b6473-3f55-1511-ad33-e1b79d113b2e 339b6473-3f55-1511-ad33-e1b79d113b2e}" - testing.go:149: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:35.491Z [INFO] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: aborting pipeline replication: peer="{Voter 339b6473-3f55-1511-ad33-e1b79d113b2e 339b6473-3f55-1511-ad33-e1b79d113b2e}" -testLoggerAdapter verbose: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:35.491Z [INFO] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: aborting pipeline replication: peer="{Voter d0462cea-3ad4-806d-b04c-dd014ba238ba d0462cea-3ad4-806d-b04c-dd014ba238ba}" - testing.go:149: e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: 2025-02-09T06:52:35.491Z [INFO] e4741432-ae0e-b56f-7bb9-6d8b50e4cc7d: aborting pipeline replication: peer="{Voter d0462cea-3ad4-806d-b04c-dd014ba238ba d0462cea-3ad4-806d-b04c-dd014ba238ba}" ---- PASS: TestRaft_ProtocolVersion_Upgrade_1_2 (0.58s) +testLoggerAdapter verbose: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.342Z [INFO] b9457696-ac28-8234-e302-d4d80c0ecbab: initial configuration: index=1 servers="[{Suffrage:Voter ID:b9457696-ac28-8234-e302-d4d80c0ecbab Address:b9457696-ac28-8234-e302-d4d80c0ecbab} {Suffrage:Voter ID:a103ce52-02c2-7fc3-414c-c6f567929843 Address:a103ce52-02c2-7fc3-414c-c6f567929843}]" + testing.go:149: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.342Z [INFO] b9457696-ac28-8234-e302-d4d80c0ecbab: initial configuration: index=1 servers="[{Suffrage:Voter ID:b9457696-ac28-8234-e302-d4d80c0ecbab Address:b9457696-ac28-8234-e302-d4d80c0ecbab} {Suffrage:Voter ID:a103ce52-02c2-7fc3-414c-c6f567929843 Address:a103ce52-02c2-7fc3-414c-c6f567929843}]" +testLoggerAdapter verbose: a103ce52-02c2-7fc3-414c-c6f567929843: 2024-01-08T00:31:10.342Z [INFO] a103ce52-02c2-7fc3-414c-c6f567929843: initial configuration: index=1 servers="[{Suffrage:Voter ID:b9457696-ac28-8234-e302-d4d80c0ecbab Address:b9457696-ac28-8234-e302-d4d80c0ecbab} {Suffrage:Voter ID:a103ce52-02c2-7fc3-414c-c6f567929843 Address:a103ce52-02c2-7fc3-414c-c6f567929843}]" + testing.go:149: a103ce52-02c2-7fc3-414c-c6f567929843: 2024-01-08T00:31:10.342Z [INFO] a103ce52-02c2-7fc3-414c-c6f567929843: initial configuration: index=1 servers="[{Suffrage:Voter ID:b9457696-ac28-8234-e302-d4d80c0ecbab Address:b9457696-ac28-8234-e302-d4d80c0ecbab} {Suffrage:Voter ID:a103ce52-02c2-7fc3-414c-c6f567929843 Address:a103ce52-02c2-7fc3-414c-c6f567929843}]" +testLoggerAdapter verbose: 5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5: 2024-01-08T00:31:10.342Z [INFO] 5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5: initial configuration: index=0 servers=[] + testing.go:149: 5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5: 2024-01-08T00:31:10.342Z [INFO] 5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5: initial configuration: index=0 servers=[] +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:10.342Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:10.342Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.343Z [INFO] b9457696-ac28-8234-e302-d4d80c0ecbab: entering follower state: follower="Node at b9457696-ac28-8234-e302-d4d80c0ecbab [Follower]" leader= + testing.go:149: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.343Z [INFO] b9457696-ac28-8234-e302-d4d80c0ecbab: entering follower state: follower="Node at b9457696-ac28-8234-e302-d4d80c0ecbab [Follower]" leader= +testLoggerAdapter verbose: a103ce52-02c2-7fc3-414c-c6f567929843: 2024-01-08T00:31:10.343Z [INFO] a103ce52-02c2-7fc3-414c-c6f567929843: entering follower state: follower="Node at a103ce52-02c2-7fc3-414c-c6f567929843 [Follower]" leader= + testing.go:149: a103ce52-02c2-7fc3-414c-c6f567929843: 2024-01-08T00:31:10.343Z [INFO] a103ce52-02c2-7fc3-414c-c6f567929843: entering follower state: follower="Node at a103ce52-02c2-7fc3-414c-c6f567929843 [Follower]" leader= +testLoggerAdapter verbose: 5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5: 2024-01-08T00:31:10.343Z [INFO] 5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5: entering follower state: follower="Node at 5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5 [Follower]" leader= + testing.go:149: 5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5: 2024-01-08T00:31:10.343Z [INFO] 5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5: entering follower state: follower="Node at 5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5 [Follower]" leader= +testLoggerAdapter verbose: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.398Z [WARN] b9457696-ac28-8234-e302-d4d80c0ecbab: heartbeat timeout reached, starting election: last-leader= + testing.go:149: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.398Z [WARN] b9457696-ac28-8234-e302-d4d80c0ecbab: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.398Z [INFO] b9457696-ac28-8234-e302-d4d80c0ecbab: entering candidate state: node="Node at b9457696-ac28-8234-e302-d4d80c0ecbab [Candidate]" term=2 + testing.go:149: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.398Z [INFO] b9457696-ac28-8234-e302-d4d80c0ecbab: entering candidate state: node="Node at b9457696-ac28-8234-e302-d4d80c0ecbab [Candidate]" term=2 +testLoggerAdapter verbose: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.398Z [INFO] b9457696-ac28-8234-e302-d4d80c0ecbab: election won: tally=2 + testing.go:149: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.398Z [INFO] b9457696-ac28-8234-e302-d4d80c0ecbab: election won: tally=2 +testLoggerAdapter verbose: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.399Z [INFO] b9457696-ac28-8234-e302-d4d80c0ecbab: entering leader state: leader="Node at b9457696-ac28-8234-e302-d4d80c0ecbab [Leader]" + testing.go:149: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.399Z [INFO] b9457696-ac28-8234-e302-d4d80c0ecbab: entering leader state: leader="Node at b9457696-ac28-8234-e302-d4d80c0ecbab [Leader]" +testLoggerAdapter verbose: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.399Z [INFO] b9457696-ac28-8234-e302-d4d80c0ecbab: added peer, starting replication: peer=a103ce52-02c2-7fc3-414c-c6f567929843 + testing.go:149: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.399Z [INFO] b9457696-ac28-8234-e302-d4d80c0ecbab: added peer, starting replication: peer=a103ce52-02c2-7fc3-414c-c6f567929843 +testLoggerAdapter verbose: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.399Z [INFO] b9457696-ac28-8234-e302-d4d80c0ecbab: pipelining replication: peer="{Voter a103ce52-02c2-7fc3-414c-c6f567929843 a103ce52-02c2-7fc3-414c-c6f567929843}" + testing.go:149: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.399Z [INFO] b9457696-ac28-8234-e302-d4d80c0ecbab: pipelining replication: peer="{Voter a103ce52-02c2-7fc3-414c-c6f567929843 a103ce52-02c2-7fc3-414c-c6f567929843}" +testLoggerAdapter verbose: 5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5: 2024-01-08T00:31:10.434Z [WARN] 5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5: no known peers, aborting election + testing.go:149: 5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5: 2024-01-08T00:31:10.434Z [WARN] 5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5: no known peers, aborting election +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:10.504Z [INFO] stable state for Leader reached at 2024-01-08 00:31:10.399205932 +0000 UTC m=+16.446095073 (1 nodes), 56.210767ms from start of poll, 56.664996ms from cluster start. Timeout at 2024-01-08 00:31:10.504279951 +0000 UTC m=+16.551169081, 105.074008ms after stability + testing.go:149: cluster: 2024-01-08T00:31:10.504Z [INFO] stable state for Leader reached at 2024-01-08 00:31:10.399205932 +0000 UTC m=+16.446095073 (1 nodes), 56.210767ms from start of poll, 56.664996ms from cluster start. Timeout at 2024-01-08 00:31:10.504279951 +0000 UTC m=+16.551169081, 105.074008ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:10.504Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:10.504Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:10.610Z [INFO] stable state for Leader reached at 2024-01-08 00:31:10.504469733 +0000 UTC m=+16.551358873 (1 nodes), 2.172µs from start of poll, 161.928796ms from cluster start. Timeout at 2024-01-08 00:31:10.609961397 +0000 UTC m=+16.656850443, 105.49157ms after stability + testing.go:149: cluster: 2024-01-08T00:31:10.610Z [INFO] stable state for Leader reached at 2024-01-08 00:31:10.504469733 +0000 UTC m=+16.551358873 (1 nodes), 2.172µs from start of poll, 161.928796ms from cluster start. Timeout at 2024-01-08 00:31:10.609961397 +0000 UTC m=+16.656850443, 105.49157ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:10.610Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:10.610Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:10.715Z [INFO] stable state for Leader reached at 2024-01-08 00:31:10.610107923 +0000 UTC m=+16.656996976 (1 nodes), 2.233µs from start of poll, 267.566899ms from cluster start. Timeout at 2024-01-08 00:31:10.715696827 +0000 UTC m=+16.762585944, 105.588968ms after stability + testing.go:149: cluster: 2024-01-08T00:31:10.715Z [INFO] stable state for Leader reached at 2024-01-08 00:31:10.610107923 +0000 UTC m=+16.656996976 (1 nodes), 2.233µs from start of poll, 267.566899ms from cluster start. Timeout at 2024-01-08 00:31:10.715696827 +0000 UTC m=+16.762585944, 105.588968ms after stability +testLoggerAdapter verbose: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.715Z [INFO] b9457696-ac28-8234-e302-d4d80c0ecbab: updating configuration: command=AddStaging server-id=5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5 server-addr=5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5 servers="[{Suffrage:Voter ID:b9457696-ac28-8234-e302-d4d80c0ecbab Address:b9457696-ac28-8234-e302-d4d80c0ecbab} {Suffrage:Voter ID:a103ce52-02c2-7fc3-414c-c6f567929843 Address:a103ce52-02c2-7fc3-414c-c6f567929843} {Suffrage:Voter ID:5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5 Address:5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5}]" + testing.go:149: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.715Z [INFO] b9457696-ac28-8234-e302-d4d80c0ecbab: updating configuration: command=AddStaging server-id=5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5 server-addr=5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5 servers="[{Suffrage:Voter ID:b9457696-ac28-8234-e302-d4d80c0ecbab Address:b9457696-ac28-8234-e302-d4d80c0ecbab} {Suffrage:Voter ID:a103ce52-02c2-7fc3-414c-c6f567929843 Address:a103ce52-02c2-7fc3-414c-c6f567929843} {Suffrage:Voter ID:5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5 Address:5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5}]" +testLoggerAdapter verbose: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.715Z [INFO] b9457696-ac28-8234-e302-d4d80c0ecbab: added peer, starting replication: peer=5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5 + testing.go:149: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.715Z [INFO] b9457696-ac28-8234-e302-d4d80c0ecbab: added peer, starting replication: peer=5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5 +testLoggerAdapter verbose: 5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5: 2024-01-08T00:31:10.716Z [WARN] 5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5: failed to get previous log: previous-index=3 last-index=0 error="log not found" + testing.go:149: 5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5: 2024-01-08T00:31:10.716Z [WARN] 5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5: failed to get previous log: previous-index=3 last-index=0 error="log not found" +testLoggerAdapter verbose: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.716Z [WARN] b9457696-ac28-8234-e302-d4d80c0ecbab: appendEntries rejected, sending older logs: peer="{Voter 5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5 5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5}" next=1 + testing.go:149: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.716Z [WARN] b9457696-ac28-8234-e302-d4d80c0ecbab: appendEntries rejected, sending older logs: peer="{Voter 5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5 5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5}" next=1 +testLoggerAdapter verbose: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.716Z [INFO] b9457696-ac28-8234-e302-d4d80c0ecbab: pipelining replication: peer="{Voter 5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5 5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5}" + testing.go:149: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.716Z [INFO] b9457696-ac28-8234-e302-d4d80c0ecbab: pipelining replication: peer="{Voter 5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5 5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:10.716Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:10.716Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:10.821Z [INFO] stable state for Leader reached at 2024-01-08 00:31:10.716318794 +0000 UTC m=+16.763207912 (1 nodes), 2.071µs from start of poll, 373.777835ms from cluster start. Timeout at 2024-01-08 00:31:10.821598467 +0000 UTC m=+16.868487631, 105.279719ms after stability + testing.go:149: cluster: 2024-01-08T00:31:10.821Z [INFO] stable state for Leader reached at 2024-01-08 00:31:10.716318794 +0000 UTC m=+16.763207912 (1 nodes), 2.071µs from start of poll, 373.777835ms from cluster start. Timeout at 2024-01-08 00:31:10.821598467 +0000 UTC m=+16.868487631, 105.279719ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:10.821Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:10.821Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:10.927Z [INFO] stable state for Leader reached at 2024-01-08 00:31:10.821766659 +0000 UTC m=+16.868655796 (1 nodes), 2.168µs from start of poll, 479.225719ms from cluster start. Timeout at 2024-01-08 00:31:10.927784329 +0000 UTC m=+16.974673372, 106.017576ms after stability + testing.go:149: cluster: 2024-01-08T00:31:10.927Z [INFO] stable state for Leader reached at 2024-01-08 00:31:10.821766659 +0000 UTC m=+16.868655796 (1 nodes), 2.168µs from start of poll, 479.225719ms from cluster start. Timeout at 2024-01-08 00:31:10.927784329 +0000 UTC m=+16.974673372, 106.017576ms after stability +testLoggerAdapter verbose: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.927Z [INFO] b9457696-ac28-8234-e302-d4d80c0ecbab: updating configuration: command=RemoveServer server-id=5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5 server-addr= servers="[{Suffrage:Voter ID:b9457696-ac28-8234-e302-d4d80c0ecbab Address:b9457696-ac28-8234-e302-d4d80c0ecbab} {Suffrage:Voter ID:a103ce52-02c2-7fc3-414c-c6f567929843 Address:a103ce52-02c2-7fc3-414c-c6f567929843}]" + testing.go:149: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.927Z [INFO] b9457696-ac28-8234-e302-d4d80c0ecbab: updating configuration: command=RemoveServer server-id=5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5 server-addr= servers="[{Suffrage:Voter ID:b9457696-ac28-8234-e302-d4d80c0ecbab Address:b9457696-ac28-8234-e302-d4d80c0ecbab} {Suffrage:Voter ID:a103ce52-02c2-7fc3-414c-c6f567929843 Address:a103ce52-02c2-7fc3-414c-c6f567929843}]" +testLoggerAdapter verbose: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.928Z [INFO] b9457696-ac28-8234-e302-d4d80c0ecbab: removed peer, stopping replication: peer=5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5 last-index=4 + testing.go:149: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.928Z [INFO] b9457696-ac28-8234-e302-d4d80c0ecbab: removed peer, stopping replication: peer=5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5 last-index=4 +testLoggerAdapter verbose: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.928Z [INFO] b9457696-ac28-8234-e302-d4d80c0ecbab: aborting pipeline replication: peer="{Voter 5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5 5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5}" + testing.go:149: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.928Z [INFO] b9457696-ac28-8234-e302-d4d80c0ecbab: aborting pipeline replication: peer="{Voter 5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5 5e7010f2-f7be-ebb0-77fc-bfc3f13e34f5}" +testLoggerAdapter verbose: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.928Z [INFO] b9457696-ac28-8234-e302-d4d80c0ecbab: aborting pipeline replication: peer="{Voter a103ce52-02c2-7fc3-414c-c6f567929843 a103ce52-02c2-7fc3-414c-c6f567929843}" + testing.go:149: b9457696-ac28-8234-e302-d4d80c0ecbab: 2024-01-08T00:31:10.928Z [INFO] b9457696-ac28-8234-e302-d4d80c0ecbab: aborting pipeline replication: peer="{Voter a103ce52-02c2-7fc3-414c-c6f567929843 a103ce52-02c2-7fc3-414c-c6f567929843}" +--- PASS: TestRaft_ProtocolVersion_Upgrade_1_2 (0.59s) === RUN TestRaft_ProtocolVersion_Upgrade_2_3 -testLoggerAdapter verbose: 0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d: 2025-02-09T06:52:35.491Z [INFO] 0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d: initial configuration: index=1 servers="[{Suffrage:Voter ID:0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d Address:0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d} {Suffrage:Voter ID:2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1 Address:2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1}]" - testing.go:149: 0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d: 2025-02-09T06:52:35.491Z [INFO] 0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d: initial configuration: index=1 servers="[{Suffrage:Voter ID:0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d Address:0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d} {Suffrage:Voter ID:2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1 Address:2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1}]" -testLoggerAdapter verbose: 0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d: 2025-02-09T06:52:35.492Z [INFO] 0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d: entering follower state: follower="Node at 0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d [Follower]" leader= - testing.go:149: 0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d: 2025-02-09T06:52:35.492Z [INFO] 0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d: entering follower state: follower="Node at 0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d [Follower]" leader= -testLoggerAdapter verbose: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.492Z [INFO] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: initial configuration: index=1 servers="[{Suffrage:Voter ID:0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d Address:0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d} {Suffrage:Voter ID:2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1 Address:2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1}]" - testing.go:149: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.492Z [INFO] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: initial configuration: index=1 servers="[{Suffrage:Voter ID:0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d Address:0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d} {Suffrage:Voter ID:2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1 Address:2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:35.493Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T06:52:35.493Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.493Z [INFO] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: entering follower state: follower="Node at 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1 [Follower]" leader= - testing.go:149: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.493Z [INFO] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: entering follower state: follower="Node at 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1 [Follower]" leader= -testLoggerAdapter verbose: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.547Z [WARN] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: heartbeat timeout reached, starting election: last-leader= - testing.go:149: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.547Z [WARN] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.547Z [INFO] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: entering candidate state: node="Node at 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1 [Candidate]" term=2 - testing.go:149: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.547Z [INFO] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: entering candidate state: node="Node at 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1 [Candidate]" term=2 -testLoggerAdapter verbose: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.547Z [INFO] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: election won: tally=2 - testing.go:149: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.547Z [INFO] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: election won: tally=2 -testLoggerAdapter verbose: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.547Z [INFO] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: entering leader state: leader="Node at 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1 [Leader]" - testing.go:149: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.547Z [INFO] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: entering leader state: leader="Node at 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1 [Leader]" -testLoggerAdapter verbose: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.547Z [INFO] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: added peer, starting replication: peer=0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d - testing.go:149: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.547Z [INFO] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: added peer, starting replication: peer=0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d -testLoggerAdapter verbose: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.548Z [INFO] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: pipelining replication: peer="{Voter 0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d 0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d}" - testing.go:149: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.548Z [INFO] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: pipelining replication: peer="{Voter 0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d 0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:35.653Z [INFO] stable state for Follower reached at 2025-02-09 06:52:35.547984666 +0000 UTC m=+16.526793452 (1 nodes), 54.84146ms from start of poll, 56.070103ms from cluster start. Timeout at 2025-02-09 06:52:35.653052271 +0000 UTC m=+16.631861041, 105.067589ms after stability - testing.go:149: cluster: 2025-02-09T06:52:35.653Z [INFO] stable state for Follower reached at 2025-02-09 06:52:35.547984666 +0000 UTC m=+16.526793452 (1 nodes), 54.84146ms from start of poll, 56.070103ms from cluster start. Timeout at 2025-02-09 06:52:35.653052271 +0000 UTC m=+16.631861041, 105.067589ms after stability -testLoggerAdapter verbose: server-bdfe75b5-1662-6cf4-9646-fa7a9f725c92: 2025-02-09T06:52:35.653Z [INFO] server-bdfe75b5-1662-6cf4-9646-fa7a9f725c92: initial configuration: index=0 servers=[] - testing.go:149: server-bdfe75b5-1662-6cf4-9646-fa7a9f725c92: 2025-02-09T06:52:35.653Z [INFO] server-bdfe75b5-1662-6cf4-9646-fa7a9f725c92: initial configuration: index=0 servers=[] -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:35.653Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-bdfe75b5-1662-6cf4-9646-fa7a9f725c92: 2025-02-09T06:52:35.653Z [INFO] server-bdfe75b5-1662-6cf4-9646-fa7a9f725c92: entering follower state: follower="Node at bdfe75b5-1662-6cf4-9646-fa7a9f725c92 [Follower]" leader= - testing.go:149: cluster: 2025-02-09T06:52:35.653Z [INFO] starting stability test: raft-state=Leader - testing.go:149: server-bdfe75b5-1662-6cf4-9646-fa7a9f725c92: 2025-02-09T06:52:35.653Z [INFO] server-bdfe75b5-1662-6cf4-9646-fa7a9f725c92: entering follower state: follower="Node at bdfe75b5-1662-6cf4-9646-fa7a9f725c92 [Follower]" leader= -testLoggerAdapter verbose: server-bdfe75b5-1662-6cf4-9646-fa7a9f725c92: 2025-02-09T06:52:35.708Z [WARN] server-bdfe75b5-1662-6cf4-9646-fa7a9f725c92: no known peers, aborting election - testing.go:149: server-bdfe75b5-1662-6cf4-9646-fa7a9f725c92: 2025-02-09T06:52:35.708Z [WARN] server-bdfe75b5-1662-6cf4-9646-fa7a9f725c92: no known peers, aborting election -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:35.758Z [INFO] stable state for Leader reached at 2025-02-09 06:52:35.65347388 +0000 UTC m=+16.632282681 (1 nodes), 951ns from start of poll, 161.559332ms from cluster start. Timeout at 2025-02-09 06:52:35.758820084 +0000 UTC m=+16.737628844, 105.346163ms after stability - testing.go:149: cluster: 2025-02-09T06:52:35.758Z [INFO] stable state for Leader reached at 2025-02-09 06:52:35.65347388 +0000 UTC m=+16.632282681 (1 nodes), 951ns from start of poll, 161.559332ms from cluster start. Timeout at 2025-02-09 06:52:35.758820084 +0000 UTC m=+16.737628844, 105.346163ms after stability -testLoggerAdapter verbose: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.758Z [INFO] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: updating configuration: command=AddStaging server-id=server-bdfe75b5-1662-6cf4-9646-fa7a9f725c92 server-addr=bdfe75b5-1662-6cf4-9646-fa7a9f725c92 servers="[{Suffrage:Voter ID:0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d Address:0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d} {Suffrage:Voter ID:2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1 Address:2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1} {Suffrage:Voter ID:server-bdfe75b5-1662-6cf4-9646-fa7a9f725c92 Address:bdfe75b5-1662-6cf4-9646-fa7a9f725c92}]" - testing.go:149: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.758Z [INFO] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: updating configuration: command=AddStaging server-id=server-bdfe75b5-1662-6cf4-9646-fa7a9f725c92 server-addr=bdfe75b5-1662-6cf4-9646-fa7a9f725c92 servers="[{Suffrage:Voter ID:0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d Address:0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d} {Suffrage:Voter ID:2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1 Address:2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1} {Suffrage:Voter ID:server-bdfe75b5-1662-6cf4-9646-fa7a9f725c92 Address:bdfe75b5-1662-6cf4-9646-fa7a9f725c92}]" -testLoggerAdapter verbose: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.758Z [INFO] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: added peer, starting replication: peer=server-bdfe75b5-1662-6cf4-9646-fa7a9f725c92 - testing.go:149: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.758Z [INFO] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: added peer, starting replication: peer=server-bdfe75b5-1662-6cf4-9646-fa7a9f725c92 -testLoggerAdapter verbose: server-bdfe75b5-1662-6cf4-9646-fa7a9f725c92: 2025-02-09T06:52:35.759Z [WARN] server-bdfe75b5-1662-6cf4-9646-fa7a9f725c92: failed to get previous log: previous-index=3 last-index=0 error="log not found" - testing.go:149: server-bdfe75b5-1662-6cf4-9646-fa7a9f725c92: 2025-02-09T06:52:35.759Z [WARN] server-bdfe75b5-1662-6cf4-9646-fa7a9f725c92: failed to get previous log: previous-index=3 last-index=0 error="log not found" -testLoggerAdapter verbose: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.759Z [WARN] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: appendEntries rejected, sending older logs: peer="{Voter server-bdfe75b5-1662-6cf4-9646-fa7a9f725c92 bdfe75b5-1662-6cf4-9646-fa7a9f725c92}" next=1 - testing.go:149: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.759Z [WARN] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: appendEntries rejected, sending older logs: peer="{Voter server-bdfe75b5-1662-6cf4-9646-fa7a9f725c92 bdfe75b5-1662-6cf4-9646-fa7a9f725c92}" next=1 -testLoggerAdapter verbose: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.759Z [INFO] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: pipelining replication: peer="{Voter server-bdfe75b5-1662-6cf4-9646-fa7a9f725c92 bdfe75b5-1662-6cf4-9646-fa7a9f725c92}" - testing.go:149: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.759Z [INFO] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: pipelining replication: peer="{Voter server-bdfe75b5-1662-6cf4-9646-fa7a9f725c92 bdfe75b5-1662-6cf4-9646-fa7a9f725c92}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:35.764Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:35.764Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:35.869Z [INFO] stable state for Leader reached at 2025-02-09 06:52:35.764172007 +0000 UTC m=+16.742980774 (1 nodes), 509ns from start of poll, 272.257425ms from cluster start. Timeout at 2025-02-09 06:52:35.869249677 +0000 UTC m=+16.848058435, 105.077661ms after stability - testing.go:149: cluster: 2025-02-09T06:52:35.869Z [INFO] stable state for Leader reached at 2025-02-09 06:52:35.764172007 +0000 UTC m=+16.742980774 (1 nodes), 509ns from start of poll, 272.257425ms from cluster start. Timeout at 2025-02-09 06:52:35.869249677 +0000 UTC m=+16.848058435, 105.077661ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:35.869Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:35.869Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:35.974Z [INFO] stable state for Leader reached at 2025-02-09 06:52:35.869317261 +0000 UTC m=+16.848126026 (1 nodes), 463ns from start of poll, 377.402677ms from cluster start. Timeout at 2025-02-09 06:52:35.974319964 +0000 UTC m=+16.953128728, 105.002702ms after stability - testing.go:149: cluster: 2025-02-09T06:52:35.974Z [INFO] stable state for Leader reached at 2025-02-09 06:52:35.869317261 +0000 UTC m=+16.848126026 (1 nodes), 463ns from start of poll, 377.402677ms from cluster start. Timeout at 2025-02-09 06:52:35.974319964 +0000 UTC m=+16.953128728, 105.002702ms after stability -testLoggerAdapter verbose: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.974Z [INFO] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: updating configuration: command=RemoveServer server-id=0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d server-addr= servers="[{Suffrage:Voter ID:2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1 Address:2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1} {Suffrage:Voter ID:server-bdfe75b5-1662-6cf4-9646-fa7a9f725c92 Address:bdfe75b5-1662-6cf4-9646-fa7a9f725c92}]" - testing.go:149: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.974Z [INFO] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: updating configuration: command=RemoveServer server-id=0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d server-addr= servers="[{Suffrage:Voter ID:2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1 Address:2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1} {Suffrage:Voter ID:server-bdfe75b5-1662-6cf4-9646-fa7a9f725c92 Address:bdfe75b5-1662-6cf4-9646-fa7a9f725c92}]" -testLoggerAdapter verbose: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.974Z [INFO] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: removed peer, stopping replication: peer=0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d last-index=4 - testing.go:149: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.974Z [INFO] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: removed peer, stopping replication: peer=0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d last-index=4 -testLoggerAdapter verbose: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.974Z [INFO] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: aborting pipeline replication: peer="{Voter 0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d 0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d}" - testing.go:149: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.974Z [INFO] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: aborting pipeline replication: peer="{Voter 0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d 0b7d4ad4-ad8d-76a5-bf5d-c4d610f9b45d}" -testLoggerAdapter verbose: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.974Z [INFO] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: aborting pipeline replication: peer="{Voter server-bdfe75b5-1662-6cf4-9646-fa7a9f725c92 bdfe75b5-1662-6cf4-9646-fa7a9f725c92}" - testing.go:149: 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: 2025-02-09T06:52:35.974Z [INFO] 2a5e0774-93ab-3cc3-b4a3-b5f8da3f4cd1: aborting pipeline replication: peer="{Voter server-bdfe75b5-1662-6cf4-9646-fa7a9f725c92 bdfe75b5-1662-6cf4-9646-fa7a9f725c92}" ---- PASS: TestRaft_ProtocolVersion_Upgrade_2_3 (0.48s) +testLoggerAdapter verbose: 783def06-61e2-6284-e103-d42777c4e599: 2024-01-08T00:31:10.929Z [INFO] 783def06-61e2-6284-e103-d42777c4e599: initial configuration: index=1 servers="[{Suffrage:Voter ID:783def06-61e2-6284-e103-d42777c4e599 Address:783def06-61e2-6284-e103-d42777c4e599} {Suffrage:Voter ID:3a19eeea-b34d-9c5b-75ab-9592d167d11d Address:3a19eeea-b34d-9c5b-75ab-9592d167d11d}]" + testing.go:149: 783def06-61e2-6284-e103-d42777c4e599: 2024-01-08T00:31:10.929Z [INFO] 783def06-61e2-6284-e103-d42777c4e599: initial configuration: index=1 servers="[{Suffrage:Voter ID:783def06-61e2-6284-e103-d42777c4e599 Address:783def06-61e2-6284-e103-d42777c4e599} {Suffrage:Voter ID:3a19eeea-b34d-9c5b-75ab-9592d167d11d Address:3a19eeea-b34d-9c5b-75ab-9592d167d11d}]" +testLoggerAdapter verbose: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:10.929Z [INFO] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: initial configuration: index=1 servers="[{Suffrage:Voter ID:783def06-61e2-6284-e103-d42777c4e599 Address:783def06-61e2-6284-e103-d42777c4e599} {Suffrage:Voter ID:3a19eeea-b34d-9c5b-75ab-9592d167d11d Address:3a19eeea-b34d-9c5b-75ab-9592d167d11d}]" + testing.go:149: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:10.929Z [INFO] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: initial configuration: index=1 servers="[{Suffrage:Voter ID:783def06-61e2-6284-e103-d42777c4e599 Address:783def06-61e2-6284-e103-d42777c4e599} {Suffrage:Voter ID:3a19eeea-b34d-9c5b-75ab-9592d167d11d Address:3a19eeea-b34d-9c5b-75ab-9592d167d11d}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:10.929Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T00:31:10.929Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: 783def06-61e2-6284-e103-d42777c4e599: 2024-01-08T00:31:10.929Z [INFO] 783def06-61e2-6284-e103-d42777c4e599: entering follower state: follower="Node at 783def06-61e2-6284-e103-d42777c4e599 [Follower]" leader= + testing.go:149: 783def06-61e2-6284-e103-d42777c4e599: 2024-01-08T00:31:10.929Z [INFO] 783def06-61e2-6284-e103-d42777c4e599: entering follower state: follower="Node at 783def06-61e2-6284-e103-d42777c4e599 [Follower]" leader= +testLoggerAdapter verbose: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:10.929Z [INFO] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: entering follower state: follower="Node at 3a19eeea-b34d-9c5b-75ab-9592d167d11d [Follower]" leader= + testing.go:149: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:10.929Z [INFO] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: entering follower state: follower="Node at 3a19eeea-b34d-9c5b-75ab-9592d167d11d [Follower]" leader= +testLoggerAdapter verbose: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:10.991Z [WARN] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: heartbeat timeout reached, starting election: last-leader= + testing.go:149: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:10.991Z [WARN] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:10.991Z [INFO] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: entering candidate state: node="Node at 3a19eeea-b34d-9c5b-75ab-9592d167d11d [Candidate]" term=2 + testing.go:149: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:10.991Z [INFO] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: entering candidate state: node="Node at 3a19eeea-b34d-9c5b-75ab-9592d167d11d [Candidate]" term=2 +testLoggerAdapter verbose: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:10.991Z [INFO] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: election won: tally=2 + testing.go:149: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:10.991Z [INFO] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: election won: tally=2 +testLoggerAdapter verbose: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:10.991Z [INFO] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: entering leader state: leader="Node at 3a19eeea-b34d-9c5b-75ab-9592d167d11d [Leader]" + testing.go:149: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:10.991Z [INFO] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: entering leader state: leader="Node at 3a19eeea-b34d-9c5b-75ab-9592d167d11d [Leader]" +testLoggerAdapter verbose: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:10.991Z [INFO] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: added peer, starting replication: peer=783def06-61e2-6284-e103-d42777c4e599 + testing.go:149: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:10.991Z [INFO] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: added peer, starting replication: peer=783def06-61e2-6284-e103-d42777c4e599 +testLoggerAdapter verbose: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:10.991Z [INFO] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: pipelining replication: peer="{Voter 783def06-61e2-6284-e103-d42777c4e599 783def06-61e2-6284-e103-d42777c4e599}" + testing.go:149: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:10.991Z [INFO] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: pipelining replication: peer="{Voter 783def06-61e2-6284-e103-d42777c4e599 783def06-61e2-6284-e103-d42777c4e599}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:11.097Z [INFO] stable state for Follower reached at 2024-01-08 00:31:10.991966166 +0000 UTC m=+17.038855307 (1 nodes), 62.563749ms from start of poll, 62.783608ms from cluster start. Timeout at 2024-01-08 00:31:11.097106562 +0000 UTC m=+17.143995710, 105.140403ms after stability + testing.go:149: cluster: 2024-01-08T00:31:11.097Z [INFO] stable state for Follower reached at 2024-01-08 00:31:10.991966166 +0000 UTC m=+17.038855307 (1 nodes), 62.563749ms from start of poll, 62.783608ms from cluster start. Timeout at 2024-01-08 00:31:11.097106562 +0000 UTC m=+17.143995710, 105.140403ms after stability +testLoggerAdapter verbose: server-36a8cd05-eb25-d102-ba2d-fb1cdd102525: 2024-01-08T00:31:11.097Z [INFO] server-36a8cd05-eb25-d102-ba2d-fb1cdd102525: initial configuration: index=0 servers=[] + testing.go:149: server-36a8cd05-eb25-d102-ba2d-fb1cdd102525: 2024-01-08T00:31:11.097Z [INFO] server-36a8cd05-eb25-d102-ba2d-fb1cdd102525: initial configuration: index=0 servers=[] +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:11.097Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:11.097Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-36a8cd05-eb25-d102-ba2d-fb1cdd102525: 2024-01-08T00:31:11.097Z [INFO] server-36a8cd05-eb25-d102-ba2d-fb1cdd102525: entering follower state: follower="Node at 36a8cd05-eb25-d102-ba2d-fb1cdd102525 [Follower]" leader= + testing.go:149: server-36a8cd05-eb25-d102-ba2d-fb1cdd102525: 2024-01-08T00:31:11.097Z [INFO] server-36a8cd05-eb25-d102-ba2d-fb1cdd102525: entering follower state: follower="Node at 36a8cd05-eb25-d102-ba2d-fb1cdd102525 [Follower]" leader= +testLoggerAdapter verbose: server-36a8cd05-eb25-d102-ba2d-fb1cdd102525: 2024-01-08T00:31:11.196Z [WARN] server-36a8cd05-eb25-d102-ba2d-fb1cdd102525: no known peers, aborting election + testing.go:149: server-36a8cd05-eb25-d102-ba2d-fb1cdd102525: 2024-01-08T00:31:11.196Z [WARN] server-36a8cd05-eb25-d102-ba2d-fb1cdd102525: no known peers, aborting election +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:11.203Z [INFO] stable state for Leader reached at 2024-01-08 00:31:11.097730621 +0000 UTC m=+17.144619758 (1 nodes), 2.014µs from start of poll, 168.548059ms from cluster start. Timeout at 2024-01-08 00:31:11.203093211 +0000 UTC m=+17.249982423, 105.362665ms after stability + testing.go:149: cluster: 2024-01-08T00:31:11.203Z [INFO] stable state for Leader reached at 2024-01-08 00:31:11.097730621 +0000 UTC m=+17.144619758 (1 nodes), 2.014µs from start of poll, 168.548059ms from cluster start. Timeout at 2024-01-08 00:31:11.203093211 +0000 UTC m=+17.249982423, 105.362665ms after stability +testLoggerAdapter verbose: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:11.203Z [INFO] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: updating configuration: command=AddStaging server-id=server-36a8cd05-eb25-d102-ba2d-fb1cdd102525 server-addr=36a8cd05-eb25-d102-ba2d-fb1cdd102525 servers="[{Suffrage:Voter ID:783def06-61e2-6284-e103-d42777c4e599 Address:783def06-61e2-6284-e103-d42777c4e599} {Suffrage:Voter ID:3a19eeea-b34d-9c5b-75ab-9592d167d11d Address:3a19eeea-b34d-9c5b-75ab-9592d167d11d} {Suffrage:Voter ID:server-36a8cd05-eb25-d102-ba2d-fb1cdd102525 Address:36a8cd05-eb25-d102-ba2d-fb1cdd102525}]" + testing.go:149: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:11.203Z [INFO] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: updating configuration: command=AddStaging server-id=server-36a8cd05-eb25-d102-ba2d-fb1cdd102525 server-addr=36a8cd05-eb25-d102-ba2d-fb1cdd102525 servers="[{Suffrage:Voter ID:783def06-61e2-6284-e103-d42777c4e599 Address:783def06-61e2-6284-e103-d42777c4e599} {Suffrage:Voter ID:3a19eeea-b34d-9c5b-75ab-9592d167d11d Address:3a19eeea-b34d-9c5b-75ab-9592d167d11d} {Suffrage:Voter ID:server-36a8cd05-eb25-d102-ba2d-fb1cdd102525 Address:36a8cd05-eb25-d102-ba2d-fb1cdd102525}]" +testLoggerAdapter verbose: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:11.203Z [INFO] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: added peer, starting replication: peer=server-36a8cd05-eb25-d102-ba2d-fb1cdd102525 + testing.go:149: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:11.203Z [INFO] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: added peer, starting replication: peer=server-36a8cd05-eb25-d102-ba2d-fb1cdd102525 +testLoggerAdapter verbose: server-36a8cd05-eb25-d102-ba2d-fb1cdd102525: 2024-01-08T00:31:11.203Z [WARN] server-36a8cd05-eb25-d102-ba2d-fb1cdd102525: failed to get previous log: previous-index=3 last-index=0 error="log not found" + testing.go:149: server-36a8cd05-eb25-d102-ba2d-fb1cdd102525: 2024-01-08T00:31:11.203Z [WARN] server-36a8cd05-eb25-d102-ba2d-fb1cdd102525: failed to get previous log: previous-index=3 last-index=0 error="log not found" +testLoggerAdapter verbose: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:11.203Z [WARN] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: appendEntries rejected, sending older logs: peer="{Voter server-36a8cd05-eb25-d102-ba2d-fb1cdd102525 36a8cd05-eb25-d102-ba2d-fb1cdd102525}" next=1 + testing.go:149: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:11.203Z [WARN] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: appendEntries rejected, sending older logs: peer="{Voter server-36a8cd05-eb25-d102-ba2d-fb1cdd102525 36a8cd05-eb25-d102-ba2d-fb1cdd102525}" next=1 +testLoggerAdapter verbose: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:11.203Z [INFO] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: pipelining replication: peer="{Voter server-36a8cd05-eb25-d102-ba2d-fb1cdd102525 36a8cd05-eb25-d102-ba2d-fb1cdd102525}" + testing.go:149: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:11.203Z [INFO] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: pipelining replication: peer="{Voter server-36a8cd05-eb25-d102-ba2d-fb1cdd102525 36a8cd05-eb25-d102-ba2d-fb1cdd102525}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:11.208Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:11.208Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:11.314Z [INFO] stable state for Leader reached at 2024-01-08 00:31:11.208895789 +0000 UTC m=+17.255784943 (1 nodes), 2.474µs from start of poll, 279.713244ms from cluster start. Timeout at 2024-01-08 00:31:11.314468917 +0000 UTC m=+17.361358107, 105.573164ms after stability + testing.go:149: cluster: 2024-01-08T00:31:11.314Z [INFO] stable state for Leader reached at 2024-01-08 00:31:11.208895789 +0000 UTC m=+17.255784943 (1 nodes), 2.474µs from start of poll, 279.713244ms from cluster start. Timeout at 2024-01-08 00:31:11.314468917 +0000 UTC m=+17.361358107, 105.573164ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:11.314Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:11.314Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:11.420Z [INFO] stable state for Leader reached at 2024-01-08 00:31:11.314641551 +0000 UTC m=+17.361530644 (1 nodes), 2.044µs from start of poll, 385.458945ms from cluster start. Timeout at 2024-01-08 00:31:11.42021097 +0000 UTC m=+17.467100141, 105.569497ms after stability + testing.go:149: cluster: 2024-01-08T00:31:11.420Z [INFO] stable state for Leader reached at 2024-01-08 00:31:11.314641551 +0000 UTC m=+17.361530644 (1 nodes), 2.044µs from start of poll, 385.458945ms from cluster start. Timeout at 2024-01-08 00:31:11.42021097 +0000 UTC m=+17.467100141, 105.569497ms after stability +testLoggerAdapter verbose: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:11.420Z [INFO] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: updating configuration: command=RemoveServer server-id=783def06-61e2-6284-e103-d42777c4e599 server-addr= servers="[{Suffrage:Voter ID:3a19eeea-b34d-9c5b-75ab-9592d167d11d Address:3a19eeea-b34d-9c5b-75ab-9592d167d11d} {Suffrage:Voter ID:server-36a8cd05-eb25-d102-ba2d-fb1cdd102525 Address:36a8cd05-eb25-d102-ba2d-fb1cdd102525}]" + testing.go:149: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:11.420Z [INFO] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: updating configuration: command=RemoveServer server-id=783def06-61e2-6284-e103-d42777c4e599 server-addr= servers="[{Suffrage:Voter ID:3a19eeea-b34d-9c5b-75ab-9592d167d11d Address:3a19eeea-b34d-9c5b-75ab-9592d167d11d} {Suffrage:Voter ID:server-36a8cd05-eb25-d102-ba2d-fb1cdd102525 Address:36a8cd05-eb25-d102-ba2d-fb1cdd102525}]" +testLoggerAdapter verbose: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:11.420Z [INFO] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: removed peer, stopping replication: peer=783def06-61e2-6284-e103-d42777c4e599 last-index=4 + testing.go:149: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:11.420Z [INFO] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: removed peer, stopping replication: peer=783def06-61e2-6284-e103-d42777c4e599 last-index=4 +testLoggerAdapter verbose: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:11.420Z [INFO] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: aborting pipeline replication: peer="{Voter 783def06-61e2-6284-e103-d42777c4e599 783def06-61e2-6284-e103-d42777c4e599}" + testing.go:149: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:11.420Z [INFO] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: aborting pipeline replication: peer="{Voter 783def06-61e2-6284-e103-d42777c4e599 783def06-61e2-6284-e103-d42777c4e599}" +testLoggerAdapter verbose: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:11.420Z [INFO] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: aborting pipeline replication: peer="{Voter server-36a8cd05-eb25-d102-ba2d-fb1cdd102525 36a8cd05-eb25-d102-ba2d-fb1cdd102525}" + testing.go:149: 3a19eeea-b34d-9c5b-75ab-9592d167d11d: 2024-01-08T00:31:11.420Z [INFO] 3a19eeea-b34d-9c5b-75ab-9592d167d11d: aborting pipeline replication: peer="{Voter server-36a8cd05-eb25-d102-ba2d-fb1cdd102525 36a8cd05-eb25-d102-ba2d-fb1cdd102525}" +--- PASS: TestRaft_ProtocolVersion_Upgrade_2_3 (0.49s) === RUN TestRaft_LeadershipTransferInProgress --- PASS: TestRaft_LeadershipTransferInProgress (0.00s) === RUN TestRaft_LeadershipTransferPickServer --- PASS: TestRaft_LeadershipTransferPickServer (0.00s) === RUN TestRaft_LeadershipTransfer -testLoggerAdapter verbose: server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: 2025-02-09T06:52:35.975Z [INFO] server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2 Address:2903c9a1-c302-a0a9-75da-aba72d8c7ea2} {Suffrage:Voter ID:server-742753e4-5df7-c5ee-04d2-b8e401f8c46f Address:742753e4-5df7-c5ee-04d2-b8e401f8c46f} {Suffrage:Voter ID:server-7abad551-1625-795a-768a-2502ef371445 Address:7abad551-1625-795a-768a-2502ef371445}]" - testing.go:149: server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: 2025-02-09T06:52:35.975Z [INFO] server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2 Address:2903c9a1-c302-a0a9-75da-aba72d8c7ea2} {Suffrage:Voter ID:server-742753e4-5df7-c5ee-04d2-b8e401f8c46f Address:742753e4-5df7-c5ee-04d2-b8e401f8c46f} {Suffrage:Voter ID:server-7abad551-1625-795a-768a-2502ef371445 Address:7abad551-1625-795a-768a-2502ef371445}]" -testLoggerAdapter verbose: server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: 2025-02-09T06:52:35.975Z [INFO] server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: entering follower state: follower="Node at 2903c9a1-c302-a0a9-75da-aba72d8c7ea2 [Follower]" leader= - testing.go:149: server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: 2025-02-09T06:52:35.975Z [INFO] server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: entering follower state: follower="Node at 2903c9a1-c302-a0a9-75da-aba72d8c7ea2 [Follower]" leader= -testLoggerAdapter verbose: server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: 2025-02-09T06:52:35.975Z [INFO] server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2 Address:2903c9a1-c302-a0a9-75da-aba72d8c7ea2} {Suffrage:Voter ID:server-742753e4-5df7-c5ee-04d2-b8e401f8c46f Address:742753e4-5df7-c5ee-04d2-b8e401f8c46f} {Suffrage:Voter ID:server-7abad551-1625-795a-768a-2502ef371445 Address:7abad551-1625-795a-768a-2502ef371445}]" - testing.go:149: server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: 2025-02-09T06:52:35.975Z [INFO] server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2 Address:2903c9a1-c302-a0a9-75da-aba72d8c7ea2} {Suffrage:Voter ID:server-742753e4-5df7-c5ee-04d2-b8e401f8c46f Address:742753e4-5df7-c5ee-04d2-b8e401f8c46f} {Suffrage:Voter ID:server-7abad551-1625-795a-768a-2502ef371445 Address:7abad551-1625-795a-768a-2502ef371445}]" -testLoggerAdapter verbose: server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: 2025-02-09T06:52:35.975Z [INFO] server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: entering follower state: follower="Node at 742753e4-5df7-c5ee-04d2-b8e401f8c46f [Follower]" leader= - testing.go:149: server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: 2025-02-09T06:52:35.975Z [INFO] server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: entering follower state: follower="Node at 742753e4-5df7-c5ee-04d2-b8e401f8c46f [Follower]" leader= -testLoggerAdapter verbose: server-7abad551-1625-795a-768a-2502ef371445: 2025-02-09T06:52:35.975Z [INFO] server-7abad551-1625-795a-768a-2502ef371445: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2 Address:2903c9a1-c302-a0a9-75da-aba72d8c7ea2} {Suffrage:Voter ID:server-742753e4-5df7-c5ee-04d2-b8e401f8c46f Address:742753e4-5df7-c5ee-04d2-b8e401f8c46f} {Suffrage:Voter ID:server-7abad551-1625-795a-768a-2502ef371445 Address:7abad551-1625-795a-768a-2502ef371445}]" - testing.go:149: server-7abad551-1625-795a-768a-2502ef371445: 2025-02-09T06:52:35.975Z [INFO] server-7abad551-1625-795a-768a-2502ef371445: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2 Address:2903c9a1-c302-a0a9-75da-aba72d8c7ea2} {Suffrage:Voter ID:server-742753e4-5df7-c5ee-04d2-b8e401f8c46f Address:742753e4-5df7-c5ee-04d2-b8e401f8c46f} {Suffrage:Voter ID:server-7abad551-1625-795a-768a-2502ef371445 Address:7abad551-1625-795a-768a-2502ef371445}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:35.975Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:35.975Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-7abad551-1625-795a-768a-2502ef371445: 2025-02-09T06:52:35.975Z [INFO] server-7abad551-1625-795a-768a-2502ef371445: entering follower state: follower="Node at 7abad551-1625-795a-768a-2502ef371445 [Follower]" leader= - testing.go:149: server-7abad551-1625-795a-768a-2502ef371445: 2025-02-09T06:52:35.975Z [INFO] server-7abad551-1625-795a-768a-2502ef371445: entering follower state: follower="Node at 7abad551-1625-795a-768a-2502ef371445 [Follower]" leader= -testLoggerAdapter verbose: server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: 2025-02-09T06:52:36.040Z [WARN] server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: 2025-02-09T06:52:36.040Z [WARN] server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: 2025-02-09T06:52:36.040Z [INFO] server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: entering candidate state: node="Node at 742753e4-5df7-c5ee-04d2-b8e401f8c46f [Candidate]" term=2 - testing.go:149: server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: 2025-02-09T06:52:36.040Z [INFO] server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: entering candidate state: node="Node at 742753e4-5df7-c5ee-04d2-b8e401f8c46f [Candidate]" term=2 -testLoggerAdapter verbose: server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: 2025-02-09T06:52:36.040Z [INFO] server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: election won: tally=2 - testing.go:149: server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: 2025-02-09T06:52:36.040Z [INFO] server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: election won: tally=2 -testLoggerAdapter verbose: server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: 2025-02-09T06:52:36.040Z [INFO] server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: entering leader state: leader="Node at 742753e4-5df7-c5ee-04d2-b8e401f8c46f [Leader]" - testing.go:149: server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: 2025-02-09T06:52:36.040Z [INFO] server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: entering leader state: leader="Node at 742753e4-5df7-c5ee-04d2-b8e401f8c46f [Leader]" -testLoggerAdapter verbose: server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: 2025-02-09T06:52:36.040Z [INFO] server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: added peer, starting replication: peer=server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2 - testing.go:149: server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: 2025-02-09T06:52:36.040Z [INFO] server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: added peer, starting replication: peer=server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2 -testLoggerAdapter verbose: server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: 2025-02-09T06:52:36.040Z [INFO] server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: added peer, starting replication: peer=server-7abad551-1625-795a-768a-2502ef371445 - testing.go:149: server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: 2025-02-09T06:52:36.040Z [INFO] server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: added peer, starting replication: peer=server-7abad551-1625-795a-768a-2502ef371445 -testLoggerAdapter verbose: server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: 2025-02-09T06:52:36.040Z [INFO] server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: pipelining replication: peer="{Voter server-7abad551-1625-795a-768a-2502ef371445 7abad551-1625-795a-768a-2502ef371445}" - testing.go:149: server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: 2025-02-09T06:52:36.040Z [INFO] server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: pipelining replication: peer="{Voter server-7abad551-1625-795a-768a-2502ef371445 7abad551-1625-795a-768a-2502ef371445}" -testLoggerAdapter verbose: server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: 2025-02-09T06:52:36.040Z [INFO] server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: pipelining replication: peer="{Voter server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2 2903c9a1-c302-a0a9-75da-aba72d8c7ea2}" - testing.go:149: server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: 2025-02-09T06:52:36.040Z [INFO] server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: pipelining replication: peer="{Voter server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2 2903c9a1-c302-a0a9-75da-aba72d8c7ea2}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:36.146Z [INFO] stable state for Leader reached at 2025-02-09 06:52:36.040531437 +0000 UTC m=+17.019340202 (1 nodes), 64.752728ms from start of poll, 65.082035ms from cluster start. Timeout at 2025-02-09 06:52:36.146242084 +0000 UTC m=+17.125050843, 105.710641ms after stability - testing.go:149: cluster: 2025-02-09T06:52:36.146Z [INFO] stable state for Leader reached at 2025-02-09 06:52:36.040531437 +0000 UTC m=+17.019340202 (1 nodes), 64.752728ms from start of poll, 65.082035ms from cluster start. Timeout at 2025-02-09 06:52:36.146242084 +0000 UTC m=+17.125050843, 105.710641ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:36.146Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:36.146Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:36.251Z [INFO] stable state for Leader reached at 2025-02-09 06:52:36.146308148 +0000 UTC m=+17.125116913 (1 nodes), 481ns from start of poll, 170.858746ms from cluster start. Timeout at 2025-02-09 06:52:36.251499544 +0000 UTC m=+17.230308313, 105.1914ms after stability - testing.go:149: cluster: 2025-02-09T06:52:36.251Z [INFO] stable state for Leader reached at 2025-02-09 06:52:36.146308148 +0000 UTC m=+17.125116913 (1 nodes), 481ns from start of poll, 170.858746ms from cluster start. Timeout at 2025-02-09 06:52:36.251499544 +0000 UTC m=+17.230308313, 105.1914ms after stability -testLoggerAdapter verbose: server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: 2025-02-09T06:52:36.251Z [INFO] server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: entering candidate state: node="Node at 2903c9a1-c302-a0a9-75da-aba72d8c7ea2 [Candidate]" term=3 - testing.go:149: server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: 2025-02-09T06:52:36.251Z [INFO] server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: entering candidate state: node="Node at 2903c9a1-c302-a0a9-75da-aba72d8c7ea2 [Candidate]" term=3 -testLoggerAdapter verbose: server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: 2025-02-09T06:52:36.251Z [INFO] server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: election won: tally=2 - testing.go:149: server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: 2025-02-09T06:52:36.251Z [INFO] server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: election won: tally=2 -testLoggerAdapter verbose: server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: 2025-02-09T06:52:36.251Z [INFO] server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: entering leader state: leader="Node at 2903c9a1-c302-a0a9-75da-aba72d8c7ea2 [Leader]" - testing.go:149: server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: 2025-02-09T06:52:36.251Z [INFO] server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: entering leader state: leader="Node at 2903c9a1-c302-a0a9-75da-aba72d8c7ea2 [Leader]" -testLoggerAdapter verbose: server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: 2025-02-09T06:52:36.251Z [INFO] server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: added peer, starting replication: peer=server-742753e4-5df7-c5ee-04d2-b8e401f8c46f - testing.go:149: server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: 2025-02-09T06:52:36.251Z [INFO] server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: added peer, starting replication: peer=server-742753e4-5df7-c5ee-04d2-b8e401f8c46f -testLoggerAdapter verbose: server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: 2025-02-09T06:52:36.251Z [INFO] server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: added peer, starting replication: peer=server-7abad551-1625-795a-768a-2502ef371445 - testing.go:149: server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: 2025-02-09T06:52:36.251Z [INFO] server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: added peer, starting replication: peer=server-7abad551-1625-795a-768a-2502ef371445 -testLoggerAdapter verbose: server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: 2025-02-09T06:52:36.251Z [INFO] server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: entering follower state: follower="Node at 742753e4-5df7-c5ee-04d2-b8e401f8c46f [Follower]" leader= -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:36.251Z [INFO] starting stability test: raft-state=Leader - testing.go:149: server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: 2025-02-09T06:52:36.251Z [INFO] server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: entering follower state: follower="Node at 742753e4-5df7-c5ee-04d2-b8e401f8c46f [Follower]" leader= -testLoggerAdapter verbose: server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: 2025-02-09T06:52:36.251Z [INFO] server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: pipelining replication: peer="{Voter server-7abad551-1625-795a-768a-2502ef371445 7abad551-1625-795a-768a-2502ef371445}" - testing.go:149: cluster: 2025-02-09T06:52:36.251Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: 2025-02-09T06:52:36.251Z [INFO] server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: aborting pipeline replication: peer="{Voter server-7abad551-1625-795a-768a-2502ef371445 7abad551-1625-795a-768a-2502ef371445}" - testing.go:149: server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: 2025-02-09T06:52:36.251Z [INFO] server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: pipelining replication: peer="{Voter server-7abad551-1625-795a-768a-2502ef371445 7abad551-1625-795a-768a-2502ef371445}" - testing.go:149: server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: 2025-02-09T06:52:36.251Z [INFO] server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: aborting pipeline replication: peer="{Voter server-7abad551-1625-795a-768a-2502ef371445 7abad551-1625-795a-768a-2502ef371445}" -testLoggerAdapter verbose: server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: 2025-02-09T06:52:36.251Z [INFO] server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: pipelining replication: peer="{Voter server-742753e4-5df7-c5ee-04d2-b8e401f8c46f 742753e4-5df7-c5ee-04d2-b8e401f8c46f}" -testLoggerAdapter verbose: server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: 2025-02-09T06:52:36.251Z [INFO] server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: aborting pipeline replication: peer="{Voter server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2 2903c9a1-c302-a0a9-75da-aba72d8c7ea2}" - testing.go:149: server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: 2025-02-09T06:52:36.251Z [INFO] server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: pipelining replication: peer="{Voter server-742753e4-5df7-c5ee-04d2-b8e401f8c46f 742753e4-5df7-c5ee-04d2-b8e401f8c46f}" - testing.go:149: server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: 2025-02-09T06:52:36.251Z [INFO] server-742753e4-5df7-c5ee-04d2-b8e401f8c46f: aborting pipeline replication: peer="{Voter server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2 2903c9a1-c302-a0a9-75da-aba72d8c7ea2}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:36.357Z [INFO] stable state for Leader reached at 2025-02-09 06:52:36.251784547 +0000 UTC m=+17.230593320 (1 nodes), 11.096µs from start of poll, 276.335153ms from cluster start. Timeout at 2025-02-09 06:52:36.357094604 +0000 UTC m=+17.335903372, 105.310052ms after stability - testing.go:149: cluster: 2025-02-09T06:52:36.357Z [INFO] stable state for Leader reached at 2025-02-09 06:52:36.251784547 +0000 UTC m=+17.230593320 (1 nodes), 11.096µs from start of poll, 276.335153ms from cluster start. Timeout at 2025-02-09 06:52:36.357094604 +0000 UTC m=+17.335903372, 105.310052ms after stability -testLoggerAdapter verbose: server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: 2025-02-09T06:52:36.357Z [INFO] server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: aborting pipeline replication: peer="{Voter server-742753e4-5df7-c5ee-04d2-b8e401f8c46f 742753e4-5df7-c5ee-04d2-b8e401f8c46f}" - testing.go:149: server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: 2025-02-09T06:52:36.357Z [INFO] server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: aborting pipeline replication: peer="{Voter server-742753e4-5df7-c5ee-04d2-b8e401f8c46f 742753e4-5df7-c5ee-04d2-b8e401f8c46f}" -testLoggerAdapter verbose: server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: 2025-02-09T06:52:36.357Z [INFO] server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: aborting pipeline replication: peer="{Voter server-7abad551-1625-795a-768a-2502ef371445 7abad551-1625-795a-768a-2502ef371445}" - testing.go:149: server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: 2025-02-09T06:52:36.357Z [INFO] server-2903c9a1-c302-a0a9-75da-aba72d8c7ea2: aborting pipeline replication: peer="{Voter server-7abad551-1625-795a-768a-2502ef371445 7abad551-1625-795a-768a-2502ef371445}" ---- PASS: TestRaft_LeadershipTransfer (0.38s) +testLoggerAdapter verbose: server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: 2024-01-08T00:31:11.422Z [INFO] server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3dbabc23-a375-d2ba-b0f8-902fa23b8958 Address:3dbabc23-a375-d2ba-b0f8-902fa23b8958} {Suffrage:Voter ID:server-031f68ac-1e78-0df1-54e4-4f634840d4d0 Address:031f68ac-1e78-0df1-54e4-4f634840d4d0} {Suffrage:Voter ID:server-52bb4c20-c73a-5f29-2c36-f49961f116a9 Address:52bb4c20-c73a-5f29-2c36-f49961f116a9}]" + testing.go:149: server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: 2024-01-08T00:31:11.422Z [INFO] server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3dbabc23-a375-d2ba-b0f8-902fa23b8958 Address:3dbabc23-a375-d2ba-b0f8-902fa23b8958} {Suffrage:Voter ID:server-031f68ac-1e78-0df1-54e4-4f634840d4d0 Address:031f68ac-1e78-0df1-54e4-4f634840d4d0} {Suffrage:Voter ID:server-52bb4c20-c73a-5f29-2c36-f49961f116a9 Address:52bb4c20-c73a-5f29-2c36-f49961f116a9}]" +testLoggerAdapter verbose: server-031f68ac-1e78-0df1-54e4-4f634840d4d0: 2024-01-08T00:31:11.422Z [INFO] server-031f68ac-1e78-0df1-54e4-4f634840d4d0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3dbabc23-a375-d2ba-b0f8-902fa23b8958 Address:3dbabc23-a375-d2ba-b0f8-902fa23b8958} {Suffrage:Voter ID:server-031f68ac-1e78-0df1-54e4-4f634840d4d0 Address:031f68ac-1e78-0df1-54e4-4f634840d4d0} {Suffrage:Voter ID:server-52bb4c20-c73a-5f29-2c36-f49961f116a9 Address:52bb4c20-c73a-5f29-2c36-f49961f116a9}]" + testing.go:149: server-031f68ac-1e78-0df1-54e4-4f634840d4d0: 2024-01-08T00:31:11.422Z [INFO] server-031f68ac-1e78-0df1-54e4-4f634840d4d0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3dbabc23-a375-d2ba-b0f8-902fa23b8958 Address:3dbabc23-a375-d2ba-b0f8-902fa23b8958} {Suffrage:Voter ID:server-031f68ac-1e78-0df1-54e4-4f634840d4d0 Address:031f68ac-1e78-0df1-54e4-4f634840d4d0} {Suffrage:Voter ID:server-52bb4c20-c73a-5f29-2c36-f49961f116a9 Address:52bb4c20-c73a-5f29-2c36-f49961f116a9}]" +testLoggerAdapter verbose: server-52bb4c20-c73a-5f29-2c36-f49961f116a9: 2024-01-08T00:31:11.422Z [INFO] server-52bb4c20-c73a-5f29-2c36-f49961f116a9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3dbabc23-a375-d2ba-b0f8-902fa23b8958 Address:3dbabc23-a375-d2ba-b0f8-902fa23b8958} {Suffrage:Voter ID:server-031f68ac-1e78-0df1-54e4-4f634840d4d0 Address:031f68ac-1e78-0df1-54e4-4f634840d4d0} {Suffrage:Voter ID:server-52bb4c20-c73a-5f29-2c36-f49961f116a9 Address:52bb4c20-c73a-5f29-2c36-f49961f116a9}]" + testing.go:149: server-52bb4c20-c73a-5f29-2c36-f49961f116a9: 2024-01-08T00:31:11.422Z [INFO] server-52bb4c20-c73a-5f29-2c36-f49961f116a9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3dbabc23-a375-d2ba-b0f8-902fa23b8958 Address:3dbabc23-a375-d2ba-b0f8-902fa23b8958} {Suffrage:Voter ID:server-031f68ac-1e78-0df1-54e4-4f634840d4d0 Address:031f68ac-1e78-0df1-54e4-4f634840d4d0} {Suffrage:Voter ID:server-52bb4c20-c73a-5f29-2c36-f49961f116a9 Address:52bb4c20-c73a-5f29-2c36-f49961f116a9}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:11.422Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:11.422Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: 2024-01-08T00:31:11.422Z [INFO] server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: entering follower state: follower="Node at 3dbabc23-a375-d2ba-b0f8-902fa23b8958 [Follower]" leader= + testing.go:149: server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: 2024-01-08T00:31:11.422Z [INFO] server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: entering follower state: follower="Node at 3dbabc23-a375-d2ba-b0f8-902fa23b8958 [Follower]" leader= +testLoggerAdapter verbose: server-031f68ac-1e78-0df1-54e4-4f634840d4d0: 2024-01-08T00:31:11.423Z [INFO] server-031f68ac-1e78-0df1-54e4-4f634840d4d0: entering follower state: follower="Node at 031f68ac-1e78-0df1-54e4-4f634840d4d0 [Follower]" leader= + testing.go:149: server-031f68ac-1e78-0df1-54e4-4f634840d4d0: 2024-01-08T00:31:11.423Z [INFO] server-031f68ac-1e78-0df1-54e4-4f634840d4d0: entering follower state: follower="Node at 031f68ac-1e78-0df1-54e4-4f634840d4d0 [Follower]" leader= +testLoggerAdapter verbose: server-52bb4c20-c73a-5f29-2c36-f49961f116a9: 2024-01-08T00:31:11.423Z [INFO] server-52bb4c20-c73a-5f29-2c36-f49961f116a9: entering follower state: follower="Node at 52bb4c20-c73a-5f29-2c36-f49961f116a9 [Follower]" leader= + testing.go:149: server-52bb4c20-c73a-5f29-2c36-f49961f116a9: 2024-01-08T00:31:11.423Z [INFO] server-52bb4c20-c73a-5f29-2c36-f49961f116a9: entering follower state: follower="Node at 52bb4c20-c73a-5f29-2c36-f49961f116a9 [Follower]" leader= +testLoggerAdapter verbose: server-031f68ac-1e78-0df1-54e4-4f634840d4d0: 2024-01-08T00:31:11.476Z [WARN] server-031f68ac-1e78-0df1-54e4-4f634840d4d0: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-031f68ac-1e78-0df1-54e4-4f634840d4d0: 2024-01-08T00:31:11.476Z [WARN] server-031f68ac-1e78-0df1-54e4-4f634840d4d0: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-031f68ac-1e78-0df1-54e4-4f634840d4d0: 2024-01-08T00:31:11.476Z [INFO] server-031f68ac-1e78-0df1-54e4-4f634840d4d0: entering candidate state: node="Node at 031f68ac-1e78-0df1-54e4-4f634840d4d0 [Candidate]" term=2 + testing.go:149: server-031f68ac-1e78-0df1-54e4-4f634840d4d0: 2024-01-08T00:31:11.476Z [INFO] server-031f68ac-1e78-0df1-54e4-4f634840d4d0: entering candidate state: node="Node at 031f68ac-1e78-0df1-54e4-4f634840d4d0 [Candidate]" term=2 +testLoggerAdapter verbose: server-031f68ac-1e78-0df1-54e4-4f634840d4d0: 2024-01-08T00:31:11.476Z [INFO] server-031f68ac-1e78-0df1-54e4-4f634840d4d0: election won: tally=2 + testing.go:149: server-031f68ac-1e78-0df1-54e4-4f634840d4d0: 2024-01-08T00:31:11.476Z [INFO] server-031f68ac-1e78-0df1-54e4-4f634840d4d0: election won: tally=2 +testLoggerAdapter verbose: server-031f68ac-1e78-0df1-54e4-4f634840d4d0: 2024-01-08T00:31:11.476Z [INFO] server-031f68ac-1e78-0df1-54e4-4f634840d4d0: entering leader state: leader="Node at 031f68ac-1e78-0df1-54e4-4f634840d4d0 [Leader]" + testing.go:149: server-031f68ac-1e78-0df1-54e4-4f634840d4d0: 2024-01-08T00:31:11.476Z [INFO] server-031f68ac-1e78-0df1-54e4-4f634840d4d0: entering leader state: leader="Node at 031f68ac-1e78-0df1-54e4-4f634840d4d0 [Leader]" +testLoggerAdapter verbose: server-031f68ac-1e78-0df1-54e4-4f634840d4d0: 2024-01-08T00:31:11.476Z [INFO] server-031f68ac-1e78-0df1-54e4-4f634840d4d0: added peer, starting replication: peer=server-3dbabc23-a375-d2ba-b0f8-902fa23b8958 + testing.go:149: server-031f68ac-1e78-0df1-54e4-4f634840d4d0: 2024-01-08T00:31:11.476Z [INFO] server-031f68ac-1e78-0df1-54e4-4f634840d4d0: added peer, starting replication: peer=server-3dbabc23-a375-d2ba-b0f8-902fa23b8958 +testLoggerAdapter verbose: server-031f68ac-1e78-0df1-54e4-4f634840d4d0: 2024-01-08T00:31:11.476Z [INFO] server-031f68ac-1e78-0df1-54e4-4f634840d4d0: added peer, starting replication: peer=server-52bb4c20-c73a-5f29-2c36-f49961f116a9 + testing.go:149: server-031f68ac-1e78-0df1-54e4-4f634840d4d0: 2024-01-08T00:31:11.476Z [INFO] server-031f68ac-1e78-0df1-54e4-4f634840d4d0: added peer, starting replication: peer=server-52bb4c20-c73a-5f29-2c36-f49961f116a9 +testLoggerAdapter verbose: server-031f68ac-1e78-0df1-54e4-4f634840d4d0: 2024-01-08T00:31:11.477Z [INFO] server-031f68ac-1e78-0df1-54e4-4f634840d4d0: pipelining replication: peer="{Voter server-52bb4c20-c73a-5f29-2c36-f49961f116a9 52bb4c20-c73a-5f29-2c36-f49961f116a9}" + testing.go:149: server-031f68ac-1e78-0df1-54e4-4f634840d4d0: 2024-01-08T00:31:11.477Z [INFO] server-031f68ac-1e78-0df1-54e4-4f634840d4d0: pipelining replication: peer="{Voter server-52bb4c20-c73a-5f29-2c36-f49961f116a9 52bb4c20-c73a-5f29-2c36-f49961f116a9}" +testLoggerAdapter verbose: server-031f68ac-1e78-0df1-54e4-4f634840d4d0: 2024-01-08T00:31:11.477Z [INFO] server-031f68ac-1e78-0df1-54e4-4f634840d4d0: pipelining replication: peer="{Voter server-3dbabc23-a375-d2ba-b0f8-902fa23b8958 3dbabc23-a375-d2ba-b0f8-902fa23b8958}" + testing.go:149: server-031f68ac-1e78-0df1-54e4-4f634840d4d0: 2024-01-08T00:31:11.477Z [INFO] server-031f68ac-1e78-0df1-54e4-4f634840d4d0: pipelining replication: peer="{Voter server-3dbabc23-a375-d2ba-b0f8-902fa23b8958 3dbabc23-a375-d2ba-b0f8-902fa23b8958}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:11.582Z [INFO] stable state for Leader reached at 2024-01-08 00:31:11.477420469 +0000 UTC m=+17.524309579 (1 nodes), 54.502469ms from start of poll, 55.074981ms from cluster start. Timeout at 2024-01-08 00:31:11.582750866 +0000 UTC m=+17.629640071, 105.330492ms after stability + testing.go:149: cluster: 2024-01-08T00:31:11.582Z [INFO] stable state for Leader reached at 2024-01-08 00:31:11.477420469 +0000 UTC m=+17.524309579 (1 nodes), 54.502469ms from start of poll, 55.074981ms from cluster start. Timeout at 2024-01-08 00:31:11.582750866 +0000 UTC m=+17.629640071, 105.330492ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:11.582Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:11.582Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:11.688Z [INFO] stable state for Leader reached at 2024-01-08 00:31:11.582907337 +0000 UTC m=+17.629796433 (1 nodes), 2.27µs from start of poll, 160.561835ms from cluster start. Timeout at 2024-01-08 00:31:11.688642576 +0000 UTC m=+17.735531675, 105.735242ms after stability + testing.go:149: cluster: 2024-01-08T00:31:11.688Z [INFO] stable state for Leader reached at 2024-01-08 00:31:11.582907337 +0000 UTC m=+17.629796433 (1 nodes), 2.27µs from start of poll, 160.561835ms from cluster start. Timeout at 2024-01-08 00:31:11.688642576 +0000 UTC m=+17.735531675, 105.735242ms after stability +testLoggerAdapter verbose: server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: 2024-01-08T00:31:11.688Z [INFO] server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: entering candidate state: node="Node at 3dbabc23-a375-d2ba-b0f8-902fa23b8958 [Candidate]" term=3 + testing.go:149: server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: 2024-01-08T00:31:11.688Z [INFO] server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: entering candidate state: node="Node at 3dbabc23-a375-d2ba-b0f8-902fa23b8958 [Candidate]" term=3 +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:11.688Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:11.688Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: 2024-01-08T00:31:11.689Z [INFO] server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: election won: tally=2 + testing.go:149: server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: 2024-01-08T00:31:11.689Z [INFO] server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: election won: tally=2 +testLoggerAdapter verbose: server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: 2024-01-08T00:31:11.689Z [INFO] server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: entering leader state: leader="Node at 3dbabc23-a375-d2ba-b0f8-902fa23b8958 [Leader]" +testLoggerAdapter verbose: server-031f68ac-1e78-0df1-54e4-4f634840d4d0: 2024-01-08T00:31:11.689Z [INFO] server-031f68ac-1e78-0df1-54e4-4f634840d4d0: entering follower state: follower="Node at 031f68ac-1e78-0df1-54e4-4f634840d4d0 [Follower]" leader= + testing.go:149: server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: 2024-01-08T00:31:11.689Z [INFO] server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: entering leader state: leader="Node at 3dbabc23-a375-d2ba-b0f8-902fa23b8958 [Leader]" +testLoggerAdapter verbose: server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: 2024-01-08T00:31:11.689Z [INFO] server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: added peer, starting replication: peer=server-031f68ac-1e78-0df1-54e4-4f634840d4d0 + testing.go:149: server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: 2024-01-08T00:31:11.689Z [INFO] server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: added peer, starting replication: peer=server-031f68ac-1e78-0df1-54e4-4f634840d4d0 +testLoggerAdapter verbose: server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: 2024-01-08T00:31:11.689Z [INFO] server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: added peer, starting replication: peer=server-52bb4c20-c73a-5f29-2c36-f49961f116a9 + testing.go:149: server-031f68ac-1e78-0df1-54e4-4f634840d4d0: 2024-01-08T00:31:11.689Z [INFO] server-031f68ac-1e78-0df1-54e4-4f634840d4d0: entering follower state: follower="Node at 031f68ac-1e78-0df1-54e4-4f634840d4d0 [Follower]" leader= + testing.go:149: server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: 2024-01-08T00:31:11.689Z [INFO] server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: added peer, starting replication: peer=server-52bb4c20-c73a-5f29-2c36-f49961f116a9 +testLoggerAdapter verbose: server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: 2024-01-08T00:31:11.689Z [INFO] server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: pipelining replication: peer="{Voter server-031f68ac-1e78-0df1-54e4-4f634840d4d0 031f68ac-1e78-0df1-54e4-4f634840d4d0}" + testing.go:149: server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: 2024-01-08T00:31:11.689Z [INFO] server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: pipelining replication: peer="{Voter server-031f68ac-1e78-0df1-54e4-4f634840d4d0 031f68ac-1e78-0df1-54e4-4f634840d4d0}" +testLoggerAdapter verbose: server-031f68ac-1e78-0df1-54e4-4f634840d4d0: 2024-01-08T00:31:11.689Z [INFO] server-031f68ac-1e78-0df1-54e4-4f634840d4d0: aborting pipeline replication: peer="{Voter server-3dbabc23-a375-d2ba-b0f8-902fa23b8958 3dbabc23-a375-d2ba-b0f8-902fa23b8958}" +testLoggerAdapter verbose: server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: 2024-01-08T00:31:11.689Z [INFO] server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: pipelining replication: peer="{Voter server-52bb4c20-c73a-5f29-2c36-f49961f116a9 52bb4c20-c73a-5f29-2c36-f49961f116a9}" + testing.go:149: server-031f68ac-1e78-0df1-54e4-4f634840d4d0: 2024-01-08T00:31:11.689Z [INFO] server-031f68ac-1e78-0df1-54e4-4f634840d4d0: aborting pipeline replication: peer="{Voter server-3dbabc23-a375-d2ba-b0f8-902fa23b8958 3dbabc23-a375-d2ba-b0f8-902fa23b8958}" +testLoggerAdapter verbose: server-031f68ac-1e78-0df1-54e4-4f634840d4d0: 2024-01-08T00:31:11.689Z [INFO] server-031f68ac-1e78-0df1-54e4-4f634840d4d0: aborting pipeline replication: peer="{Voter server-52bb4c20-c73a-5f29-2c36-f49961f116a9 52bb4c20-c73a-5f29-2c36-f49961f116a9}" + testing.go:149: server-031f68ac-1e78-0df1-54e4-4f634840d4d0: 2024-01-08T00:31:11.689Z [INFO] server-031f68ac-1e78-0df1-54e4-4f634840d4d0: aborting pipeline replication: peer="{Voter server-52bb4c20-c73a-5f29-2c36-f49961f116a9 52bb4c20-c73a-5f29-2c36-f49961f116a9}" + testing.go:149: server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: 2024-01-08T00:31:11.689Z [INFO] server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: pipelining replication: peer="{Voter server-52bb4c20-c73a-5f29-2c36-f49961f116a9 52bb4c20-c73a-5f29-2c36-f49961f116a9}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:11.794Z [INFO] stable state for Leader reached at 2024-01-08 00:31:11.689150918 +0000 UTC m=+17.736040039 (1 nodes), 119.765µs from start of poll, 266.805441ms from cluster start. Timeout at 2024-01-08 00:31:11.794660931 +0000 UTC m=+17.841550033, 105.509994ms after stability + testing.go:149: cluster: 2024-01-08T00:31:11.794Z [INFO] stable state for Leader reached at 2024-01-08 00:31:11.689150918 +0000 UTC m=+17.736040039 (1 nodes), 119.765µs from start of poll, 266.805441ms from cluster start. Timeout at 2024-01-08 00:31:11.794660931 +0000 UTC m=+17.841550033, 105.509994ms after stability +testLoggerAdapter verbose: server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: 2024-01-08T00:31:11.794Z [INFO] server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: aborting pipeline replication: peer="{Voter server-52bb4c20-c73a-5f29-2c36-f49961f116a9 52bb4c20-c73a-5f29-2c36-f49961f116a9}" + testing.go:149: server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: 2024-01-08T00:31:11.794Z [INFO] server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: aborting pipeline replication: peer="{Voter server-52bb4c20-c73a-5f29-2c36-f49961f116a9 52bb4c20-c73a-5f29-2c36-f49961f116a9}" +testLoggerAdapter verbose: server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: 2024-01-08T00:31:11.794Z [INFO] server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: aborting pipeline replication: peer="{Voter server-031f68ac-1e78-0df1-54e4-4f634840d4d0 031f68ac-1e78-0df1-54e4-4f634840d4d0}" + testing.go:149: server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: 2024-01-08T00:31:11.794Z [INFO] server-3dbabc23-a375-d2ba-b0f8-902fa23b8958: aborting pipeline replication: peer="{Voter server-031f68ac-1e78-0df1-54e4-4f634840d4d0 031f68ac-1e78-0df1-54e4-4f634840d4d0}" +--- PASS: TestRaft_LeadershipTransfer (0.37s) === RUN TestRaft_LeadershipTransferWithOneNode -testLoggerAdapter verbose: server-eb11ecf7-adbe-fb0e-7d9e-a51a1433868e: 2025-02-09T06:52:36.357Z [INFO] server-eb11ecf7-adbe-fb0e-7d9e-a51a1433868e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eb11ecf7-adbe-fb0e-7d9e-a51a1433868e Address:eb11ecf7-adbe-fb0e-7d9e-a51a1433868e}]" - testing.go:149: server-eb11ecf7-adbe-fb0e-7d9e-a51a1433868e: 2025-02-09T06:52:36.357Z [INFO] server-eb11ecf7-adbe-fb0e-7d9e-a51a1433868e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-eb11ecf7-adbe-fb0e-7d9e-a51a1433868e Address:eb11ecf7-adbe-fb0e-7d9e-a51a1433868e}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:36.357Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:36.357Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-eb11ecf7-adbe-fb0e-7d9e-a51a1433868e: 2025-02-09T06:52:36.358Z [INFO] server-eb11ecf7-adbe-fb0e-7d9e-a51a1433868e: entering follower state: follower="Node at eb11ecf7-adbe-fb0e-7d9e-a51a1433868e [Follower]" leader= - testing.go:149: server-eb11ecf7-adbe-fb0e-7d9e-a51a1433868e: 2025-02-09T06:52:36.358Z [INFO] server-eb11ecf7-adbe-fb0e-7d9e-a51a1433868e: entering follower state: follower="Node at eb11ecf7-adbe-fb0e-7d9e-a51a1433868e [Follower]" leader= -testLoggerAdapter verbose: server-eb11ecf7-adbe-fb0e-7d9e-a51a1433868e: 2025-02-09T06:52:36.412Z [WARN] server-eb11ecf7-adbe-fb0e-7d9e-a51a1433868e: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-eb11ecf7-adbe-fb0e-7d9e-a51a1433868e: 2025-02-09T06:52:36.412Z [WARN] server-eb11ecf7-adbe-fb0e-7d9e-a51a1433868e: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-eb11ecf7-adbe-fb0e-7d9e-a51a1433868e: 2025-02-09T06:52:36.412Z [INFO] server-eb11ecf7-adbe-fb0e-7d9e-a51a1433868e: entering candidate state: node="Node at eb11ecf7-adbe-fb0e-7d9e-a51a1433868e [Candidate]" term=2 - testing.go:149: server-eb11ecf7-adbe-fb0e-7d9e-a51a1433868e: 2025-02-09T06:52:36.412Z [INFO] server-eb11ecf7-adbe-fb0e-7d9e-a51a1433868e: entering candidate state: node="Node at eb11ecf7-adbe-fb0e-7d9e-a51a1433868e [Candidate]" term=2 -testLoggerAdapter verbose: server-eb11ecf7-adbe-fb0e-7d9e-a51a1433868e: 2025-02-09T06:52:36.412Z [INFO] server-eb11ecf7-adbe-fb0e-7d9e-a51a1433868e: election won: tally=1 - testing.go:149: server-eb11ecf7-adbe-fb0e-7d9e-a51a1433868e: 2025-02-09T06:52:36.412Z [INFO] server-eb11ecf7-adbe-fb0e-7d9e-a51a1433868e: election won: tally=1 -testLoggerAdapter verbose: server-eb11ecf7-adbe-fb0e-7d9e-a51a1433868e: 2025-02-09T06:52:36.412Z [INFO] server-eb11ecf7-adbe-fb0e-7d9e-a51a1433868e: entering leader state: leader="Node at eb11ecf7-adbe-fb0e-7d9e-a51a1433868e [Leader]" - testing.go:149: server-eb11ecf7-adbe-fb0e-7d9e-a51a1433868e: 2025-02-09T06:52:36.412Z [INFO] server-eb11ecf7-adbe-fb0e-7d9e-a51a1433868e: entering leader state: leader="Node at eb11ecf7-adbe-fb0e-7d9e-a51a1433868e [Leader]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:36.518Z [INFO] stable state for Leader reached at 2025-02-09 06:52:36.412989292 +0000 UTC m=+17.391798057 (1 nodes), 54.976152ms from start of poll, 55.140166ms from cluster start. Timeout at 2025-02-09 06:52:36.518446607 +0000 UTC m=+17.497255379, 105.457322ms after stability - testing.go:149: cluster: 2025-02-09T06:52:36.518Z [INFO] stable state for Leader reached at 2025-02-09 06:52:36.412989292 +0000 UTC m=+17.391798057 (1 nodes), 54.976152ms from start of poll, 55.140166ms from cluster start. Timeout at 2025-02-09 06:52:36.518446607 +0000 UTC m=+17.497255379, 105.457322ms after stability ---- PASS: TestRaft_LeadershipTransferWithOneNode (0.16s) +testLoggerAdapter verbose: server-e3816327-22ab-a511-1bb4-bd211d644cc7: 2024-01-08T00:31:11.796Z [INFO] server-e3816327-22ab-a511-1bb4-bd211d644cc7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e3816327-22ab-a511-1bb4-bd211d644cc7 Address:e3816327-22ab-a511-1bb4-bd211d644cc7}]" + testing.go:149: server-e3816327-22ab-a511-1bb4-bd211d644cc7: 2024-01-08T00:31:11.796Z [INFO] server-e3816327-22ab-a511-1bb4-bd211d644cc7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e3816327-22ab-a511-1bb4-bd211d644cc7 Address:e3816327-22ab-a511-1bb4-bd211d644cc7}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:11.796Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:11.796Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-e3816327-22ab-a511-1bb4-bd211d644cc7: 2024-01-08T00:31:11.796Z [INFO] server-e3816327-22ab-a511-1bb4-bd211d644cc7: entering follower state: follower="Node at e3816327-22ab-a511-1bb4-bd211d644cc7 [Follower]" leader= + testing.go:149: server-e3816327-22ab-a511-1bb4-bd211d644cc7: 2024-01-08T00:31:11.796Z [INFO] server-e3816327-22ab-a511-1bb4-bd211d644cc7: entering follower state: follower="Node at e3816327-22ab-a511-1bb4-bd211d644cc7 [Follower]" leader= +testLoggerAdapter verbose: server-e3816327-22ab-a511-1bb4-bd211d644cc7: 2024-01-08T00:31:11.873Z [WARN] server-e3816327-22ab-a511-1bb4-bd211d644cc7: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-e3816327-22ab-a511-1bb4-bd211d644cc7: 2024-01-08T00:31:11.873Z [WARN] server-e3816327-22ab-a511-1bb4-bd211d644cc7: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-e3816327-22ab-a511-1bb4-bd211d644cc7: 2024-01-08T00:31:11.873Z [INFO] server-e3816327-22ab-a511-1bb4-bd211d644cc7: entering candidate state: node="Node at e3816327-22ab-a511-1bb4-bd211d644cc7 [Candidate]" term=2 + testing.go:149: server-e3816327-22ab-a511-1bb4-bd211d644cc7: 2024-01-08T00:31:11.873Z [INFO] server-e3816327-22ab-a511-1bb4-bd211d644cc7: entering candidate state: node="Node at e3816327-22ab-a511-1bb4-bd211d644cc7 [Candidate]" term=2 +testLoggerAdapter verbose: server-e3816327-22ab-a511-1bb4-bd211d644cc7: 2024-01-08T00:31:11.873Z [INFO] server-e3816327-22ab-a511-1bb4-bd211d644cc7: election won: tally=1 + testing.go:149: server-e3816327-22ab-a511-1bb4-bd211d644cc7: 2024-01-08T00:31:11.873Z [INFO] server-e3816327-22ab-a511-1bb4-bd211d644cc7: election won: tally=1 +testLoggerAdapter verbose: server-e3816327-22ab-a511-1bb4-bd211d644cc7: 2024-01-08T00:31:11.873Z [INFO] server-e3816327-22ab-a511-1bb4-bd211d644cc7: entering leader state: leader="Node at e3816327-22ab-a511-1bb4-bd211d644cc7 [Leader]" + testing.go:149: server-e3816327-22ab-a511-1bb4-bd211d644cc7: 2024-01-08T00:31:11.873Z [INFO] server-e3816327-22ab-a511-1bb4-bd211d644cc7: entering leader state: leader="Node at e3816327-22ab-a511-1bb4-bd211d644cc7 [Leader]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:11.978Z [INFO] stable state for Leader reached at 2024-01-08 00:31:11.873408787 +0000 UTC m=+17.920297987 (1 nodes), 77.166729ms from start of poll, 77.424167ms from cluster start. Timeout at 2024-01-08 00:31:11.978719871 +0000 UTC m=+18.025609160, 105.311173ms after stability + testing.go:149: cluster: 2024-01-08T00:31:11.978Z [INFO] stable state for Leader reached at 2024-01-08 00:31:11.873408787 +0000 UTC m=+17.920297987 (1 nodes), 77.166729ms from start of poll, 77.424167ms from cluster start. Timeout at 2024-01-08 00:31:11.978719871 +0000 UTC m=+18.025609160, 105.311173ms after stability +--- PASS: TestRaft_LeadershipTransferWithOneNode (0.18s) === RUN TestRaft_LeadershipTransferWithSevenNodes -testLoggerAdapter verbose: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.519Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9eb65806-9efa-c938-bc7e-a31eed882685 Address:9eb65806-9efa-c938-bc7e-a31eed882685} {Suffrage:Voter ID:server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd Address:b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd} {Suffrage:Voter ID:server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8 Address:758cf40e-3688-7e45-04ad-ef7e8d7fe5c8} {Suffrage:Voter ID:server-31718b08-787f-39a7-143e-bd02cb427cd2 Address:31718b08-787f-39a7-143e-bd02cb427cd2} {Suffrage:Voter ID:server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef Address:8a069cab-9494-f9fa-96fb-2f80e9ef85ef} {Suffrage:Voter ID:server-7f28fa71-9f71-f154-6c0f-02ecacf2d840 Address:7f28fa71-9f71-f154-6c0f-02ecacf2d840} {Suffrage:Voter ID:server-99c861f7-d592-33f2-7a9e-bccd0bd28952 Address:99c861f7-d592-33f2-7a9e-bccd0bd28952}]" - testing.go:149: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.519Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9eb65806-9efa-c938-bc7e-a31eed882685 Address:9eb65806-9efa-c938-bc7e-a31eed882685} {Suffrage:Voter ID:server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd Address:b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd} {Suffrage:Voter ID:server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8 Address:758cf40e-3688-7e45-04ad-ef7e8d7fe5c8} {Suffrage:Voter ID:server-31718b08-787f-39a7-143e-bd02cb427cd2 Address:31718b08-787f-39a7-143e-bd02cb427cd2} {Suffrage:Voter ID:server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef Address:8a069cab-9494-f9fa-96fb-2f80e9ef85ef} {Suffrage:Voter ID:server-7f28fa71-9f71-f154-6c0f-02ecacf2d840 Address:7f28fa71-9f71-f154-6c0f-02ecacf2d840} {Suffrage:Voter ID:server-99c861f7-d592-33f2-7a9e-bccd0bd28952 Address:99c861f7-d592-33f2-7a9e-bccd0bd28952}]" -testLoggerAdapter verbose: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.519Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: entering follower state: follower="Node at 9eb65806-9efa-c938-bc7e-a31eed882685 [Follower]" leader= - testing.go:149: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.519Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: entering follower state: follower="Node at 9eb65806-9efa-c938-bc7e-a31eed882685 [Follower]" leader= -testLoggerAdapter verbose: server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd: 2025-02-09T06:52:36.519Z [INFO] server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9eb65806-9efa-c938-bc7e-a31eed882685 Address:9eb65806-9efa-c938-bc7e-a31eed882685} {Suffrage:Voter ID:server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd Address:b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd} {Suffrage:Voter ID:server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8 Address:758cf40e-3688-7e45-04ad-ef7e8d7fe5c8} {Suffrage:Voter ID:server-31718b08-787f-39a7-143e-bd02cb427cd2 Address:31718b08-787f-39a7-143e-bd02cb427cd2} {Suffrage:Voter ID:server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef Address:8a069cab-9494-f9fa-96fb-2f80e9ef85ef} {Suffrage:Voter ID:server-7f28fa71-9f71-f154-6c0f-02ecacf2d840 Address:7f28fa71-9f71-f154-6c0f-02ecacf2d840} {Suffrage:Voter ID:server-99c861f7-d592-33f2-7a9e-bccd0bd28952 Address:99c861f7-d592-33f2-7a9e-bccd0bd28952}]" - testing.go:149: server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd: 2025-02-09T06:52:36.519Z [INFO] server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9eb65806-9efa-c938-bc7e-a31eed882685 Address:9eb65806-9efa-c938-bc7e-a31eed882685} {Suffrage:Voter ID:server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd Address:b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd} {Suffrage:Voter ID:server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8 Address:758cf40e-3688-7e45-04ad-ef7e8d7fe5c8} {Suffrage:Voter ID:server-31718b08-787f-39a7-143e-bd02cb427cd2 Address:31718b08-787f-39a7-143e-bd02cb427cd2} {Suffrage:Voter ID:server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef Address:8a069cab-9494-f9fa-96fb-2f80e9ef85ef} {Suffrage:Voter ID:server-7f28fa71-9f71-f154-6c0f-02ecacf2d840 Address:7f28fa71-9f71-f154-6c0f-02ecacf2d840} {Suffrage:Voter ID:server-99c861f7-d592-33f2-7a9e-bccd0bd28952 Address:99c861f7-d592-33f2-7a9e-bccd0bd28952}]" -testLoggerAdapter verbose: server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd: 2025-02-09T06:52:36.519Z [INFO] server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd: entering follower state: follower="Node at b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd [Follower]" leader= - testing.go:149: server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd: 2025-02-09T06:52:36.519Z [INFO] server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd: entering follower state: follower="Node at b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd [Follower]" leader= -testLoggerAdapter verbose: server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8: 2025-02-09T06:52:36.519Z [INFO] server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9eb65806-9efa-c938-bc7e-a31eed882685 Address:9eb65806-9efa-c938-bc7e-a31eed882685} {Suffrage:Voter ID:server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd Address:b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd} {Suffrage:Voter ID:server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8 Address:758cf40e-3688-7e45-04ad-ef7e8d7fe5c8} {Suffrage:Voter ID:server-31718b08-787f-39a7-143e-bd02cb427cd2 Address:31718b08-787f-39a7-143e-bd02cb427cd2} {Suffrage:Voter ID:server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef Address:8a069cab-9494-f9fa-96fb-2f80e9ef85ef} {Suffrage:Voter ID:server-7f28fa71-9f71-f154-6c0f-02ecacf2d840 Address:7f28fa71-9f71-f154-6c0f-02ecacf2d840} {Suffrage:Voter ID:server-99c861f7-d592-33f2-7a9e-bccd0bd28952 Address:99c861f7-d592-33f2-7a9e-bccd0bd28952}]" - testing.go:149: server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8: 2025-02-09T06:52:36.519Z [INFO] server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9eb65806-9efa-c938-bc7e-a31eed882685 Address:9eb65806-9efa-c938-bc7e-a31eed882685} {Suffrage:Voter ID:server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd Address:b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd} {Suffrage:Voter ID:server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8 Address:758cf40e-3688-7e45-04ad-ef7e8d7fe5c8} {Suffrage:Voter ID:server-31718b08-787f-39a7-143e-bd02cb427cd2 Address:31718b08-787f-39a7-143e-bd02cb427cd2} {Suffrage:Voter ID:server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef Address:8a069cab-9494-f9fa-96fb-2f80e9ef85ef} {Suffrage:Voter ID:server-7f28fa71-9f71-f154-6c0f-02ecacf2d840 Address:7f28fa71-9f71-f154-6c0f-02ecacf2d840} {Suffrage:Voter ID:server-99c861f7-d592-33f2-7a9e-bccd0bd28952 Address:99c861f7-d592-33f2-7a9e-bccd0bd28952}]" -testLoggerAdapter verbose: server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8: 2025-02-09T06:52:36.519Z [INFO] server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8: entering follower state: follower="Node at 758cf40e-3688-7e45-04ad-ef7e8d7fe5c8 [Follower]" leader= - testing.go:149: server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8: 2025-02-09T06:52:36.519Z [INFO] server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8: entering follower state: follower="Node at 758cf40e-3688-7e45-04ad-ef7e8d7fe5c8 [Follower]" leader= -testLoggerAdapter verbose: server-31718b08-787f-39a7-143e-bd02cb427cd2: 2025-02-09T06:52:36.520Z [INFO] server-31718b08-787f-39a7-143e-bd02cb427cd2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9eb65806-9efa-c938-bc7e-a31eed882685 Address:9eb65806-9efa-c938-bc7e-a31eed882685} {Suffrage:Voter ID:server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd Address:b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd} {Suffrage:Voter ID:server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8 Address:758cf40e-3688-7e45-04ad-ef7e8d7fe5c8} {Suffrage:Voter ID:server-31718b08-787f-39a7-143e-bd02cb427cd2 Address:31718b08-787f-39a7-143e-bd02cb427cd2} {Suffrage:Voter ID:server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef Address:8a069cab-9494-f9fa-96fb-2f80e9ef85ef} {Suffrage:Voter ID:server-7f28fa71-9f71-f154-6c0f-02ecacf2d840 Address:7f28fa71-9f71-f154-6c0f-02ecacf2d840} {Suffrage:Voter ID:server-99c861f7-d592-33f2-7a9e-bccd0bd28952 Address:99c861f7-d592-33f2-7a9e-bccd0bd28952}]" - testing.go:149: server-31718b08-787f-39a7-143e-bd02cb427cd2: 2025-02-09T06:52:36.520Z [INFO] server-31718b08-787f-39a7-143e-bd02cb427cd2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9eb65806-9efa-c938-bc7e-a31eed882685 Address:9eb65806-9efa-c938-bc7e-a31eed882685} {Suffrage:Voter ID:server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd Address:b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd} {Suffrage:Voter ID:server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8 Address:758cf40e-3688-7e45-04ad-ef7e8d7fe5c8} {Suffrage:Voter ID:server-31718b08-787f-39a7-143e-bd02cb427cd2 Address:31718b08-787f-39a7-143e-bd02cb427cd2} {Suffrage:Voter ID:server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef Address:8a069cab-9494-f9fa-96fb-2f80e9ef85ef} {Suffrage:Voter ID:server-7f28fa71-9f71-f154-6c0f-02ecacf2d840 Address:7f28fa71-9f71-f154-6c0f-02ecacf2d840} {Suffrage:Voter ID:server-99c861f7-d592-33f2-7a9e-bccd0bd28952 Address:99c861f7-d592-33f2-7a9e-bccd0bd28952}]" -testLoggerAdapter verbose: server-31718b08-787f-39a7-143e-bd02cb427cd2: 2025-02-09T06:52:36.520Z [INFO] server-31718b08-787f-39a7-143e-bd02cb427cd2: entering follower state: follower="Node at 31718b08-787f-39a7-143e-bd02cb427cd2 [Follower]" leader= - testing.go:149: server-31718b08-787f-39a7-143e-bd02cb427cd2: 2025-02-09T06:52:36.520Z [INFO] server-31718b08-787f-39a7-143e-bd02cb427cd2: entering follower state: follower="Node at 31718b08-787f-39a7-143e-bd02cb427cd2 [Follower]" leader= -testLoggerAdapter verbose: server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef: 2025-02-09T06:52:36.520Z [INFO] server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9eb65806-9efa-c938-bc7e-a31eed882685 Address:9eb65806-9efa-c938-bc7e-a31eed882685} {Suffrage:Voter ID:server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd Address:b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd} {Suffrage:Voter ID:server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8 Address:758cf40e-3688-7e45-04ad-ef7e8d7fe5c8} {Suffrage:Voter ID:server-31718b08-787f-39a7-143e-bd02cb427cd2 Address:31718b08-787f-39a7-143e-bd02cb427cd2} {Suffrage:Voter ID:server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef Address:8a069cab-9494-f9fa-96fb-2f80e9ef85ef} {Suffrage:Voter ID:server-7f28fa71-9f71-f154-6c0f-02ecacf2d840 Address:7f28fa71-9f71-f154-6c0f-02ecacf2d840} {Suffrage:Voter ID:server-99c861f7-d592-33f2-7a9e-bccd0bd28952 Address:99c861f7-d592-33f2-7a9e-bccd0bd28952}]" - testing.go:149: server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef: 2025-02-09T06:52:36.520Z [INFO] server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9eb65806-9efa-c938-bc7e-a31eed882685 Address:9eb65806-9efa-c938-bc7e-a31eed882685} {Suffrage:Voter ID:server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd Address:b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd} {Suffrage:Voter ID:server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8 Address:758cf40e-3688-7e45-04ad-ef7e8d7fe5c8} {Suffrage:Voter ID:server-31718b08-787f-39a7-143e-bd02cb427cd2 Address:31718b08-787f-39a7-143e-bd02cb427cd2} {Suffrage:Voter ID:server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef Address:8a069cab-9494-f9fa-96fb-2f80e9ef85ef} {Suffrage:Voter ID:server-7f28fa71-9f71-f154-6c0f-02ecacf2d840 Address:7f28fa71-9f71-f154-6c0f-02ecacf2d840} {Suffrage:Voter ID:server-99c861f7-d592-33f2-7a9e-bccd0bd28952 Address:99c861f7-d592-33f2-7a9e-bccd0bd28952}]" -testLoggerAdapter verbose: server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef: 2025-02-09T06:52:36.520Z [INFO] server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef: entering follower state: follower="Node at 8a069cab-9494-f9fa-96fb-2f80e9ef85ef [Follower]" leader= - testing.go:149: server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef: 2025-02-09T06:52:36.520Z [INFO] server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef: entering follower state: follower="Node at 8a069cab-9494-f9fa-96fb-2f80e9ef85ef [Follower]" leader= -testLoggerAdapter verbose: server-7f28fa71-9f71-f154-6c0f-02ecacf2d840: 2025-02-09T06:52:36.520Z [INFO] server-7f28fa71-9f71-f154-6c0f-02ecacf2d840: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9eb65806-9efa-c938-bc7e-a31eed882685 Address:9eb65806-9efa-c938-bc7e-a31eed882685} {Suffrage:Voter ID:server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd Address:b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd} {Suffrage:Voter ID:server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8 Address:758cf40e-3688-7e45-04ad-ef7e8d7fe5c8} {Suffrage:Voter ID:server-31718b08-787f-39a7-143e-bd02cb427cd2 Address:31718b08-787f-39a7-143e-bd02cb427cd2} {Suffrage:Voter ID:server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef Address:8a069cab-9494-f9fa-96fb-2f80e9ef85ef} {Suffrage:Voter ID:server-7f28fa71-9f71-f154-6c0f-02ecacf2d840 Address:7f28fa71-9f71-f154-6c0f-02ecacf2d840} {Suffrage:Voter ID:server-99c861f7-d592-33f2-7a9e-bccd0bd28952 Address:99c861f7-d592-33f2-7a9e-bccd0bd28952}]" - testing.go:149: server-7f28fa71-9f71-f154-6c0f-02ecacf2d840: 2025-02-09T06:52:36.520Z [INFO] server-7f28fa71-9f71-f154-6c0f-02ecacf2d840: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9eb65806-9efa-c938-bc7e-a31eed882685 Address:9eb65806-9efa-c938-bc7e-a31eed882685} {Suffrage:Voter ID:server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd Address:b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd} {Suffrage:Voter ID:server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8 Address:758cf40e-3688-7e45-04ad-ef7e8d7fe5c8} {Suffrage:Voter ID:server-31718b08-787f-39a7-143e-bd02cb427cd2 Address:31718b08-787f-39a7-143e-bd02cb427cd2} {Suffrage:Voter ID:server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef Address:8a069cab-9494-f9fa-96fb-2f80e9ef85ef} {Suffrage:Voter ID:server-7f28fa71-9f71-f154-6c0f-02ecacf2d840 Address:7f28fa71-9f71-f154-6c0f-02ecacf2d840} {Suffrage:Voter ID:server-99c861f7-d592-33f2-7a9e-bccd0bd28952 Address:99c861f7-d592-33f2-7a9e-bccd0bd28952}]" -testLoggerAdapter verbose: server-7f28fa71-9f71-f154-6c0f-02ecacf2d840: 2025-02-09T06:52:36.520Z [INFO] server-7f28fa71-9f71-f154-6c0f-02ecacf2d840: entering follower state: follower="Node at 7f28fa71-9f71-f154-6c0f-02ecacf2d840 [Follower]" leader= - testing.go:149: server-7f28fa71-9f71-f154-6c0f-02ecacf2d840: 2025-02-09T06:52:36.520Z [INFO] server-7f28fa71-9f71-f154-6c0f-02ecacf2d840: entering follower state: follower="Node at 7f28fa71-9f71-f154-6c0f-02ecacf2d840 [Follower]" leader= -testLoggerAdapter verbose: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.520Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9eb65806-9efa-c938-bc7e-a31eed882685 Address:9eb65806-9efa-c938-bc7e-a31eed882685} {Suffrage:Voter ID:server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd Address:b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd} {Suffrage:Voter ID:server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8 Address:758cf40e-3688-7e45-04ad-ef7e8d7fe5c8} {Suffrage:Voter ID:server-31718b08-787f-39a7-143e-bd02cb427cd2 Address:31718b08-787f-39a7-143e-bd02cb427cd2} {Suffrage:Voter ID:server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef Address:8a069cab-9494-f9fa-96fb-2f80e9ef85ef} {Suffrage:Voter ID:server-7f28fa71-9f71-f154-6c0f-02ecacf2d840 Address:7f28fa71-9f71-f154-6c0f-02ecacf2d840} {Suffrage:Voter ID:server-99c861f7-d592-33f2-7a9e-bccd0bd28952 Address:99c861f7-d592-33f2-7a9e-bccd0bd28952}]" - testing.go:149: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.520Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9eb65806-9efa-c938-bc7e-a31eed882685 Address:9eb65806-9efa-c938-bc7e-a31eed882685} {Suffrage:Voter ID:server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd Address:b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd} {Suffrage:Voter ID:server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8 Address:758cf40e-3688-7e45-04ad-ef7e8d7fe5c8} {Suffrage:Voter ID:server-31718b08-787f-39a7-143e-bd02cb427cd2 Address:31718b08-787f-39a7-143e-bd02cb427cd2} {Suffrage:Voter ID:server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef Address:8a069cab-9494-f9fa-96fb-2f80e9ef85ef} {Suffrage:Voter ID:server-7f28fa71-9f71-f154-6c0f-02ecacf2d840 Address:7f28fa71-9f71-f154-6c0f-02ecacf2d840} {Suffrage:Voter ID:server-99c861f7-d592-33f2-7a9e-bccd0bd28952 Address:99c861f7-d592-33f2-7a9e-bccd0bd28952}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:36.520Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:36.520Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.520Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: entering follower state: follower="Node at 99c861f7-d592-33f2-7a9e-bccd0bd28952 [Follower]" leader= - testing.go:149: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.520Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: entering follower state: follower="Node at 99c861f7-d592-33f2-7a9e-bccd0bd28952 [Follower]" leader= -testLoggerAdapter verbose: server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd: 2025-02-09T06:52:36.586Z [WARN] server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.586Z [WARN] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd: 2025-02-09T06:52:36.586Z [WARN] server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd: 2025-02-09T06:52:36.586Z [INFO] server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd: entering candidate state: node="Node at b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd [Candidate]" term=2 - testing.go:149: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.586Z [WARN] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.586Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: entering candidate state: node="Node at 99c861f7-d592-33f2-7a9e-bccd0bd28952 [Candidate]" term=2 - testing.go:149: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.586Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: entering candidate state: node="Node at 99c861f7-d592-33f2-7a9e-bccd0bd28952 [Candidate]" term=2 - testing.go:149: server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd: 2025-02-09T06:52:36.586Z [INFO] server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd: entering candidate state: node="Node at b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd [Candidate]" term=2 -testLoggerAdapter verbose: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.586Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: election won: tally=4 - testing.go:149: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.586Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: election won: tally=4 -testLoggerAdapter verbose: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.586Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: entering leader state: leader="Node at 99c861f7-d592-33f2-7a9e-bccd0bd28952 [Leader]" - testing.go:149: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.586Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: entering leader state: leader="Node at 99c861f7-d592-33f2-7a9e-bccd0bd28952 [Leader]" -testLoggerAdapter verbose: server-31718b08-787f-39a7-143e-bd02cb427cd2: 2025-02-09T06:52:36.586Z [INFO] server-31718b08-787f-39a7-143e-bd02cb427cd2: duplicate requestVote for same term: term=2 -testLoggerAdapter verbose: server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef: 2025-02-09T06:52:36.586Z [INFO] server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef: duplicate requestVote for same term: term=2 -testLoggerAdapter verbose: server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd: 2025-02-09T06:52:36.586Z [INFO] server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd: duplicate requestVote for same term: term=2 -testLoggerAdapter verbose: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.586Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: added peer, starting replication: peer=server-9eb65806-9efa-c938-bc7e-a31eed882685 - testing.go:149: server-31718b08-787f-39a7-143e-bd02cb427cd2: 2025-02-09T06:52:36.586Z [INFO] server-31718b08-787f-39a7-143e-bd02cb427cd2: duplicate requestVote for same term: term=2 -testLoggerAdapter verbose: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.586Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: duplicate requestVote for same term: term=2 -testLoggerAdapter verbose: server-7f28fa71-9f71-f154-6c0f-02ecacf2d840: 2025-02-09T06:52:36.586Z [INFO] server-7f28fa71-9f71-f154-6c0f-02ecacf2d840: duplicate requestVote for same term: term=2 -testLoggerAdapter verbose: server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8: 2025-02-09T06:52:36.586Z [INFO] server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8: duplicate requestVote for same term: term=2 - testing.go:149: server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef: 2025-02-09T06:52:36.586Z [INFO] server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef: duplicate requestVote for same term: term=2 - testing.go:149: server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd: 2025-02-09T06:52:36.586Z [INFO] server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd: duplicate requestVote for same term: term=2 - testing.go:149: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.586Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: added peer, starting replication: peer=server-9eb65806-9efa-c938-bc7e-a31eed882685 -testLoggerAdapter verbose: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.586Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: added peer, starting replication: peer=server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd - testing.go:149: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.586Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: added peer, starting replication: peer=server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd -testLoggerAdapter verbose: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.586Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: added peer, starting replication: peer=server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8 - testing.go:149: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.586Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: added peer, starting replication: peer=server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8 -testLoggerAdapter verbose: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.586Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: added peer, starting replication: peer=server-31718b08-787f-39a7-143e-bd02cb427cd2 - testing.go:149: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.586Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: duplicate requestVote for same term: term=2 -testLoggerAdapter verbose: server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd: 2025-02-09T06:52:36.586Z [INFO] server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd: entering follower state: follower="Node at b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd [Follower]" leader=99c861f7-d592-33f2-7a9e-bccd0bd28952 - testing.go:149: server-7f28fa71-9f71-f154-6c0f-02ecacf2d840: 2025-02-09T06:52:36.586Z [INFO] server-7f28fa71-9f71-f154-6c0f-02ecacf2d840: duplicate requestVote for same term: term=2 - testing.go:149: server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8: 2025-02-09T06:52:36.586Z [INFO] server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8: duplicate requestVote for same term: term=2 - testing.go:149: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.586Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: added peer, starting replication: peer=server-31718b08-787f-39a7-143e-bd02cb427cd2 -testLoggerAdapter verbose: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.586Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: added peer, starting replication: peer=server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef - testing.go:149: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.586Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: added peer, starting replication: peer=server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef -testLoggerAdapter verbose: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.586Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: added peer, starting replication: peer=server-7f28fa71-9f71-f154-6c0f-02ecacf2d840 - testing.go:149: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.586Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: added peer, starting replication: peer=server-7f28fa71-9f71-f154-6c0f-02ecacf2d840 -testLoggerAdapter verbose: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.586Z [WARN] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: rejecting vote request since we have a leader: from=b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd leader=99c861f7-d592-33f2-7a9e-bccd0bd28952 - testing.go:149: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.586Z [WARN] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: rejecting vote request since we have a leader: from=b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd leader=99c861f7-d592-33f2-7a9e-bccd0bd28952 - testing.go:149: server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd: 2025-02-09T06:52:36.586Z [INFO] server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd: entering follower state: follower="Node at b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd [Follower]" leader=99c861f7-d592-33f2-7a9e-bccd0bd28952 -testLoggerAdapter verbose: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.586Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: pipelining replication: peer="{Voter server-9eb65806-9efa-c938-bc7e-a31eed882685 9eb65806-9efa-c938-bc7e-a31eed882685}" - testing.go:149: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.586Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: pipelining replication: peer="{Voter server-9eb65806-9efa-c938-bc7e-a31eed882685 9eb65806-9efa-c938-bc7e-a31eed882685}" -testLoggerAdapter verbose: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.586Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: pipelining replication: peer="{Voter server-31718b08-787f-39a7-143e-bd02cb427cd2 31718b08-787f-39a7-143e-bd02cb427cd2}" - testing.go:149: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.586Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: pipelining replication: peer="{Voter server-31718b08-787f-39a7-143e-bd02cb427cd2 31718b08-787f-39a7-143e-bd02cb427cd2}" -testLoggerAdapter verbose: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.587Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: pipelining replication: peer="{Voter server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef 8a069cab-9494-f9fa-96fb-2f80e9ef85ef}" - testing.go:149: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.587Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: pipelining replication: peer="{Voter server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef 8a069cab-9494-f9fa-96fb-2f80e9ef85ef}" -testLoggerAdapter verbose: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.587Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: pipelining replication: peer="{Voter server-7f28fa71-9f71-f154-6c0f-02ecacf2d840 7f28fa71-9f71-f154-6c0f-02ecacf2d840}" - testing.go:149: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.587Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: pipelining replication: peer="{Voter server-7f28fa71-9f71-f154-6c0f-02ecacf2d840 7f28fa71-9f71-f154-6c0f-02ecacf2d840}" -testLoggerAdapter verbose: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.586Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: pipelining replication: peer="{Voter server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd}" - testing.go:149: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.586Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: pipelining replication: peer="{Voter server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd}" -testLoggerAdapter verbose: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.586Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: pipelining replication: peer="{Voter server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8 758cf40e-3688-7e45-04ad-ef7e8d7fe5c8}" - testing.go:149: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.586Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: pipelining replication: peer="{Voter server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8 758cf40e-3688-7e45-04ad-ef7e8d7fe5c8}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:36.692Z [INFO] stable state for Leader reached at 2025-02-09 06:52:36.587016707 +0000 UTC m=+17.565825471 (1 nodes), 66.567044ms from start of poll, 67.438302ms from cluster start. Timeout at 2025-02-09 06:52:36.692038475 +0000 UTC m=+17.670847232, 105.021761ms after stability - testing.go:149: cluster: 2025-02-09T06:52:36.692Z [INFO] stable state for Leader reached at 2025-02-09 06:52:36.587016707 +0000 UTC m=+17.565825471 (1 nodes), 66.567044ms from start of poll, 67.438302ms from cluster start. Timeout at 2025-02-09 06:52:36.692038475 +0000 UTC m=+17.670847232, 105.021761ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:36.692Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T06:52:36.692Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:36.797Z [INFO] stable state for Follower reached at 2025-02-09 06:52:36.692107714 +0000 UTC m=+17.670916482 (6 nodes), 1.158µs from start of poll, 172.529313ms from cluster start. Timeout at 2025-02-09 06:52:36.797237756 +0000 UTC m=+17.776046513, 105.130031ms after stability - testing.go:149: cluster: 2025-02-09T06:52:36.797Z [INFO] stable state for Follower reached at 2025-02-09 06:52:36.692107714 +0000 UTC m=+17.670916482 (6 nodes), 1.158µs from start of poll, 172.529313ms from cluster start. Timeout at 2025-02-09 06:52:36.797237756 +0000 UTC m=+17.776046513, 105.130031ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:36.797Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:36.797Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:36.902Z [INFO] stable state for Leader reached at 2025-02-09 06:52:36.797301925 +0000 UTC m=+17.776110699 (1 nodes), 520ns from start of poll, 277.72353ms from cluster start. Timeout at 2025-02-09 06:52:36.902450091 +0000 UTC m=+17.881258851, 105.148152ms after stability - testing.go:149: cluster: 2025-02-09T06:52:36.902Z [INFO] stable state for Leader reached at 2025-02-09 06:52:36.797301925 +0000 UTC m=+17.776110699 (1 nodes), 520ns from start of poll, 277.72353ms from cluster start. Timeout at 2025-02-09 06:52:36.902450091 +0000 UTC m=+17.881258851, 105.148152ms after stability -testLoggerAdapter verbose: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.902Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: entering candidate state: node="Node at 9eb65806-9efa-c938-bc7e-a31eed882685 [Candidate]" term=3 - testing.go:149: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.902Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: entering candidate state: node="Node at 9eb65806-9efa-c938-bc7e-a31eed882685 [Candidate]" term=3 -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:36.902Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:36.902Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.902Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: election won: tally=4 -testLoggerAdapter verbose: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.902Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: entering follower state: follower="Node at 99c861f7-d592-33f2-7a9e-bccd0bd28952 [Follower]" leader= - testing.go:149: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.902Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: election won: tally=4 -testLoggerAdapter verbose: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.902Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: entering leader state: leader="Node at 9eb65806-9efa-c938-bc7e-a31eed882685 [Leader]" - testing.go:149: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.902Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: entering leader state: leader="Node at 9eb65806-9efa-c938-bc7e-a31eed882685 [Leader]" -testLoggerAdapter verbose: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.902Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: added peer, starting replication: peer=server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd - testing.go:149: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.902Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: added peer, starting replication: peer=server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd -testLoggerAdapter verbose: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.902Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: added peer, starting replication: peer=server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8 - testing.go:149: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.902Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: added peer, starting replication: peer=server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8 -testLoggerAdapter verbose: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.902Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: added peer, starting replication: peer=server-31718b08-787f-39a7-143e-bd02cb427cd2 - testing.go:149: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.902Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: added peer, starting replication: peer=server-31718b08-787f-39a7-143e-bd02cb427cd2 -testLoggerAdapter verbose: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.902Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: pipelining replication: peer="{Voter server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd}" - testing.go:149: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.902Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: pipelining replication: peer="{Voter server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd}" -testLoggerAdapter verbose: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.902Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: added peer, starting replication: peer=server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef - testing.go:149: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.902Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: entering follower state: follower="Node at 99c861f7-d592-33f2-7a9e-bccd0bd28952 [Follower]" leader= -testLoggerAdapter verbose: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.902Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: aborting pipeline replication: peer="{Voter server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8 758cf40e-3688-7e45-04ad-ef7e8d7fe5c8}" - testing.go:149: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.902Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: aborting pipeline replication: peer="{Voter server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8 758cf40e-3688-7e45-04ad-ef7e8d7fe5c8}" - testing.go:149: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.902Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: added peer, starting replication: peer=server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef -testLoggerAdapter verbose: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.902Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: added peer, starting replication: peer=server-7f28fa71-9f71-f154-6c0f-02ecacf2d840 - testing.go:149: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.902Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: added peer, starting replication: peer=server-7f28fa71-9f71-f154-6c0f-02ecacf2d840 -testLoggerAdapter verbose: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.902Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: added peer, starting replication: peer=server-99c861f7-d592-33f2-7a9e-bccd0bd28952 - testing.go:149: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.902Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: added peer, starting replication: peer=server-99c861f7-d592-33f2-7a9e-bccd0bd28952 -testLoggerAdapter verbose: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.902Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: aborting pipeline replication: peer="{Voter server-31718b08-787f-39a7-143e-bd02cb427cd2 31718b08-787f-39a7-143e-bd02cb427cd2}" - testing.go:149: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.902Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: aborting pipeline replication: peer="{Voter server-31718b08-787f-39a7-143e-bd02cb427cd2 31718b08-787f-39a7-143e-bd02cb427cd2}" -testLoggerAdapter verbose: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.902Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: pipelining replication: peer="{Voter server-31718b08-787f-39a7-143e-bd02cb427cd2 31718b08-787f-39a7-143e-bd02cb427cd2}" - testing.go:149: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.902Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: pipelining replication: peer="{Voter server-31718b08-787f-39a7-143e-bd02cb427cd2 31718b08-787f-39a7-143e-bd02cb427cd2}" -testLoggerAdapter verbose: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.902Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: pipelining replication: peer="{Voter server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef 8a069cab-9494-f9fa-96fb-2f80e9ef85ef}" - testing.go:149: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.902Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: pipelining replication: peer="{Voter server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef 8a069cab-9494-f9fa-96fb-2f80e9ef85ef}" -testLoggerAdapter verbose: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.902Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: pipelining replication: peer="{Voter server-7f28fa71-9f71-f154-6c0f-02ecacf2d840 7f28fa71-9f71-f154-6c0f-02ecacf2d840}" - testing.go:149: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.902Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: pipelining replication: peer="{Voter server-7f28fa71-9f71-f154-6c0f-02ecacf2d840 7f28fa71-9f71-f154-6c0f-02ecacf2d840}" -testLoggerAdapter verbose: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.903Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: pipelining replication: peer="{Voter server-99c861f7-d592-33f2-7a9e-bccd0bd28952 99c861f7-d592-33f2-7a9e-bccd0bd28952}" - testing.go:149: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.903Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: pipelining replication: peer="{Voter server-99c861f7-d592-33f2-7a9e-bccd0bd28952 99c861f7-d592-33f2-7a9e-bccd0bd28952}" -testLoggerAdapter verbose: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.902Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: aborting pipeline replication: peer="{Voter server-9eb65806-9efa-c938-bc7e-a31eed882685 9eb65806-9efa-c938-bc7e-a31eed882685}" - testing.go:149: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.902Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: aborting pipeline replication: peer="{Voter server-9eb65806-9efa-c938-bc7e-a31eed882685 9eb65806-9efa-c938-bc7e-a31eed882685}" -testLoggerAdapter verbose: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.902Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: pipelining replication: peer="{Voter server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8 758cf40e-3688-7e45-04ad-ef7e8d7fe5c8}" - testing.go:149: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:36.902Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: pipelining replication: peer="{Voter server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8 758cf40e-3688-7e45-04ad-ef7e8d7fe5c8}" -testLoggerAdapter verbose: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.902Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: aborting pipeline replication: peer="{Voter server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef 8a069cab-9494-f9fa-96fb-2f80e9ef85ef}" - testing.go:149: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.902Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: aborting pipeline replication: peer="{Voter server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef 8a069cab-9494-f9fa-96fb-2f80e9ef85ef}" -testLoggerAdapter verbose: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.902Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: aborting pipeline replication: peer="{Voter server-7f28fa71-9f71-f154-6c0f-02ecacf2d840 7f28fa71-9f71-f154-6c0f-02ecacf2d840}" - testing.go:149: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.902Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: aborting pipeline replication: peer="{Voter server-7f28fa71-9f71-f154-6c0f-02ecacf2d840 7f28fa71-9f71-f154-6c0f-02ecacf2d840}" -testLoggerAdapter verbose: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.902Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: aborting pipeline replication: peer="{Voter server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd}" - testing.go:149: server-99c861f7-d592-33f2-7a9e-bccd0bd28952: 2025-02-09T06:52:36.902Z [INFO] server-99c861f7-d592-33f2-7a9e-bccd0bd28952: aborting pipeline replication: peer="{Voter server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:37.007Z [INFO] stable state for Leader reached at 2025-02-09 06:52:36.902673379 +0000 UTC m=+17.881482158 (1 nodes), 82.949µs from start of poll, 383.094989ms from cluster start. Timeout at 2025-02-09 06:52:37.007835369 +0000 UTC m=+17.986644136, 105.161978ms after stability - testing.go:149: cluster: 2025-02-09T06:52:37.007Z [INFO] stable state for Leader reached at 2025-02-09 06:52:36.902673379 +0000 UTC m=+17.881482158 (1 nodes), 82.949µs from start of poll, 383.094989ms from cluster start. Timeout at 2025-02-09 06:52:37.007835369 +0000 UTC m=+17.986644136, 105.161978ms after stability -testLoggerAdapter verbose: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:37.007Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: aborting pipeline replication: peer="{Voter server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8 758cf40e-3688-7e45-04ad-ef7e8d7fe5c8}" - testing.go:149: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:37.007Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: aborting pipeline replication: peer="{Voter server-758cf40e-3688-7e45-04ad-ef7e8d7fe5c8 758cf40e-3688-7e45-04ad-ef7e8d7fe5c8}" -testLoggerAdapter verbose: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:37.007Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: aborting pipeline replication: peer="{Voter server-31718b08-787f-39a7-143e-bd02cb427cd2 31718b08-787f-39a7-143e-bd02cb427cd2}" - testing.go:149: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:37.007Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: aborting pipeline replication: peer="{Voter server-31718b08-787f-39a7-143e-bd02cb427cd2 31718b08-787f-39a7-143e-bd02cb427cd2}" -testLoggerAdapter verbose: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:37.007Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: aborting pipeline replication: peer="{Voter server-7f28fa71-9f71-f154-6c0f-02ecacf2d840 7f28fa71-9f71-f154-6c0f-02ecacf2d840}" - testing.go:149: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:37.007Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: aborting pipeline replication: peer="{Voter server-7f28fa71-9f71-f154-6c0f-02ecacf2d840 7f28fa71-9f71-f154-6c0f-02ecacf2d840}" -testLoggerAdapter verbose: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:37.007Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: aborting pipeline replication: peer="{Voter server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef 8a069cab-9494-f9fa-96fb-2f80e9ef85ef}" - testing.go:149: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:37.007Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: aborting pipeline replication: peer="{Voter server-8a069cab-9494-f9fa-96fb-2f80e9ef85ef 8a069cab-9494-f9fa-96fb-2f80e9ef85ef}" -testLoggerAdapter verbose: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:37.007Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: aborting pipeline replication: peer="{Voter server-99c861f7-d592-33f2-7a9e-bccd0bd28952 99c861f7-d592-33f2-7a9e-bccd0bd28952}" - testing.go:149: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:37.007Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: aborting pipeline replication: peer="{Voter server-99c861f7-d592-33f2-7a9e-bccd0bd28952 99c861f7-d592-33f2-7a9e-bccd0bd28952}" -testLoggerAdapter verbose: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:37.007Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: aborting pipeline replication: peer="{Voter server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd}" - testing.go:149: server-9eb65806-9efa-c938-bc7e-a31eed882685: 2025-02-09T06:52:37.007Z [INFO] server-9eb65806-9efa-c938-bc7e-a31eed882685: aborting pipeline replication: peer="{Voter server-b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd b73c3e1e-8f7f-2fb5-0eb8-199764cbf5bd}" +testLoggerAdapter verbose: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:11.980Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-164b7adb-6988-0e3c-a0e4-f6962a583cb3 Address:164b7adb-6988-0e3c-a0e4-f6962a583cb3} {Suffrage:Voter ID:server-98bf8875-a96c-1ab1-046c-136db591cd9d Address:98bf8875-a96c-1ab1-046c-136db591cd9d} {Suffrage:Voter ID:server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8 Address:28ff6f15-37c2-bfde-bfce-97bb80b2cac8} {Suffrage:Voter ID:server-d6eecb40-44a2-80ed-0099-f25daa7cb905 Address:d6eecb40-44a2-80ed-0099-f25daa7cb905} {Suffrage:Voter ID:server-674c7145-0e20-c026-98a5-1bbf859cf227 Address:674c7145-0e20-c026-98a5-1bbf859cf227} {Suffrage:Voter ID:server-e0ed200a-903b-96cd-1145-6098b46ee1cd Address:e0ed200a-903b-96cd-1145-6098b46ee1cd} {Suffrage:Voter ID:server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6 Address:7f4593f0-d11d-5c51-be28-fb05ca53dfd6}]" + testing.go:149: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:11.980Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-164b7adb-6988-0e3c-a0e4-f6962a583cb3 Address:164b7adb-6988-0e3c-a0e4-f6962a583cb3} {Suffrage:Voter ID:server-98bf8875-a96c-1ab1-046c-136db591cd9d Address:98bf8875-a96c-1ab1-046c-136db591cd9d} {Suffrage:Voter ID:server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8 Address:28ff6f15-37c2-bfde-bfce-97bb80b2cac8} {Suffrage:Voter ID:server-d6eecb40-44a2-80ed-0099-f25daa7cb905 Address:d6eecb40-44a2-80ed-0099-f25daa7cb905} {Suffrage:Voter ID:server-674c7145-0e20-c026-98a5-1bbf859cf227 Address:674c7145-0e20-c026-98a5-1bbf859cf227} {Suffrage:Voter ID:server-e0ed200a-903b-96cd-1145-6098b46ee1cd Address:e0ed200a-903b-96cd-1145-6098b46ee1cd} {Suffrage:Voter ID:server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6 Address:7f4593f0-d11d-5c51-be28-fb05ca53dfd6}]" +testLoggerAdapter verbose: server-98bf8875-a96c-1ab1-046c-136db591cd9d: 2024-01-08T00:31:11.981Z [INFO] server-98bf8875-a96c-1ab1-046c-136db591cd9d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-164b7adb-6988-0e3c-a0e4-f6962a583cb3 Address:164b7adb-6988-0e3c-a0e4-f6962a583cb3} {Suffrage:Voter ID:server-98bf8875-a96c-1ab1-046c-136db591cd9d Address:98bf8875-a96c-1ab1-046c-136db591cd9d} {Suffrage:Voter ID:server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8 Address:28ff6f15-37c2-bfde-bfce-97bb80b2cac8} {Suffrage:Voter ID:server-d6eecb40-44a2-80ed-0099-f25daa7cb905 Address:d6eecb40-44a2-80ed-0099-f25daa7cb905} {Suffrage:Voter ID:server-674c7145-0e20-c026-98a5-1bbf859cf227 Address:674c7145-0e20-c026-98a5-1bbf859cf227} {Suffrage:Voter ID:server-e0ed200a-903b-96cd-1145-6098b46ee1cd Address:e0ed200a-903b-96cd-1145-6098b46ee1cd} {Suffrage:Voter ID:server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6 Address:7f4593f0-d11d-5c51-be28-fb05ca53dfd6}]" + testing.go:149: server-98bf8875-a96c-1ab1-046c-136db591cd9d: 2024-01-08T00:31:11.981Z [INFO] server-98bf8875-a96c-1ab1-046c-136db591cd9d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-164b7adb-6988-0e3c-a0e4-f6962a583cb3 Address:164b7adb-6988-0e3c-a0e4-f6962a583cb3} {Suffrage:Voter ID:server-98bf8875-a96c-1ab1-046c-136db591cd9d Address:98bf8875-a96c-1ab1-046c-136db591cd9d} {Suffrage:Voter ID:server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8 Address:28ff6f15-37c2-bfde-bfce-97bb80b2cac8} {Suffrage:Voter ID:server-d6eecb40-44a2-80ed-0099-f25daa7cb905 Address:d6eecb40-44a2-80ed-0099-f25daa7cb905} {Suffrage:Voter ID:server-674c7145-0e20-c026-98a5-1bbf859cf227 Address:674c7145-0e20-c026-98a5-1bbf859cf227} {Suffrage:Voter ID:server-e0ed200a-903b-96cd-1145-6098b46ee1cd Address:e0ed200a-903b-96cd-1145-6098b46ee1cd} {Suffrage:Voter ID:server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6 Address:7f4593f0-d11d-5c51-be28-fb05ca53dfd6}]" +testLoggerAdapter verbose: server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8: 2024-01-08T00:31:11.981Z [INFO] server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-164b7adb-6988-0e3c-a0e4-f6962a583cb3 Address:164b7adb-6988-0e3c-a0e4-f6962a583cb3} {Suffrage:Voter ID:server-98bf8875-a96c-1ab1-046c-136db591cd9d Address:98bf8875-a96c-1ab1-046c-136db591cd9d} {Suffrage:Voter ID:server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8 Address:28ff6f15-37c2-bfde-bfce-97bb80b2cac8} {Suffrage:Voter ID:server-d6eecb40-44a2-80ed-0099-f25daa7cb905 Address:d6eecb40-44a2-80ed-0099-f25daa7cb905} {Suffrage:Voter ID:server-674c7145-0e20-c026-98a5-1bbf859cf227 Address:674c7145-0e20-c026-98a5-1bbf859cf227} {Suffrage:Voter ID:server-e0ed200a-903b-96cd-1145-6098b46ee1cd Address:e0ed200a-903b-96cd-1145-6098b46ee1cd} {Suffrage:Voter ID:server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6 Address:7f4593f0-d11d-5c51-be28-fb05ca53dfd6}]" + testing.go:149: server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8: 2024-01-08T00:31:11.981Z [INFO] server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-164b7adb-6988-0e3c-a0e4-f6962a583cb3 Address:164b7adb-6988-0e3c-a0e4-f6962a583cb3} {Suffrage:Voter ID:server-98bf8875-a96c-1ab1-046c-136db591cd9d Address:98bf8875-a96c-1ab1-046c-136db591cd9d} {Suffrage:Voter ID:server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8 Address:28ff6f15-37c2-bfde-bfce-97bb80b2cac8} {Suffrage:Voter ID:server-d6eecb40-44a2-80ed-0099-f25daa7cb905 Address:d6eecb40-44a2-80ed-0099-f25daa7cb905} {Suffrage:Voter ID:server-674c7145-0e20-c026-98a5-1bbf859cf227 Address:674c7145-0e20-c026-98a5-1bbf859cf227} {Suffrage:Voter ID:server-e0ed200a-903b-96cd-1145-6098b46ee1cd Address:e0ed200a-903b-96cd-1145-6098b46ee1cd} {Suffrage:Voter ID:server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6 Address:7f4593f0-d11d-5c51-be28-fb05ca53dfd6}]" +testLoggerAdapter verbose: server-d6eecb40-44a2-80ed-0099-f25daa7cb905: 2024-01-08T00:31:11.981Z [INFO] server-d6eecb40-44a2-80ed-0099-f25daa7cb905: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-164b7adb-6988-0e3c-a0e4-f6962a583cb3 Address:164b7adb-6988-0e3c-a0e4-f6962a583cb3} {Suffrage:Voter ID:server-98bf8875-a96c-1ab1-046c-136db591cd9d Address:98bf8875-a96c-1ab1-046c-136db591cd9d} {Suffrage:Voter ID:server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8 Address:28ff6f15-37c2-bfde-bfce-97bb80b2cac8} {Suffrage:Voter ID:server-d6eecb40-44a2-80ed-0099-f25daa7cb905 Address:d6eecb40-44a2-80ed-0099-f25daa7cb905} {Suffrage:Voter ID:server-674c7145-0e20-c026-98a5-1bbf859cf227 Address:674c7145-0e20-c026-98a5-1bbf859cf227} {Suffrage:Voter ID:server-e0ed200a-903b-96cd-1145-6098b46ee1cd Address:e0ed200a-903b-96cd-1145-6098b46ee1cd} {Suffrage:Voter ID:server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6 Address:7f4593f0-d11d-5c51-be28-fb05ca53dfd6}]" + testing.go:149: server-d6eecb40-44a2-80ed-0099-f25daa7cb905: 2024-01-08T00:31:11.981Z [INFO] server-d6eecb40-44a2-80ed-0099-f25daa7cb905: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-164b7adb-6988-0e3c-a0e4-f6962a583cb3 Address:164b7adb-6988-0e3c-a0e4-f6962a583cb3} {Suffrage:Voter ID:server-98bf8875-a96c-1ab1-046c-136db591cd9d Address:98bf8875-a96c-1ab1-046c-136db591cd9d} {Suffrage:Voter ID:server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8 Address:28ff6f15-37c2-bfde-bfce-97bb80b2cac8} {Suffrage:Voter ID:server-d6eecb40-44a2-80ed-0099-f25daa7cb905 Address:d6eecb40-44a2-80ed-0099-f25daa7cb905} {Suffrage:Voter ID:server-674c7145-0e20-c026-98a5-1bbf859cf227 Address:674c7145-0e20-c026-98a5-1bbf859cf227} {Suffrage:Voter ID:server-e0ed200a-903b-96cd-1145-6098b46ee1cd Address:e0ed200a-903b-96cd-1145-6098b46ee1cd} {Suffrage:Voter ID:server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6 Address:7f4593f0-d11d-5c51-be28-fb05ca53dfd6}]" +testLoggerAdapter verbose: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:11.981Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-164b7adb-6988-0e3c-a0e4-f6962a583cb3 Address:164b7adb-6988-0e3c-a0e4-f6962a583cb3} {Suffrage:Voter ID:server-98bf8875-a96c-1ab1-046c-136db591cd9d Address:98bf8875-a96c-1ab1-046c-136db591cd9d} {Suffrage:Voter ID:server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8 Address:28ff6f15-37c2-bfde-bfce-97bb80b2cac8} {Suffrage:Voter ID:server-d6eecb40-44a2-80ed-0099-f25daa7cb905 Address:d6eecb40-44a2-80ed-0099-f25daa7cb905} {Suffrage:Voter ID:server-674c7145-0e20-c026-98a5-1bbf859cf227 Address:674c7145-0e20-c026-98a5-1bbf859cf227} {Suffrage:Voter ID:server-e0ed200a-903b-96cd-1145-6098b46ee1cd Address:e0ed200a-903b-96cd-1145-6098b46ee1cd} {Suffrage:Voter ID:server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6 Address:7f4593f0-d11d-5c51-be28-fb05ca53dfd6}]" + testing.go:149: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:11.981Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-164b7adb-6988-0e3c-a0e4-f6962a583cb3 Address:164b7adb-6988-0e3c-a0e4-f6962a583cb3} {Suffrage:Voter ID:server-98bf8875-a96c-1ab1-046c-136db591cd9d Address:98bf8875-a96c-1ab1-046c-136db591cd9d} {Suffrage:Voter ID:server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8 Address:28ff6f15-37c2-bfde-bfce-97bb80b2cac8} {Suffrage:Voter ID:server-d6eecb40-44a2-80ed-0099-f25daa7cb905 Address:d6eecb40-44a2-80ed-0099-f25daa7cb905} {Suffrage:Voter ID:server-674c7145-0e20-c026-98a5-1bbf859cf227 Address:674c7145-0e20-c026-98a5-1bbf859cf227} {Suffrage:Voter ID:server-e0ed200a-903b-96cd-1145-6098b46ee1cd Address:e0ed200a-903b-96cd-1145-6098b46ee1cd} {Suffrage:Voter ID:server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6 Address:7f4593f0-d11d-5c51-be28-fb05ca53dfd6}]" +testLoggerAdapter verbose: server-e0ed200a-903b-96cd-1145-6098b46ee1cd: 2024-01-08T00:31:11.981Z [INFO] server-e0ed200a-903b-96cd-1145-6098b46ee1cd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-164b7adb-6988-0e3c-a0e4-f6962a583cb3 Address:164b7adb-6988-0e3c-a0e4-f6962a583cb3} {Suffrage:Voter ID:server-98bf8875-a96c-1ab1-046c-136db591cd9d Address:98bf8875-a96c-1ab1-046c-136db591cd9d} {Suffrage:Voter ID:server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8 Address:28ff6f15-37c2-bfde-bfce-97bb80b2cac8} {Suffrage:Voter ID:server-d6eecb40-44a2-80ed-0099-f25daa7cb905 Address:d6eecb40-44a2-80ed-0099-f25daa7cb905} {Suffrage:Voter ID:server-674c7145-0e20-c026-98a5-1bbf859cf227 Address:674c7145-0e20-c026-98a5-1bbf859cf227} {Suffrage:Voter ID:server-e0ed200a-903b-96cd-1145-6098b46ee1cd Address:e0ed200a-903b-96cd-1145-6098b46ee1cd} {Suffrage:Voter ID:server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6 Address:7f4593f0-d11d-5c51-be28-fb05ca53dfd6}]" + testing.go:149: server-e0ed200a-903b-96cd-1145-6098b46ee1cd: 2024-01-08T00:31:11.981Z [INFO] server-e0ed200a-903b-96cd-1145-6098b46ee1cd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-164b7adb-6988-0e3c-a0e4-f6962a583cb3 Address:164b7adb-6988-0e3c-a0e4-f6962a583cb3} {Suffrage:Voter ID:server-98bf8875-a96c-1ab1-046c-136db591cd9d Address:98bf8875-a96c-1ab1-046c-136db591cd9d} {Suffrage:Voter ID:server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8 Address:28ff6f15-37c2-bfde-bfce-97bb80b2cac8} {Suffrage:Voter ID:server-d6eecb40-44a2-80ed-0099-f25daa7cb905 Address:d6eecb40-44a2-80ed-0099-f25daa7cb905} {Suffrage:Voter ID:server-674c7145-0e20-c026-98a5-1bbf859cf227 Address:674c7145-0e20-c026-98a5-1bbf859cf227} {Suffrage:Voter ID:server-e0ed200a-903b-96cd-1145-6098b46ee1cd Address:e0ed200a-903b-96cd-1145-6098b46ee1cd} {Suffrage:Voter ID:server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6 Address:7f4593f0-d11d-5c51-be28-fb05ca53dfd6}]" +testLoggerAdapter verbose: server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6: 2024-01-08T00:31:11.982Z [INFO] server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-164b7adb-6988-0e3c-a0e4-f6962a583cb3 Address:164b7adb-6988-0e3c-a0e4-f6962a583cb3} {Suffrage:Voter ID:server-98bf8875-a96c-1ab1-046c-136db591cd9d Address:98bf8875-a96c-1ab1-046c-136db591cd9d} {Suffrage:Voter ID:server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8 Address:28ff6f15-37c2-bfde-bfce-97bb80b2cac8} {Suffrage:Voter ID:server-d6eecb40-44a2-80ed-0099-f25daa7cb905 Address:d6eecb40-44a2-80ed-0099-f25daa7cb905} {Suffrage:Voter ID:server-674c7145-0e20-c026-98a5-1bbf859cf227 Address:674c7145-0e20-c026-98a5-1bbf859cf227} {Suffrage:Voter ID:server-e0ed200a-903b-96cd-1145-6098b46ee1cd Address:e0ed200a-903b-96cd-1145-6098b46ee1cd} {Suffrage:Voter ID:server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6 Address:7f4593f0-d11d-5c51-be28-fb05ca53dfd6}]" + testing.go:149: server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6: 2024-01-08T00:31:11.982Z [INFO] server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-164b7adb-6988-0e3c-a0e4-f6962a583cb3 Address:164b7adb-6988-0e3c-a0e4-f6962a583cb3} {Suffrage:Voter ID:server-98bf8875-a96c-1ab1-046c-136db591cd9d Address:98bf8875-a96c-1ab1-046c-136db591cd9d} {Suffrage:Voter ID:server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8 Address:28ff6f15-37c2-bfde-bfce-97bb80b2cac8} {Suffrage:Voter ID:server-d6eecb40-44a2-80ed-0099-f25daa7cb905 Address:d6eecb40-44a2-80ed-0099-f25daa7cb905} {Suffrage:Voter ID:server-674c7145-0e20-c026-98a5-1bbf859cf227 Address:674c7145-0e20-c026-98a5-1bbf859cf227} {Suffrage:Voter ID:server-e0ed200a-903b-96cd-1145-6098b46ee1cd Address:e0ed200a-903b-96cd-1145-6098b46ee1cd} {Suffrage:Voter ID:server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6 Address:7f4593f0-d11d-5c51-be28-fb05ca53dfd6}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:11.982Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:11.982Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:11.982Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: entering follower state: follower="Node at 164b7adb-6988-0e3c-a0e4-f6962a583cb3 [Follower]" leader= + testing.go:149: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:11.982Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: entering follower state: follower="Node at 164b7adb-6988-0e3c-a0e4-f6962a583cb3 [Follower]" leader= +testLoggerAdapter verbose: server-98bf8875-a96c-1ab1-046c-136db591cd9d: 2024-01-08T00:31:11.982Z [INFO] server-98bf8875-a96c-1ab1-046c-136db591cd9d: entering follower state: follower="Node at 98bf8875-a96c-1ab1-046c-136db591cd9d [Follower]" leader= + testing.go:149: server-98bf8875-a96c-1ab1-046c-136db591cd9d: 2024-01-08T00:31:11.982Z [INFO] server-98bf8875-a96c-1ab1-046c-136db591cd9d: entering follower state: follower="Node at 98bf8875-a96c-1ab1-046c-136db591cd9d [Follower]" leader= +testLoggerAdapter verbose: server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8: 2024-01-08T00:31:11.982Z [INFO] server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8: entering follower state: follower="Node at 28ff6f15-37c2-bfde-bfce-97bb80b2cac8 [Follower]" leader= + testing.go:149: server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8: 2024-01-08T00:31:11.982Z [INFO] server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8: entering follower state: follower="Node at 28ff6f15-37c2-bfde-bfce-97bb80b2cac8 [Follower]" leader= +testLoggerAdapter verbose: server-d6eecb40-44a2-80ed-0099-f25daa7cb905: 2024-01-08T00:31:11.982Z [INFO] server-d6eecb40-44a2-80ed-0099-f25daa7cb905: entering follower state: follower="Node at d6eecb40-44a2-80ed-0099-f25daa7cb905 [Follower]" leader= + testing.go:149: server-d6eecb40-44a2-80ed-0099-f25daa7cb905: 2024-01-08T00:31:11.982Z [INFO] server-d6eecb40-44a2-80ed-0099-f25daa7cb905: entering follower state: follower="Node at d6eecb40-44a2-80ed-0099-f25daa7cb905 [Follower]" leader= +testLoggerAdapter verbose: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:11.982Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: entering follower state: follower="Node at 674c7145-0e20-c026-98a5-1bbf859cf227 [Follower]" leader= + testing.go:149: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:11.982Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: entering follower state: follower="Node at 674c7145-0e20-c026-98a5-1bbf859cf227 [Follower]" leader= +testLoggerAdapter verbose: server-e0ed200a-903b-96cd-1145-6098b46ee1cd: 2024-01-08T00:31:11.982Z [INFO] server-e0ed200a-903b-96cd-1145-6098b46ee1cd: entering follower state: follower="Node at e0ed200a-903b-96cd-1145-6098b46ee1cd [Follower]" leader= + testing.go:149: server-e0ed200a-903b-96cd-1145-6098b46ee1cd: 2024-01-08T00:31:11.982Z [INFO] server-e0ed200a-903b-96cd-1145-6098b46ee1cd: entering follower state: follower="Node at e0ed200a-903b-96cd-1145-6098b46ee1cd [Follower]" leader= +testLoggerAdapter verbose: server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6: 2024-01-08T00:31:11.982Z [INFO] server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6: entering follower state: follower="Node at 7f4593f0-d11d-5c51-be28-fb05ca53dfd6 [Follower]" leader= + testing.go:149: server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6: 2024-01-08T00:31:11.982Z [INFO] server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6: entering follower state: follower="Node at 7f4593f0-d11d-5c51-be28-fb05ca53dfd6 [Follower]" leader= +testLoggerAdapter verbose: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.039Z [WARN] server-674c7145-0e20-c026-98a5-1bbf859cf227: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.039Z [WARN] server-674c7145-0e20-c026-98a5-1bbf859cf227: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.039Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: entering candidate state: node="Node at 674c7145-0e20-c026-98a5-1bbf859cf227 [Candidate]" term=2 + testing.go:149: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.039Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: entering candidate state: node="Node at 674c7145-0e20-c026-98a5-1bbf859cf227 [Candidate]" term=2 +testLoggerAdapter verbose: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.040Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: election won: tally=4 + testing.go:149: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.040Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: election won: tally=4 +testLoggerAdapter verbose: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.040Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: entering leader state: leader="Node at 674c7145-0e20-c026-98a5-1bbf859cf227 [Leader]" + testing.go:149: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.040Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: entering leader state: leader="Node at 674c7145-0e20-c026-98a5-1bbf859cf227 [Leader]" +testLoggerAdapter verbose: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.040Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: added peer, starting replication: peer=server-164b7adb-6988-0e3c-a0e4-f6962a583cb3 + testing.go:149: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.040Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: added peer, starting replication: peer=server-164b7adb-6988-0e3c-a0e4-f6962a583cb3 +testLoggerAdapter verbose: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.040Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: added peer, starting replication: peer=server-98bf8875-a96c-1ab1-046c-136db591cd9d + testing.go:149: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.040Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: added peer, starting replication: peer=server-98bf8875-a96c-1ab1-046c-136db591cd9d +testLoggerAdapter verbose: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.040Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: added peer, starting replication: peer=server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8 + testing.go:149: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.040Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: added peer, starting replication: peer=server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8 +testLoggerAdapter verbose: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.040Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: added peer, starting replication: peer=server-d6eecb40-44a2-80ed-0099-f25daa7cb905 + testing.go:149: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.040Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: added peer, starting replication: peer=server-d6eecb40-44a2-80ed-0099-f25daa7cb905 +testLoggerAdapter verbose: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.040Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: added peer, starting replication: peer=server-e0ed200a-903b-96cd-1145-6098b46ee1cd + testing.go:149: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.040Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: added peer, starting replication: peer=server-e0ed200a-903b-96cd-1145-6098b46ee1cd +testLoggerAdapter verbose: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.040Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: added peer, starting replication: peer=server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6 + testing.go:149: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.040Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: added peer, starting replication: peer=server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6 +testLoggerAdapter verbose: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.040Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: pipelining replication: peer="{Voter server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6 7f4593f0-d11d-5c51-be28-fb05ca53dfd6}" + testing.go:149: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.040Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: pipelining replication: peer="{Voter server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6 7f4593f0-d11d-5c51-be28-fb05ca53dfd6}" +testLoggerAdapter verbose: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.040Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: pipelining replication: peer="{Voter server-164b7adb-6988-0e3c-a0e4-f6962a583cb3 164b7adb-6988-0e3c-a0e4-f6962a583cb3}" + testing.go:149: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.040Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: pipelining replication: peer="{Voter server-164b7adb-6988-0e3c-a0e4-f6962a583cb3 164b7adb-6988-0e3c-a0e4-f6962a583cb3}" +testLoggerAdapter verbose: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.040Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: pipelining replication: peer="{Voter server-98bf8875-a96c-1ab1-046c-136db591cd9d 98bf8875-a96c-1ab1-046c-136db591cd9d}" + testing.go:149: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.040Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: pipelining replication: peer="{Voter server-98bf8875-a96c-1ab1-046c-136db591cd9d 98bf8875-a96c-1ab1-046c-136db591cd9d}" +testLoggerAdapter verbose: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.041Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: pipelining replication: peer="{Voter server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8 28ff6f15-37c2-bfde-bfce-97bb80b2cac8}" + testing.go:149: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.041Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: pipelining replication: peer="{Voter server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8 28ff6f15-37c2-bfde-bfce-97bb80b2cac8}" +testLoggerAdapter verbose: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.041Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: pipelining replication: peer="{Voter server-d6eecb40-44a2-80ed-0099-f25daa7cb905 d6eecb40-44a2-80ed-0099-f25daa7cb905}" + testing.go:149: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.041Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: pipelining replication: peer="{Voter server-d6eecb40-44a2-80ed-0099-f25daa7cb905 d6eecb40-44a2-80ed-0099-f25daa7cb905}" +testLoggerAdapter verbose: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.041Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: pipelining replication: peer="{Voter server-e0ed200a-903b-96cd-1145-6098b46ee1cd e0ed200a-903b-96cd-1145-6098b46ee1cd}" + testing.go:149: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.041Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: pipelining replication: peer="{Voter server-e0ed200a-903b-96cd-1145-6098b46ee1cd e0ed200a-903b-96cd-1145-6098b46ee1cd}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:12.146Z [INFO] stable state for Leader reached at 2024-01-08 00:31:12.04082743 +0000 UTC m=+18.087716628 (1 nodes), 58.589005ms from start of poll, 60.255995ms from cluster start. Timeout at 2024-01-08 00:31:12.146464813 +0000 UTC m=+18.193353987, 105.637359ms after stability + testing.go:149: cluster: 2024-01-08T00:31:12.146Z [INFO] stable state for Leader reached at 2024-01-08 00:31:12.04082743 +0000 UTC m=+18.087716628 (1 nodes), 58.589005ms from start of poll, 60.255995ms from cluster start. Timeout at 2024-01-08 00:31:12.146464813 +0000 UTC m=+18.193353987, 105.637359ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:12.146Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T00:31:12.146Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:12.252Z [INFO] stable state for Follower reached at 2024-01-08 00:31:12.146620129 +0000 UTC m=+18.193509341 (6 nodes), 3.66µs from start of poll, 166.048708ms from cluster start. Timeout at 2024-01-08 00:31:12.252323392 +0000 UTC m=+18.299212528, 105.703187ms after stability + testing.go:149: cluster: 2024-01-08T00:31:12.252Z [INFO] stable state for Follower reached at 2024-01-08 00:31:12.146620129 +0000 UTC m=+18.193509341 (6 nodes), 3.66µs from start of poll, 166.048708ms from cluster start. Timeout at 2024-01-08 00:31:12.252323392 +0000 UTC m=+18.299212528, 105.703187ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:12.252Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:12.252Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:12.359Z [INFO] stable state for Leader reached at 2024-01-08 00:31:12.253128115 +0000 UTC m=+18.300021103 (1 nodes), 11.874µs from start of poll, 272.56047ms from cluster start. Timeout at 2024-01-08 00:31:12.35916677 +0000 UTC m=+18.406055824, 106.034721ms after stability + testing.go:149: cluster: 2024-01-08T00:31:12.359Z [INFO] stable state for Leader reached at 2024-01-08 00:31:12.253128115 +0000 UTC m=+18.300021103 (1 nodes), 11.874µs from start of poll, 272.56047ms from cluster start. Timeout at 2024-01-08 00:31:12.35916677 +0000 UTC m=+18.406055824, 106.034721ms after stability +testLoggerAdapter verbose: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.359Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: entering candidate state: node="Node at 164b7adb-6988-0e3c-a0e4-f6962a583cb3 [Candidate]" term=3 + testing.go:149: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.359Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: entering candidate state: node="Node at 164b7adb-6988-0e3c-a0e4-f6962a583cb3 [Candidate]" term=3 +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:12.359Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:12.359Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.359Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: election won: tally=4 + testing.go:149: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.359Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: election won: tally=4 +testLoggerAdapter verbose: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.359Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: entering leader state: leader="Node at 164b7adb-6988-0e3c-a0e4-f6962a583cb3 [Leader]" + testing.go:149: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.359Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: entering leader state: leader="Node at 164b7adb-6988-0e3c-a0e4-f6962a583cb3 [Leader]" +testLoggerAdapter verbose: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.359Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: added peer, starting replication: peer=server-98bf8875-a96c-1ab1-046c-136db591cd9d + testing.go:149: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.359Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: added peer, starting replication: peer=server-98bf8875-a96c-1ab1-046c-136db591cd9d +testLoggerAdapter verbose: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.359Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: added peer, starting replication: peer=server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8 + testing.go:149: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.359Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: added peer, starting replication: peer=server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8 +testLoggerAdapter verbose: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.359Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: added peer, starting replication: peer=server-d6eecb40-44a2-80ed-0099-f25daa7cb905 + testing.go:149: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.359Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: added peer, starting replication: peer=server-d6eecb40-44a2-80ed-0099-f25daa7cb905 +testLoggerAdapter verbose: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.359Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: added peer, starting replication: peer=server-674c7145-0e20-c026-98a5-1bbf859cf227 + testing.go:149: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.359Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: added peer, starting replication: peer=server-674c7145-0e20-c026-98a5-1bbf859cf227 +testLoggerAdapter verbose: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.359Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: added peer, starting replication: peer=server-e0ed200a-903b-96cd-1145-6098b46ee1cd + testing.go:149: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.359Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: added peer, starting replication: peer=server-e0ed200a-903b-96cd-1145-6098b46ee1cd +testLoggerAdapter verbose: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.359Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: added peer, starting replication: peer=server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6 + testing.go:149: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.359Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: added peer, starting replication: peer=server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6 +testLoggerAdapter verbose: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.359Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: pipelining replication: peer="{Voter server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6 7f4593f0-d11d-5c51-be28-fb05ca53dfd6}" + testing.go:149: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.359Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: pipelining replication: peer="{Voter server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6 7f4593f0-d11d-5c51-be28-fb05ca53dfd6}" +testLoggerAdapter verbose: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.359Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: entering follower state: follower="Node at 674c7145-0e20-c026-98a5-1bbf859cf227 [Follower]" leader= + testing.go:149: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.359Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: entering follower state: follower="Node at 674c7145-0e20-c026-98a5-1bbf859cf227 [Follower]" leader= +testLoggerAdapter verbose: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.359Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: aborting pipeline replication: peer="{Voter server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8 28ff6f15-37c2-bfde-bfce-97bb80b2cac8}" + testing.go:149: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.359Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: aborting pipeline replication: peer="{Voter server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8 28ff6f15-37c2-bfde-bfce-97bb80b2cac8}" +testLoggerAdapter verbose: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.360Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: pipelining replication: peer="{Voter server-98bf8875-a96c-1ab1-046c-136db591cd9d 98bf8875-a96c-1ab1-046c-136db591cd9d}" + testing.go:149: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.360Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: pipelining replication: peer="{Voter server-98bf8875-a96c-1ab1-046c-136db591cd9d 98bf8875-a96c-1ab1-046c-136db591cd9d}" +testLoggerAdapter verbose: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.360Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: pipelining replication: peer="{Voter server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8 28ff6f15-37c2-bfde-bfce-97bb80b2cac8}" + testing.go:149: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.360Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: pipelining replication: peer="{Voter server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8 28ff6f15-37c2-bfde-bfce-97bb80b2cac8}" +testLoggerAdapter verbose: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.360Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: pipelining replication: peer="{Voter server-d6eecb40-44a2-80ed-0099-f25daa7cb905 d6eecb40-44a2-80ed-0099-f25daa7cb905}" + testing.go:149: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.360Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: pipelining replication: peer="{Voter server-d6eecb40-44a2-80ed-0099-f25daa7cb905 d6eecb40-44a2-80ed-0099-f25daa7cb905}" +testLoggerAdapter verbose: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.360Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: pipelining replication: peer="{Voter server-674c7145-0e20-c026-98a5-1bbf859cf227 674c7145-0e20-c026-98a5-1bbf859cf227}" + testing.go:149: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.360Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: pipelining replication: peer="{Voter server-674c7145-0e20-c026-98a5-1bbf859cf227 674c7145-0e20-c026-98a5-1bbf859cf227}" +testLoggerAdapter verbose: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.360Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: pipelining replication: peer="{Voter server-e0ed200a-903b-96cd-1145-6098b46ee1cd e0ed200a-903b-96cd-1145-6098b46ee1cd}" + testing.go:149: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.360Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: pipelining replication: peer="{Voter server-e0ed200a-903b-96cd-1145-6098b46ee1cd e0ed200a-903b-96cd-1145-6098b46ee1cd}" +testLoggerAdapter verbose: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.360Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: aborting pipeline replication: peer="{Voter server-d6eecb40-44a2-80ed-0099-f25daa7cb905 d6eecb40-44a2-80ed-0099-f25daa7cb905}" + testing.go:149: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.360Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: aborting pipeline replication: peer="{Voter server-d6eecb40-44a2-80ed-0099-f25daa7cb905 d6eecb40-44a2-80ed-0099-f25daa7cb905}" +testLoggerAdapter verbose: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.360Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: aborting pipeline replication: peer="{Voter server-e0ed200a-903b-96cd-1145-6098b46ee1cd e0ed200a-903b-96cd-1145-6098b46ee1cd}" + testing.go:149: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.360Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: aborting pipeline replication: peer="{Voter server-e0ed200a-903b-96cd-1145-6098b46ee1cd e0ed200a-903b-96cd-1145-6098b46ee1cd}" +testLoggerAdapter verbose: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.360Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: aborting pipeline replication: peer="{Voter server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6 7f4593f0-d11d-5c51-be28-fb05ca53dfd6}" + testing.go:149: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.360Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: aborting pipeline replication: peer="{Voter server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6 7f4593f0-d11d-5c51-be28-fb05ca53dfd6}" +testLoggerAdapter verbose: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.360Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: aborting pipeline replication: peer="{Voter server-164b7adb-6988-0e3c-a0e4-f6962a583cb3 164b7adb-6988-0e3c-a0e4-f6962a583cb3}" + testing.go:149: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.360Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: aborting pipeline replication: peer="{Voter server-164b7adb-6988-0e3c-a0e4-f6962a583cb3 164b7adb-6988-0e3c-a0e4-f6962a583cb3}" +testLoggerAdapter verbose: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.360Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: aborting pipeline replication: peer="{Voter server-98bf8875-a96c-1ab1-046c-136db591cd9d 98bf8875-a96c-1ab1-046c-136db591cd9d}" + testing.go:149: server-674c7145-0e20-c026-98a5-1bbf859cf227: 2024-01-08T00:31:12.360Z [INFO] server-674c7145-0e20-c026-98a5-1bbf859cf227: aborting pipeline replication: peer="{Voter server-98bf8875-a96c-1ab1-046c-136db591cd9d 98bf8875-a96c-1ab1-046c-136db591cd9d}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:12.465Z [INFO] stable state for Leader reached at 2024-01-08 00:31:12.3600618 +0000 UTC m=+18.406950852 (1 nodes), 546.604µs from start of poll, 379.490219ms from cluster start. Timeout at 2024-01-08 00:31:12.465098955 +0000 UTC m=+18.511988001, 105.037149ms after stability + testing.go:149: cluster: 2024-01-08T00:31:12.465Z [INFO] stable state for Leader reached at 2024-01-08 00:31:12.3600618 +0000 UTC m=+18.406950852 (1 nodes), 546.604µs from start of poll, 379.490219ms from cluster start. Timeout at 2024-01-08 00:31:12.465098955 +0000 UTC m=+18.511988001, 105.037149ms after stability +testLoggerAdapter verbose: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.465Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: aborting pipeline replication: peer="{Voter server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8 28ff6f15-37c2-bfde-bfce-97bb80b2cac8}" + testing.go:149: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.465Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: aborting pipeline replication: peer="{Voter server-28ff6f15-37c2-bfde-bfce-97bb80b2cac8 28ff6f15-37c2-bfde-bfce-97bb80b2cac8}" +testLoggerAdapter verbose: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.465Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: aborting pipeline replication: peer="{Voter server-d6eecb40-44a2-80ed-0099-f25daa7cb905 d6eecb40-44a2-80ed-0099-f25daa7cb905}" + testing.go:149: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.465Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: aborting pipeline replication: peer="{Voter server-d6eecb40-44a2-80ed-0099-f25daa7cb905 d6eecb40-44a2-80ed-0099-f25daa7cb905}" +testLoggerAdapter verbose: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.465Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: aborting pipeline replication: peer="{Voter server-674c7145-0e20-c026-98a5-1bbf859cf227 674c7145-0e20-c026-98a5-1bbf859cf227}" + testing.go:149: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.465Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: aborting pipeline replication: peer="{Voter server-674c7145-0e20-c026-98a5-1bbf859cf227 674c7145-0e20-c026-98a5-1bbf859cf227}" +testLoggerAdapter verbose: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.465Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: aborting pipeline replication: peer="{Voter server-e0ed200a-903b-96cd-1145-6098b46ee1cd e0ed200a-903b-96cd-1145-6098b46ee1cd}" + testing.go:149: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.465Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: aborting pipeline replication: peer="{Voter server-e0ed200a-903b-96cd-1145-6098b46ee1cd e0ed200a-903b-96cd-1145-6098b46ee1cd}" +testLoggerAdapter verbose: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.465Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: aborting pipeline replication: peer="{Voter server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6 7f4593f0-d11d-5c51-be28-fb05ca53dfd6}" + testing.go:149: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.465Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: aborting pipeline replication: peer="{Voter server-7f4593f0-d11d-5c51-be28-fb05ca53dfd6 7f4593f0-d11d-5c51-be28-fb05ca53dfd6}" +testLoggerAdapter verbose: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.465Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: aborting pipeline replication: peer="{Voter server-98bf8875-a96c-1ab1-046c-136db591cd9d 98bf8875-a96c-1ab1-046c-136db591cd9d}" + testing.go:149: server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: 2024-01-08T00:31:12.465Z [INFO] server-164b7adb-6988-0e3c-a0e4-f6962a583cb3: aborting pipeline replication: peer="{Voter server-98bf8875-a96c-1ab1-046c-136db591cd9d 98bf8875-a96c-1ab1-046c-136db591cd9d}" --- PASS: TestRaft_LeadershipTransferWithSevenNodes (0.49s) === RUN TestRaft_LeadershipTransferToInvalidID -testLoggerAdapter verbose: server-53cfd309-9471-b11e-a8d5-5e21553d627a: 2025-02-09T06:52:37.012Z [INFO] server-53cfd309-9471-b11e-a8d5-5e21553d627a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53cfd309-9471-b11e-a8d5-5e21553d627a Address:53cfd309-9471-b11e-a8d5-5e21553d627a} {Suffrage:Voter ID:server-06b4daae-7726-8ab8-97e3-c2066755bb01 Address:06b4daae-7726-8ab8-97e3-c2066755bb01} {Suffrage:Voter ID:server-731f7b0a-c194-21de-9926-3269aad60d1b Address:731f7b0a-c194-21de-9926-3269aad60d1b}]" - testing.go:149: server-53cfd309-9471-b11e-a8d5-5e21553d627a: 2025-02-09T06:52:37.012Z [INFO] server-53cfd309-9471-b11e-a8d5-5e21553d627a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53cfd309-9471-b11e-a8d5-5e21553d627a Address:53cfd309-9471-b11e-a8d5-5e21553d627a} {Suffrage:Voter ID:server-06b4daae-7726-8ab8-97e3-c2066755bb01 Address:06b4daae-7726-8ab8-97e3-c2066755bb01} {Suffrage:Voter ID:server-731f7b0a-c194-21de-9926-3269aad60d1b Address:731f7b0a-c194-21de-9926-3269aad60d1b}]" -testLoggerAdapter verbose: server-53cfd309-9471-b11e-a8d5-5e21553d627a: 2025-02-09T06:52:37.012Z [INFO] server-53cfd309-9471-b11e-a8d5-5e21553d627a: entering follower state: follower="Node at 53cfd309-9471-b11e-a8d5-5e21553d627a [Follower]" leader= - testing.go:149: server-53cfd309-9471-b11e-a8d5-5e21553d627a: 2025-02-09T06:52:37.012Z [INFO] server-53cfd309-9471-b11e-a8d5-5e21553d627a: entering follower state: follower="Node at 53cfd309-9471-b11e-a8d5-5e21553d627a [Follower]" leader= -testLoggerAdapter verbose: server-06b4daae-7726-8ab8-97e3-c2066755bb01: 2025-02-09T06:52:37.012Z [INFO] server-06b4daae-7726-8ab8-97e3-c2066755bb01: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53cfd309-9471-b11e-a8d5-5e21553d627a Address:53cfd309-9471-b11e-a8d5-5e21553d627a} {Suffrage:Voter ID:server-06b4daae-7726-8ab8-97e3-c2066755bb01 Address:06b4daae-7726-8ab8-97e3-c2066755bb01} {Suffrage:Voter ID:server-731f7b0a-c194-21de-9926-3269aad60d1b Address:731f7b0a-c194-21de-9926-3269aad60d1b}]" - testing.go:149: server-06b4daae-7726-8ab8-97e3-c2066755bb01: 2025-02-09T06:52:37.012Z [INFO] server-06b4daae-7726-8ab8-97e3-c2066755bb01: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53cfd309-9471-b11e-a8d5-5e21553d627a Address:53cfd309-9471-b11e-a8d5-5e21553d627a} {Suffrage:Voter ID:server-06b4daae-7726-8ab8-97e3-c2066755bb01 Address:06b4daae-7726-8ab8-97e3-c2066755bb01} {Suffrage:Voter ID:server-731f7b0a-c194-21de-9926-3269aad60d1b Address:731f7b0a-c194-21de-9926-3269aad60d1b}]" -testLoggerAdapter verbose: server-06b4daae-7726-8ab8-97e3-c2066755bb01: 2025-02-09T06:52:37.012Z [INFO] server-06b4daae-7726-8ab8-97e3-c2066755bb01: entering follower state: follower="Node at 06b4daae-7726-8ab8-97e3-c2066755bb01 [Follower]" leader= - testing.go:149: server-06b4daae-7726-8ab8-97e3-c2066755bb01: 2025-02-09T06:52:37.012Z [INFO] server-06b4daae-7726-8ab8-97e3-c2066755bb01: entering follower state: follower="Node at 06b4daae-7726-8ab8-97e3-c2066755bb01 [Follower]" leader= -testLoggerAdapter verbose: server-731f7b0a-c194-21de-9926-3269aad60d1b: 2025-02-09T06:52:37.012Z [INFO] server-731f7b0a-c194-21de-9926-3269aad60d1b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53cfd309-9471-b11e-a8d5-5e21553d627a Address:53cfd309-9471-b11e-a8d5-5e21553d627a} {Suffrage:Voter ID:server-06b4daae-7726-8ab8-97e3-c2066755bb01 Address:06b4daae-7726-8ab8-97e3-c2066755bb01} {Suffrage:Voter ID:server-731f7b0a-c194-21de-9926-3269aad60d1b Address:731f7b0a-c194-21de-9926-3269aad60d1b}]" - testing.go:149: server-731f7b0a-c194-21de-9926-3269aad60d1b: 2025-02-09T06:52:37.012Z [INFO] server-731f7b0a-c194-21de-9926-3269aad60d1b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53cfd309-9471-b11e-a8d5-5e21553d627a Address:53cfd309-9471-b11e-a8d5-5e21553d627a} {Suffrage:Voter ID:server-06b4daae-7726-8ab8-97e3-c2066755bb01 Address:06b4daae-7726-8ab8-97e3-c2066755bb01} {Suffrage:Voter ID:server-731f7b0a-c194-21de-9926-3269aad60d1b Address:731f7b0a-c194-21de-9926-3269aad60d1b}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:37.012Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:37.012Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-731f7b0a-c194-21de-9926-3269aad60d1b: 2025-02-09T06:52:37.012Z [INFO] server-731f7b0a-c194-21de-9926-3269aad60d1b: entering follower state: follower="Node at 731f7b0a-c194-21de-9926-3269aad60d1b [Follower]" leader= - testing.go:149: server-731f7b0a-c194-21de-9926-3269aad60d1b: 2025-02-09T06:52:37.012Z [INFO] server-731f7b0a-c194-21de-9926-3269aad60d1b: entering follower state: follower="Node at 731f7b0a-c194-21de-9926-3269aad60d1b [Follower]" leader= -testLoggerAdapter verbose: server-06b4daae-7726-8ab8-97e3-c2066755bb01: 2025-02-09T06:52:37.070Z [WARN] server-06b4daae-7726-8ab8-97e3-c2066755bb01: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-06b4daae-7726-8ab8-97e3-c2066755bb01: 2025-02-09T06:52:37.070Z [WARN] server-06b4daae-7726-8ab8-97e3-c2066755bb01: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-06b4daae-7726-8ab8-97e3-c2066755bb01: 2025-02-09T06:52:37.070Z [INFO] server-06b4daae-7726-8ab8-97e3-c2066755bb01: entering candidate state: node="Node at 06b4daae-7726-8ab8-97e3-c2066755bb01 [Candidate]" term=2 - testing.go:149: server-06b4daae-7726-8ab8-97e3-c2066755bb01: 2025-02-09T06:52:37.070Z [INFO] server-06b4daae-7726-8ab8-97e3-c2066755bb01: entering candidate state: node="Node at 06b4daae-7726-8ab8-97e3-c2066755bb01 [Candidate]" term=2 -testLoggerAdapter verbose: server-06b4daae-7726-8ab8-97e3-c2066755bb01: 2025-02-09T06:52:37.070Z [INFO] server-06b4daae-7726-8ab8-97e3-c2066755bb01: election won: tally=2 - testing.go:149: server-06b4daae-7726-8ab8-97e3-c2066755bb01: 2025-02-09T06:52:37.070Z [INFO] server-06b4daae-7726-8ab8-97e3-c2066755bb01: election won: tally=2 -testLoggerAdapter verbose: server-06b4daae-7726-8ab8-97e3-c2066755bb01: 2025-02-09T06:52:37.070Z [INFO] server-06b4daae-7726-8ab8-97e3-c2066755bb01: entering leader state: leader="Node at 06b4daae-7726-8ab8-97e3-c2066755bb01 [Leader]" - testing.go:149: server-06b4daae-7726-8ab8-97e3-c2066755bb01: 2025-02-09T06:52:37.070Z [INFO] server-06b4daae-7726-8ab8-97e3-c2066755bb01: entering leader state: leader="Node at 06b4daae-7726-8ab8-97e3-c2066755bb01 [Leader]" -testLoggerAdapter verbose: server-06b4daae-7726-8ab8-97e3-c2066755bb01: 2025-02-09T06:52:37.070Z [INFO] server-06b4daae-7726-8ab8-97e3-c2066755bb01: added peer, starting replication: peer=server-53cfd309-9471-b11e-a8d5-5e21553d627a - testing.go:149: server-06b4daae-7726-8ab8-97e3-c2066755bb01: 2025-02-09T06:52:37.070Z [INFO] server-06b4daae-7726-8ab8-97e3-c2066755bb01: added peer, starting replication: peer=server-53cfd309-9471-b11e-a8d5-5e21553d627a -testLoggerAdapter verbose: server-06b4daae-7726-8ab8-97e3-c2066755bb01: 2025-02-09T06:52:37.070Z [INFO] server-06b4daae-7726-8ab8-97e3-c2066755bb01: added peer, starting replication: peer=server-731f7b0a-c194-21de-9926-3269aad60d1b - testing.go:149: server-06b4daae-7726-8ab8-97e3-c2066755bb01: 2025-02-09T06:52:37.070Z [INFO] server-06b4daae-7726-8ab8-97e3-c2066755bb01: added peer, starting replication: peer=server-731f7b0a-c194-21de-9926-3269aad60d1b -testLoggerAdapter verbose: server-06b4daae-7726-8ab8-97e3-c2066755bb01: 2025-02-09T06:52:37.070Z [INFO] server-06b4daae-7726-8ab8-97e3-c2066755bb01: pipelining replication: peer="{Voter server-53cfd309-9471-b11e-a8d5-5e21553d627a 53cfd309-9471-b11e-a8d5-5e21553d627a}" - testing.go:149: server-06b4daae-7726-8ab8-97e3-c2066755bb01: 2025-02-09T06:52:37.070Z [INFO] server-06b4daae-7726-8ab8-97e3-c2066755bb01: pipelining replication: peer="{Voter server-53cfd309-9471-b11e-a8d5-5e21553d627a 53cfd309-9471-b11e-a8d5-5e21553d627a}" -testLoggerAdapter verbose: server-06b4daae-7726-8ab8-97e3-c2066755bb01: 2025-02-09T06:52:37.070Z [INFO] server-06b4daae-7726-8ab8-97e3-c2066755bb01: pipelining replication: peer="{Voter server-731f7b0a-c194-21de-9926-3269aad60d1b 731f7b0a-c194-21de-9926-3269aad60d1b}" - testing.go:149: server-06b4daae-7726-8ab8-97e3-c2066755bb01: 2025-02-09T06:52:37.070Z [INFO] server-06b4daae-7726-8ab8-97e3-c2066755bb01: pipelining replication: peer="{Voter server-731f7b0a-c194-21de-9926-3269aad60d1b 731f7b0a-c194-21de-9926-3269aad60d1b}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:37.175Z [INFO] stable state for Leader reached at 2025-02-09 06:52:37.070459316 +0000 UTC m=+18.049268091 (1 nodes), 58.160562ms from start of poll, 58.511395ms from cluster start. Timeout at 2025-02-09 06:52:37.175473581 +0000 UTC m=+18.154282341, 105.01425ms after stability - testing.go:149: cluster: 2025-02-09T06:52:37.175Z [INFO] stable state for Leader reached at 2025-02-09 06:52:37.070459316 +0000 UTC m=+18.049268091 (1 nodes), 58.160562ms from start of poll, 58.511395ms from cluster start. Timeout at 2025-02-09 06:52:37.175473581 +0000 UTC m=+18.154282341, 105.01425ms after stability -testLoggerAdapter verbose: server-06b4daae-7726-8ab8-97e3-c2066755bb01: 2025-02-09T06:52:37.175Z [INFO] server-06b4daae-7726-8ab8-97e3-c2066755bb01: aborting pipeline replication: peer="{Voter server-731f7b0a-c194-21de-9926-3269aad60d1b 731f7b0a-c194-21de-9926-3269aad60d1b}" - testing.go:149: server-06b4daae-7726-8ab8-97e3-c2066755bb01: 2025-02-09T06:52:37.175Z [INFO] server-06b4daae-7726-8ab8-97e3-c2066755bb01: aborting pipeline replication: peer="{Voter server-731f7b0a-c194-21de-9926-3269aad60d1b 731f7b0a-c194-21de-9926-3269aad60d1b}" -testLoggerAdapter verbose: server-06b4daae-7726-8ab8-97e3-c2066755bb01: 2025-02-09T06:52:37.175Z [INFO] server-06b4daae-7726-8ab8-97e3-c2066755bb01: aborting pipeline replication: peer="{Voter server-53cfd309-9471-b11e-a8d5-5e21553d627a 53cfd309-9471-b11e-a8d5-5e21553d627a}" - testing.go:149: server-06b4daae-7726-8ab8-97e3-c2066755bb01: 2025-02-09T06:52:37.175Z [INFO] server-06b4daae-7726-8ab8-97e3-c2066755bb01: aborting pipeline replication: peer="{Voter server-53cfd309-9471-b11e-a8d5-5e21553d627a 53cfd309-9471-b11e-a8d5-5e21553d627a}" ---- PASS: TestRaft_LeadershipTransferToInvalidID (0.16s) +testLoggerAdapter verbose: server-605a4112-d239-6ef2-fea3-50b782c01518: 2024-01-08T00:31:12.469Z [INFO] server-605a4112-d239-6ef2-fea3-50b782c01518: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-605a4112-d239-6ef2-fea3-50b782c01518 Address:605a4112-d239-6ef2-fea3-50b782c01518} {Suffrage:Voter ID:server-346fd114-a0e4-1121-32e7-78f6ad1aa53e Address:346fd114-a0e4-1121-32e7-78f6ad1aa53e} {Suffrage:Voter ID:server-6df57732-58e0-859c-a87d-724d9f58ace2 Address:6df57732-58e0-859c-a87d-724d9f58ace2}]" + testing.go:149: server-605a4112-d239-6ef2-fea3-50b782c01518: 2024-01-08T00:31:12.469Z [INFO] server-605a4112-d239-6ef2-fea3-50b782c01518: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-605a4112-d239-6ef2-fea3-50b782c01518 Address:605a4112-d239-6ef2-fea3-50b782c01518} {Suffrage:Voter ID:server-346fd114-a0e4-1121-32e7-78f6ad1aa53e Address:346fd114-a0e4-1121-32e7-78f6ad1aa53e} {Suffrage:Voter ID:server-6df57732-58e0-859c-a87d-724d9f58ace2 Address:6df57732-58e0-859c-a87d-724d9f58ace2}]" +testLoggerAdapter verbose: server-346fd114-a0e4-1121-32e7-78f6ad1aa53e: 2024-01-08T00:31:12.469Z [INFO] server-346fd114-a0e4-1121-32e7-78f6ad1aa53e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-605a4112-d239-6ef2-fea3-50b782c01518 Address:605a4112-d239-6ef2-fea3-50b782c01518} {Suffrage:Voter ID:server-346fd114-a0e4-1121-32e7-78f6ad1aa53e Address:346fd114-a0e4-1121-32e7-78f6ad1aa53e} {Suffrage:Voter ID:server-6df57732-58e0-859c-a87d-724d9f58ace2 Address:6df57732-58e0-859c-a87d-724d9f58ace2}]" + testing.go:149: server-346fd114-a0e4-1121-32e7-78f6ad1aa53e: 2024-01-08T00:31:12.469Z [INFO] server-346fd114-a0e4-1121-32e7-78f6ad1aa53e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-605a4112-d239-6ef2-fea3-50b782c01518 Address:605a4112-d239-6ef2-fea3-50b782c01518} {Suffrage:Voter ID:server-346fd114-a0e4-1121-32e7-78f6ad1aa53e Address:346fd114-a0e4-1121-32e7-78f6ad1aa53e} {Suffrage:Voter ID:server-6df57732-58e0-859c-a87d-724d9f58ace2 Address:6df57732-58e0-859c-a87d-724d9f58ace2}]" +testLoggerAdapter verbose: server-6df57732-58e0-859c-a87d-724d9f58ace2: 2024-01-08T00:31:12.469Z [INFO] server-6df57732-58e0-859c-a87d-724d9f58ace2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-605a4112-d239-6ef2-fea3-50b782c01518 Address:605a4112-d239-6ef2-fea3-50b782c01518} {Suffrage:Voter ID:server-346fd114-a0e4-1121-32e7-78f6ad1aa53e Address:346fd114-a0e4-1121-32e7-78f6ad1aa53e} {Suffrage:Voter ID:server-6df57732-58e0-859c-a87d-724d9f58ace2 Address:6df57732-58e0-859c-a87d-724d9f58ace2}]" + testing.go:149: server-6df57732-58e0-859c-a87d-724d9f58ace2: 2024-01-08T00:31:12.469Z [INFO] server-6df57732-58e0-859c-a87d-724d9f58ace2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-605a4112-d239-6ef2-fea3-50b782c01518 Address:605a4112-d239-6ef2-fea3-50b782c01518} {Suffrage:Voter ID:server-346fd114-a0e4-1121-32e7-78f6ad1aa53e Address:346fd114-a0e4-1121-32e7-78f6ad1aa53e} {Suffrage:Voter ID:server-6df57732-58e0-859c-a87d-724d9f58ace2 Address:6df57732-58e0-859c-a87d-724d9f58ace2}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:12.469Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:12.469Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-605a4112-d239-6ef2-fea3-50b782c01518: 2024-01-08T00:31:12.469Z [INFO] server-605a4112-d239-6ef2-fea3-50b782c01518: entering follower state: follower="Node at 605a4112-d239-6ef2-fea3-50b782c01518 [Follower]" leader= + testing.go:149: server-605a4112-d239-6ef2-fea3-50b782c01518: 2024-01-08T00:31:12.469Z [INFO] server-605a4112-d239-6ef2-fea3-50b782c01518: entering follower state: follower="Node at 605a4112-d239-6ef2-fea3-50b782c01518 [Follower]" leader= +testLoggerAdapter verbose: server-346fd114-a0e4-1121-32e7-78f6ad1aa53e: 2024-01-08T00:31:12.469Z [INFO] server-346fd114-a0e4-1121-32e7-78f6ad1aa53e: entering follower state: follower="Node at 346fd114-a0e4-1121-32e7-78f6ad1aa53e [Follower]" leader= + testing.go:149: server-346fd114-a0e4-1121-32e7-78f6ad1aa53e: 2024-01-08T00:31:12.469Z [INFO] server-346fd114-a0e4-1121-32e7-78f6ad1aa53e: entering follower state: follower="Node at 346fd114-a0e4-1121-32e7-78f6ad1aa53e [Follower]" leader= +testLoggerAdapter verbose: server-6df57732-58e0-859c-a87d-724d9f58ace2: 2024-01-08T00:31:12.469Z [INFO] server-6df57732-58e0-859c-a87d-724d9f58ace2: entering follower state: follower="Node at 6df57732-58e0-859c-a87d-724d9f58ace2 [Follower]" leader= + testing.go:149: server-6df57732-58e0-859c-a87d-724d9f58ace2: 2024-01-08T00:31:12.469Z [INFO] server-6df57732-58e0-859c-a87d-724d9f58ace2: entering follower state: follower="Node at 6df57732-58e0-859c-a87d-724d9f58ace2 [Follower]" leader= +testLoggerAdapter verbose: server-6df57732-58e0-859c-a87d-724d9f58ace2: 2024-01-08T00:31:12.533Z [WARN] server-6df57732-58e0-859c-a87d-724d9f58ace2: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-6df57732-58e0-859c-a87d-724d9f58ace2: 2024-01-08T00:31:12.533Z [WARN] server-6df57732-58e0-859c-a87d-724d9f58ace2: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-6df57732-58e0-859c-a87d-724d9f58ace2: 2024-01-08T00:31:12.533Z [INFO] server-6df57732-58e0-859c-a87d-724d9f58ace2: entering candidate state: node="Node at 6df57732-58e0-859c-a87d-724d9f58ace2 [Candidate]" term=2 + testing.go:149: server-6df57732-58e0-859c-a87d-724d9f58ace2: 2024-01-08T00:31:12.533Z [INFO] server-6df57732-58e0-859c-a87d-724d9f58ace2: entering candidate state: node="Node at 6df57732-58e0-859c-a87d-724d9f58ace2 [Candidate]" term=2 +testLoggerAdapter verbose: server-6df57732-58e0-859c-a87d-724d9f58ace2: 2024-01-08T00:31:12.533Z [INFO] server-6df57732-58e0-859c-a87d-724d9f58ace2: election won: tally=2 + testing.go:149: server-6df57732-58e0-859c-a87d-724d9f58ace2: 2024-01-08T00:31:12.533Z [INFO] server-6df57732-58e0-859c-a87d-724d9f58ace2: election won: tally=2 +testLoggerAdapter verbose: server-6df57732-58e0-859c-a87d-724d9f58ace2: 2024-01-08T00:31:12.533Z [INFO] server-6df57732-58e0-859c-a87d-724d9f58ace2: entering leader state: leader="Node at 6df57732-58e0-859c-a87d-724d9f58ace2 [Leader]" + testing.go:149: server-6df57732-58e0-859c-a87d-724d9f58ace2: 2024-01-08T00:31:12.533Z [INFO] server-6df57732-58e0-859c-a87d-724d9f58ace2: entering leader state: leader="Node at 6df57732-58e0-859c-a87d-724d9f58ace2 [Leader]" +testLoggerAdapter verbose: server-6df57732-58e0-859c-a87d-724d9f58ace2: 2024-01-08T00:31:12.533Z [INFO] server-6df57732-58e0-859c-a87d-724d9f58ace2: added peer, starting replication: peer=server-605a4112-d239-6ef2-fea3-50b782c01518 + testing.go:149: server-6df57732-58e0-859c-a87d-724d9f58ace2: 2024-01-08T00:31:12.533Z [INFO] server-6df57732-58e0-859c-a87d-724d9f58ace2: added peer, starting replication: peer=server-605a4112-d239-6ef2-fea3-50b782c01518 +testLoggerAdapter verbose: server-6df57732-58e0-859c-a87d-724d9f58ace2: 2024-01-08T00:31:12.533Z [INFO] server-6df57732-58e0-859c-a87d-724d9f58ace2: added peer, starting replication: peer=server-346fd114-a0e4-1121-32e7-78f6ad1aa53e + testing.go:149: server-6df57732-58e0-859c-a87d-724d9f58ace2: 2024-01-08T00:31:12.533Z [INFO] server-6df57732-58e0-859c-a87d-724d9f58ace2: added peer, starting replication: peer=server-346fd114-a0e4-1121-32e7-78f6ad1aa53e +testLoggerAdapter verbose: server-6df57732-58e0-859c-a87d-724d9f58ace2: 2024-01-08T00:31:12.533Z [INFO] server-6df57732-58e0-859c-a87d-724d9f58ace2: pipelining replication: peer="{Voter server-346fd114-a0e4-1121-32e7-78f6ad1aa53e 346fd114-a0e4-1121-32e7-78f6ad1aa53e}" + testing.go:149: server-6df57732-58e0-859c-a87d-724d9f58ace2: 2024-01-08T00:31:12.533Z [INFO] server-6df57732-58e0-859c-a87d-724d9f58ace2: pipelining replication: peer="{Voter server-346fd114-a0e4-1121-32e7-78f6ad1aa53e 346fd114-a0e4-1121-32e7-78f6ad1aa53e}" +testLoggerAdapter verbose: server-6df57732-58e0-859c-a87d-724d9f58ace2: 2024-01-08T00:31:12.533Z [INFO] server-6df57732-58e0-859c-a87d-724d9f58ace2: pipelining replication: peer="{Voter server-605a4112-d239-6ef2-fea3-50b782c01518 605a4112-d239-6ef2-fea3-50b782c01518}" + testing.go:149: server-6df57732-58e0-859c-a87d-724d9f58ace2: 2024-01-08T00:31:12.533Z [INFO] server-6df57732-58e0-859c-a87d-724d9f58ace2: pipelining replication: peer="{Voter server-605a4112-d239-6ef2-fea3-50b782c01518 605a4112-d239-6ef2-fea3-50b782c01518}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:12.639Z [INFO] stable state for Leader reached at 2024-01-08 00:31:12.533924166 +0000 UTC m=+18.580813371 (1 nodes), 64.052627ms from start of poll, 64.469859ms from cluster start. Timeout at 2024-01-08 00:31:12.639776791 +0000 UTC m=+18.686665913, 105.852542ms after stability + testing.go:149: cluster: 2024-01-08T00:31:12.639Z [INFO] stable state for Leader reached at 2024-01-08 00:31:12.533924166 +0000 UTC m=+18.580813371 (1 nodes), 64.052627ms from start of poll, 64.469859ms from cluster start. Timeout at 2024-01-08 00:31:12.639776791 +0000 UTC m=+18.686665913, 105.852542ms after stability +testLoggerAdapter verbose: server-6df57732-58e0-859c-a87d-724d9f58ace2: 2024-01-08T00:31:12.639Z [INFO] server-6df57732-58e0-859c-a87d-724d9f58ace2: aborting pipeline replication: peer="{Voter server-346fd114-a0e4-1121-32e7-78f6ad1aa53e 346fd114-a0e4-1121-32e7-78f6ad1aa53e}" + testing.go:149: server-6df57732-58e0-859c-a87d-724d9f58ace2: 2024-01-08T00:31:12.639Z [INFO] server-6df57732-58e0-859c-a87d-724d9f58ace2: aborting pipeline replication: peer="{Voter server-346fd114-a0e4-1121-32e7-78f6ad1aa53e 346fd114-a0e4-1121-32e7-78f6ad1aa53e}" +testLoggerAdapter verbose: server-6df57732-58e0-859c-a87d-724d9f58ace2: 2024-01-08T00:31:12.640Z [INFO] server-6df57732-58e0-859c-a87d-724d9f58ace2: aborting pipeline replication: peer="{Voter server-605a4112-d239-6ef2-fea3-50b782c01518 605a4112-d239-6ef2-fea3-50b782c01518}" + testing.go:149: server-6df57732-58e0-859c-a87d-724d9f58ace2: 2024-01-08T00:31:12.640Z [INFO] server-6df57732-58e0-859c-a87d-724d9f58ace2: aborting pipeline replication: peer="{Voter server-605a4112-d239-6ef2-fea3-50b782c01518 605a4112-d239-6ef2-fea3-50b782c01518}" +--- PASS: TestRaft_LeadershipTransferToInvalidID (0.17s) === RUN TestRaft_LeadershipTransferToInvalidAddress -testLoggerAdapter verbose: server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: 2025-02-09T06:52:37.176Z [INFO] server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-20e0bb57-6e37-3eb7-3c9e-dfc807551936 Address:20e0bb57-6e37-3eb7-3c9e-dfc807551936} {Suffrage:Voter ID:server-532ba30b-00ae-29b9-9acb-2a629f5e0ecf Address:532ba30b-00ae-29b9-9acb-2a629f5e0ecf} {Suffrage:Voter ID:server-e073dc33-ddd1-5e3f-af0a-1a511caf1ea6 Address:e073dc33-ddd1-5e3f-af0a-1a511caf1ea6}]" - testing.go:149: server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: 2025-02-09T06:52:37.176Z [INFO] server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-20e0bb57-6e37-3eb7-3c9e-dfc807551936 Address:20e0bb57-6e37-3eb7-3c9e-dfc807551936} {Suffrage:Voter ID:server-532ba30b-00ae-29b9-9acb-2a629f5e0ecf Address:532ba30b-00ae-29b9-9acb-2a629f5e0ecf} {Suffrage:Voter ID:server-e073dc33-ddd1-5e3f-af0a-1a511caf1ea6 Address:e073dc33-ddd1-5e3f-af0a-1a511caf1ea6}]" -testLoggerAdapter verbose: server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: 2025-02-09T06:52:37.176Z [INFO] server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: entering follower state: follower="Node at 20e0bb57-6e37-3eb7-3c9e-dfc807551936 [Follower]" leader= - testing.go:149: server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: 2025-02-09T06:52:37.176Z [INFO] server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: entering follower state: follower="Node at 20e0bb57-6e37-3eb7-3c9e-dfc807551936 [Follower]" leader= -testLoggerAdapter verbose: server-532ba30b-00ae-29b9-9acb-2a629f5e0ecf: 2025-02-09T06:52:37.176Z [INFO] server-532ba30b-00ae-29b9-9acb-2a629f5e0ecf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-20e0bb57-6e37-3eb7-3c9e-dfc807551936 Address:20e0bb57-6e37-3eb7-3c9e-dfc807551936} {Suffrage:Voter ID:server-532ba30b-00ae-29b9-9acb-2a629f5e0ecf Address:532ba30b-00ae-29b9-9acb-2a629f5e0ecf} {Suffrage:Voter ID:server-e073dc33-ddd1-5e3f-af0a-1a511caf1ea6 Address:e073dc33-ddd1-5e3f-af0a-1a511caf1ea6}]" - testing.go:149: server-532ba30b-00ae-29b9-9acb-2a629f5e0ecf: 2025-02-09T06:52:37.176Z [INFO] server-532ba30b-00ae-29b9-9acb-2a629f5e0ecf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-20e0bb57-6e37-3eb7-3c9e-dfc807551936 Address:20e0bb57-6e37-3eb7-3c9e-dfc807551936} {Suffrage:Voter ID:server-532ba30b-00ae-29b9-9acb-2a629f5e0ecf Address:532ba30b-00ae-29b9-9acb-2a629f5e0ecf} {Suffrage:Voter ID:server-e073dc33-ddd1-5e3f-af0a-1a511caf1ea6 Address:e073dc33-ddd1-5e3f-af0a-1a511caf1ea6}]" -testLoggerAdapter verbose: server-532ba30b-00ae-29b9-9acb-2a629f5e0ecf: 2025-02-09T06:52:37.176Z [INFO] server-532ba30b-00ae-29b9-9acb-2a629f5e0ecf: entering follower state: follower="Node at 532ba30b-00ae-29b9-9acb-2a629f5e0ecf [Follower]" leader= - testing.go:149: server-532ba30b-00ae-29b9-9acb-2a629f5e0ecf: 2025-02-09T06:52:37.176Z [INFO] server-532ba30b-00ae-29b9-9acb-2a629f5e0ecf: entering follower state: follower="Node at 532ba30b-00ae-29b9-9acb-2a629f5e0ecf [Follower]" leader= -testLoggerAdapter verbose: server-e073dc33-ddd1-5e3f-af0a-1a511caf1ea6: 2025-02-09T06:52:37.176Z [INFO] server-e073dc33-ddd1-5e3f-af0a-1a511caf1ea6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-20e0bb57-6e37-3eb7-3c9e-dfc807551936 Address:20e0bb57-6e37-3eb7-3c9e-dfc807551936} {Suffrage:Voter ID:server-532ba30b-00ae-29b9-9acb-2a629f5e0ecf Address:532ba30b-00ae-29b9-9acb-2a629f5e0ecf} {Suffrage:Voter ID:server-e073dc33-ddd1-5e3f-af0a-1a511caf1ea6 Address:e073dc33-ddd1-5e3f-af0a-1a511caf1ea6}]" - testing.go:149: server-e073dc33-ddd1-5e3f-af0a-1a511caf1ea6: 2025-02-09T06:52:37.176Z [INFO] server-e073dc33-ddd1-5e3f-af0a-1a511caf1ea6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-20e0bb57-6e37-3eb7-3c9e-dfc807551936 Address:20e0bb57-6e37-3eb7-3c9e-dfc807551936} {Suffrage:Voter ID:server-532ba30b-00ae-29b9-9acb-2a629f5e0ecf Address:532ba30b-00ae-29b9-9acb-2a629f5e0ecf} {Suffrage:Voter ID:server-e073dc33-ddd1-5e3f-af0a-1a511caf1ea6 Address:e073dc33-ddd1-5e3f-af0a-1a511caf1ea6}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:37.176Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T06:52:37.176Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-e073dc33-ddd1-5e3f-af0a-1a511caf1ea6: 2025-02-09T06:52:37.176Z [INFO] server-e073dc33-ddd1-5e3f-af0a-1a511caf1ea6: entering follower state: follower="Node at e073dc33-ddd1-5e3f-af0a-1a511caf1ea6 [Follower]" leader= - testing.go:149: server-e073dc33-ddd1-5e3f-af0a-1a511caf1ea6: 2025-02-09T06:52:37.176Z [INFO] server-e073dc33-ddd1-5e3f-af0a-1a511caf1ea6: entering follower state: follower="Node at e073dc33-ddd1-5e3f-af0a-1a511caf1ea6 [Follower]" leader= -testLoggerAdapter verbose: server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: 2025-02-09T06:52:37.248Z [WARN] server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: 2025-02-09T06:52:37.248Z [WARN] server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: 2025-02-09T06:52:37.248Z [INFO] server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: entering candidate state: node="Node at 20e0bb57-6e37-3eb7-3c9e-dfc807551936 [Candidate]" term=2 - testing.go:149: server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: 2025-02-09T06:52:37.248Z [INFO] server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: entering candidate state: node="Node at 20e0bb57-6e37-3eb7-3c9e-dfc807551936 [Candidate]" term=2 -testLoggerAdapter verbose: server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: 2025-02-09T06:52:37.248Z [INFO] server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: election won: tally=2 - testing.go:149: server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: 2025-02-09T06:52:37.248Z [INFO] server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: election won: tally=2 -testLoggerAdapter verbose: server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: 2025-02-09T06:52:37.248Z [INFO] server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: entering leader state: leader="Node at 20e0bb57-6e37-3eb7-3c9e-dfc807551936 [Leader]" - testing.go:149: server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: 2025-02-09T06:52:37.248Z [INFO] server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: entering leader state: leader="Node at 20e0bb57-6e37-3eb7-3c9e-dfc807551936 [Leader]" -testLoggerAdapter verbose: server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: 2025-02-09T06:52:37.248Z [INFO] server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: added peer, starting replication: peer=server-532ba30b-00ae-29b9-9acb-2a629f5e0ecf - testing.go:149: server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: 2025-02-09T06:52:37.248Z [INFO] server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: added peer, starting replication: peer=server-532ba30b-00ae-29b9-9acb-2a629f5e0ecf -testLoggerAdapter verbose: server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: 2025-02-09T06:52:37.248Z [INFO] server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: added peer, starting replication: peer=server-e073dc33-ddd1-5e3f-af0a-1a511caf1ea6 - testing.go:149: server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: 2025-02-09T06:52:37.248Z [INFO] server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: added peer, starting replication: peer=server-e073dc33-ddd1-5e3f-af0a-1a511caf1ea6 -testLoggerAdapter verbose: server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: 2025-02-09T06:52:37.248Z [INFO] server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: pipelining replication: peer="{Voter server-e073dc33-ddd1-5e3f-af0a-1a511caf1ea6 e073dc33-ddd1-5e3f-af0a-1a511caf1ea6}" - testing.go:149: server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: 2025-02-09T06:52:37.248Z [INFO] server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: pipelining replication: peer="{Voter server-e073dc33-ddd1-5e3f-af0a-1a511caf1ea6 e073dc33-ddd1-5e3f-af0a-1a511caf1ea6}" -testLoggerAdapter verbose: server-532ba30b-00ae-29b9-9acb-2a629f5e0ecf: 2025-02-09T06:52:37.248Z [WARN] server-532ba30b-00ae-29b9-9acb-2a629f5e0ecf: rejecting vote request since our last term is greater: candidate=20e0bb57-6e37-3eb7-3c9e-dfc807551936 last-term=2 last-candidate-term=1 - testing.go:149: server-532ba30b-00ae-29b9-9acb-2a629f5e0ecf: 2025-02-09T06:52:37.248Z [WARN] server-532ba30b-00ae-29b9-9acb-2a629f5e0ecf: rejecting vote request since our last term is greater: candidate=20e0bb57-6e37-3eb7-3c9e-dfc807551936 last-term=2 last-candidate-term=1 -testLoggerAdapter verbose: server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: 2025-02-09T06:52:37.249Z [INFO] server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: pipelining replication: peer="{Voter server-532ba30b-00ae-29b9-9acb-2a629f5e0ecf 532ba30b-00ae-29b9-9acb-2a629f5e0ecf}" - testing.go:149: server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: 2025-02-09T06:52:37.249Z [INFO] server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: pipelining replication: peer="{Voter server-532ba30b-00ae-29b9-9acb-2a629f5e0ecf 532ba30b-00ae-29b9-9acb-2a629f5e0ecf}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:37.354Z [INFO] stable state for Follower reached at 2025-02-09 06:52:37.248985261 +0000 UTC m=+18.227794038 (2 nodes), 72.293829ms from start of poll, 72.621465ms from cluster start. Timeout at 2025-02-09 06:52:37.354522846 +0000 UTC m=+18.333331604, 105.537566ms after stability - testing.go:149: cluster: 2025-02-09T06:52:37.354Z [INFO] stable state for Follower reached at 2025-02-09 06:52:37.248985261 +0000 UTC m=+18.227794038 (2 nodes), 72.293829ms from start of poll, 72.621465ms from cluster start. Timeout at 2025-02-09 06:52:37.354522846 +0000 UTC m=+18.333331604, 105.537566ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:37.354Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:37.354Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:37.460Z [INFO] stable state for Leader reached at 2025-02-09 06:52:37.354605392 +0000 UTC m=+18.333414155 (1 nodes), 644ns from start of poll, 178.241582ms from cluster start. Timeout at 2025-02-09 06:52:37.460508157 +0000 UTC m=+18.439316915, 105.90276ms after stability - testing.go:149: cluster: 2025-02-09T06:52:37.460Z [INFO] stable state for Leader reached at 2025-02-09 06:52:37.354605392 +0000 UTC m=+18.333414155 (1 nodes), 644ns from start of poll, 178.241582ms from cluster start. Timeout at 2025-02-09 06:52:37.460508157 +0000 UTC m=+18.439316915, 105.90276ms after stability -testLoggerAdapter verbose: server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: 2025-02-09T06:52:37.460Z [INFO] server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: aborting pipeline replication: peer="{Voter server-e073dc33-ddd1-5e3f-af0a-1a511caf1ea6 e073dc33-ddd1-5e3f-af0a-1a511caf1ea6}" - testing.go:149: server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: 2025-02-09T06:52:37.460Z [INFO] server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: aborting pipeline replication: peer="{Voter server-e073dc33-ddd1-5e3f-af0a-1a511caf1ea6 e073dc33-ddd1-5e3f-af0a-1a511caf1ea6}" -testLoggerAdapter verbose: server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: 2025-02-09T06:52:37.460Z [INFO] server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: aborting pipeline replication: peer="{Voter server-532ba30b-00ae-29b9-9acb-2a629f5e0ecf 532ba30b-00ae-29b9-9acb-2a629f5e0ecf}" - testing.go:149: server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: 2025-02-09T06:52:37.460Z [INFO] server-20e0bb57-6e37-3eb7-3c9e-dfc807551936: aborting pipeline replication: peer="{Voter server-532ba30b-00ae-29b9-9acb-2a629f5e0ecf 532ba30b-00ae-29b9-9acb-2a629f5e0ecf}" +testLoggerAdapter verbose: server-0dfcde5d-8416-36fa-d55e-16254f3730e3: 2024-01-08T00:31:12.641Z [INFO] server-0dfcde5d-8416-36fa-d55e-16254f3730e3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0dfcde5d-8416-36fa-d55e-16254f3730e3 Address:0dfcde5d-8416-36fa-d55e-16254f3730e3} {Suffrage:Voter ID:server-e0621db4-42b8-f994-878f-4853232c2e7f Address:e0621db4-42b8-f994-878f-4853232c2e7f} {Suffrage:Voter ID:server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98 Address:675b4019-5b1d-d0c3-4ac5-dbc50ec31d98}]" + testing.go:149: server-0dfcde5d-8416-36fa-d55e-16254f3730e3: 2024-01-08T00:31:12.641Z [INFO] server-0dfcde5d-8416-36fa-d55e-16254f3730e3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0dfcde5d-8416-36fa-d55e-16254f3730e3 Address:0dfcde5d-8416-36fa-d55e-16254f3730e3} {Suffrage:Voter ID:server-e0621db4-42b8-f994-878f-4853232c2e7f Address:e0621db4-42b8-f994-878f-4853232c2e7f} {Suffrage:Voter ID:server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98 Address:675b4019-5b1d-d0c3-4ac5-dbc50ec31d98}]" +testLoggerAdapter verbose: server-e0621db4-42b8-f994-878f-4853232c2e7f: 2024-01-08T00:31:12.641Z [INFO] server-e0621db4-42b8-f994-878f-4853232c2e7f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0dfcde5d-8416-36fa-d55e-16254f3730e3 Address:0dfcde5d-8416-36fa-d55e-16254f3730e3} {Suffrage:Voter ID:server-e0621db4-42b8-f994-878f-4853232c2e7f Address:e0621db4-42b8-f994-878f-4853232c2e7f} {Suffrage:Voter ID:server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98 Address:675b4019-5b1d-d0c3-4ac5-dbc50ec31d98}]" + testing.go:149: server-e0621db4-42b8-f994-878f-4853232c2e7f: 2024-01-08T00:31:12.641Z [INFO] server-e0621db4-42b8-f994-878f-4853232c2e7f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0dfcde5d-8416-36fa-d55e-16254f3730e3 Address:0dfcde5d-8416-36fa-d55e-16254f3730e3} {Suffrage:Voter ID:server-e0621db4-42b8-f994-878f-4853232c2e7f Address:e0621db4-42b8-f994-878f-4853232c2e7f} {Suffrage:Voter ID:server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98 Address:675b4019-5b1d-d0c3-4ac5-dbc50ec31d98}]" +testLoggerAdapter verbose: server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: 2024-01-08T00:31:12.641Z [INFO] server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0dfcde5d-8416-36fa-d55e-16254f3730e3 Address:0dfcde5d-8416-36fa-d55e-16254f3730e3} {Suffrage:Voter ID:server-e0621db4-42b8-f994-878f-4853232c2e7f Address:e0621db4-42b8-f994-878f-4853232c2e7f} {Suffrage:Voter ID:server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98 Address:675b4019-5b1d-d0c3-4ac5-dbc50ec31d98}]" + testing.go:149: server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: 2024-01-08T00:31:12.641Z [INFO] server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0dfcde5d-8416-36fa-d55e-16254f3730e3 Address:0dfcde5d-8416-36fa-d55e-16254f3730e3} {Suffrage:Voter ID:server-e0621db4-42b8-f994-878f-4853232c2e7f Address:e0621db4-42b8-f994-878f-4853232c2e7f} {Suffrage:Voter ID:server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98 Address:675b4019-5b1d-d0c3-4ac5-dbc50ec31d98}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:12.642Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T00:31:12.642Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: server-0dfcde5d-8416-36fa-d55e-16254f3730e3: 2024-01-08T00:31:12.642Z [INFO] server-0dfcde5d-8416-36fa-d55e-16254f3730e3: entering follower state: follower="Node at 0dfcde5d-8416-36fa-d55e-16254f3730e3 [Follower]" leader= + testing.go:149: server-0dfcde5d-8416-36fa-d55e-16254f3730e3: 2024-01-08T00:31:12.642Z [INFO] server-0dfcde5d-8416-36fa-d55e-16254f3730e3: entering follower state: follower="Node at 0dfcde5d-8416-36fa-d55e-16254f3730e3 [Follower]" leader= +testLoggerAdapter verbose: server-e0621db4-42b8-f994-878f-4853232c2e7f: 2024-01-08T00:31:12.642Z [INFO] server-e0621db4-42b8-f994-878f-4853232c2e7f: entering follower state: follower="Node at e0621db4-42b8-f994-878f-4853232c2e7f [Follower]" leader= + testing.go:149: server-e0621db4-42b8-f994-878f-4853232c2e7f: 2024-01-08T00:31:12.642Z [INFO] server-e0621db4-42b8-f994-878f-4853232c2e7f: entering follower state: follower="Node at e0621db4-42b8-f994-878f-4853232c2e7f [Follower]" leader= +testLoggerAdapter verbose: server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: 2024-01-08T00:31:12.642Z [INFO] server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: entering follower state: follower="Node at 675b4019-5b1d-d0c3-4ac5-dbc50ec31d98 [Follower]" leader= + testing.go:149: server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: 2024-01-08T00:31:12.642Z [INFO] server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: entering follower state: follower="Node at 675b4019-5b1d-d0c3-4ac5-dbc50ec31d98 [Follower]" leader= +testLoggerAdapter verbose: server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: 2024-01-08T00:31:12.714Z [WARN] server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: 2024-01-08T00:31:12.714Z [WARN] server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: 2024-01-08T00:31:12.714Z [INFO] server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: entering candidate state: node="Node at 675b4019-5b1d-d0c3-4ac5-dbc50ec31d98 [Candidate]" term=2 + testing.go:149: server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: 2024-01-08T00:31:12.714Z [INFO] server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: entering candidate state: node="Node at 675b4019-5b1d-d0c3-4ac5-dbc50ec31d98 [Candidate]" term=2 +testLoggerAdapter verbose: server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: 2024-01-08T00:31:12.715Z [INFO] server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: election won: tally=2 + testing.go:149: server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: 2024-01-08T00:31:12.715Z [INFO] server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: election won: tally=2 +testLoggerAdapter verbose: server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: 2024-01-08T00:31:12.715Z [INFO] server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: entering leader state: leader="Node at 675b4019-5b1d-d0c3-4ac5-dbc50ec31d98 [Leader]" + testing.go:149: server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: 2024-01-08T00:31:12.715Z [INFO] server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: entering leader state: leader="Node at 675b4019-5b1d-d0c3-4ac5-dbc50ec31d98 [Leader]" +testLoggerAdapter verbose: server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: 2024-01-08T00:31:12.715Z [INFO] server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: added peer, starting replication: peer=server-0dfcde5d-8416-36fa-d55e-16254f3730e3 + testing.go:149: server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: 2024-01-08T00:31:12.715Z [INFO] server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: added peer, starting replication: peer=server-0dfcde5d-8416-36fa-d55e-16254f3730e3 +testLoggerAdapter verbose: server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: 2024-01-08T00:31:12.715Z [INFO] server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: added peer, starting replication: peer=server-e0621db4-42b8-f994-878f-4853232c2e7f + testing.go:149: server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: 2024-01-08T00:31:12.715Z [INFO] server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: added peer, starting replication: peer=server-e0621db4-42b8-f994-878f-4853232c2e7f +testLoggerAdapter verbose: server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: 2024-01-08T00:31:12.715Z [INFO] server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: pipelining replication: peer="{Voter server-e0621db4-42b8-f994-878f-4853232c2e7f e0621db4-42b8-f994-878f-4853232c2e7f}" + testing.go:149: server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: 2024-01-08T00:31:12.715Z [INFO] server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: pipelining replication: peer="{Voter server-e0621db4-42b8-f994-878f-4853232c2e7f e0621db4-42b8-f994-878f-4853232c2e7f}" +testLoggerAdapter verbose: server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: 2024-01-08T00:31:12.715Z [INFO] server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: pipelining replication: peer="{Voter server-0dfcde5d-8416-36fa-d55e-16254f3730e3 0dfcde5d-8416-36fa-d55e-16254f3730e3}" + testing.go:149: server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: 2024-01-08T00:31:12.715Z [INFO] server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: pipelining replication: peer="{Voter server-0dfcde5d-8416-36fa-d55e-16254f3730e3 0dfcde5d-8416-36fa-d55e-16254f3730e3}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:12.820Z [INFO] stable state for Follower reached at 2024-01-08 00:31:12.715472135 +0000 UTC m=+18.762361186 (2 nodes), 73.434247ms from start of poll, 73.949801ms from cluster start. Timeout at 2024-01-08 00:31:12.820779602 +0000 UTC m=+18.867668713, 105.307527ms after stability + testing.go:149: cluster: 2024-01-08T00:31:12.820Z [INFO] stable state for Follower reached at 2024-01-08 00:31:12.715472135 +0000 UTC m=+18.762361186 (2 nodes), 73.434247ms from start of poll, 73.949801ms from cluster start. Timeout at 2024-01-08 00:31:12.820779602 +0000 UTC m=+18.867668713, 105.307527ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:12.820Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:12.820Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:12.926Z [INFO] stable state for Leader reached at 2024-01-08 00:31:12.820999375 +0000 UTC m=+18.867888495 (1 nodes), 2.259µs from start of poll, 179.47711ms from cluster start. Timeout at 2024-01-08 00:31:12.926825553 +0000 UTC m=+18.973714595, 105.8261ms after stability + testing.go:149: cluster: 2024-01-08T00:31:12.926Z [INFO] stable state for Leader reached at 2024-01-08 00:31:12.820999375 +0000 UTC m=+18.867888495 (1 nodes), 2.259µs from start of poll, 179.47711ms from cluster start. Timeout at 2024-01-08 00:31:12.926825553 +0000 UTC m=+18.973714595, 105.8261ms after stability +testLoggerAdapter verbose: server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: 2024-01-08T00:31:12.927Z [INFO] server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: aborting pipeline replication: peer="{Voter server-e0621db4-42b8-f994-878f-4853232c2e7f e0621db4-42b8-f994-878f-4853232c2e7f}" + testing.go:149: server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: 2024-01-08T00:31:12.927Z [INFO] server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: aborting pipeline replication: peer="{Voter server-e0621db4-42b8-f994-878f-4853232c2e7f e0621db4-42b8-f994-878f-4853232c2e7f}" +testLoggerAdapter verbose: server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: 2024-01-08T00:31:12.927Z [INFO] server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: aborting pipeline replication: peer="{Voter server-0dfcde5d-8416-36fa-d55e-16254f3730e3 0dfcde5d-8416-36fa-d55e-16254f3730e3}" + testing.go:149: server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: 2024-01-08T00:31:12.927Z [INFO] server-675b4019-5b1d-d0c3-4ac5-dbc50ec31d98: aborting pipeline replication: peer="{Voter server-0dfcde5d-8416-36fa-d55e-16254f3730e3 0dfcde5d-8416-36fa-d55e-16254f3730e3}" --- PASS: TestRaft_LeadershipTransferToInvalidAddress (0.29s) === RUN TestRaft_LeadershipTransferToBehindServer -testLoggerAdapter verbose: server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: 2025-02-09T06:52:37.461Z [INFO] server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a4bd74c1-05c6-e416-9825-d5aba531a5ad Address:a4bd74c1-05c6-e416-9825-d5aba531a5ad} {Suffrage:Voter ID:server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0 Address:408d0d66-6ddc-aa04-77f7-ccd545d2b7a0} {Suffrage:Voter ID:server-286bbf55-db71-bc61-6347-72349c32f537 Address:286bbf55-db71-bc61-6347-72349c32f537}]" - testing.go:149: server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: 2025-02-09T06:52:37.461Z [INFO] server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a4bd74c1-05c6-e416-9825-d5aba531a5ad Address:a4bd74c1-05c6-e416-9825-d5aba531a5ad} {Suffrage:Voter ID:server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0 Address:408d0d66-6ddc-aa04-77f7-ccd545d2b7a0} {Suffrage:Voter ID:server-286bbf55-db71-bc61-6347-72349c32f537 Address:286bbf55-db71-bc61-6347-72349c32f537}]" -testLoggerAdapter verbose: server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: 2025-02-09T06:52:37.461Z [INFO] server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: entering follower state: follower="Node at a4bd74c1-05c6-e416-9825-d5aba531a5ad [Follower]" leader= - testing.go:149: server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: 2025-02-09T06:52:37.461Z [INFO] server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: entering follower state: follower="Node at a4bd74c1-05c6-e416-9825-d5aba531a5ad [Follower]" leader= -testLoggerAdapter verbose: server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: 2025-02-09T06:52:37.461Z [INFO] server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a4bd74c1-05c6-e416-9825-d5aba531a5ad Address:a4bd74c1-05c6-e416-9825-d5aba531a5ad} {Suffrage:Voter ID:server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0 Address:408d0d66-6ddc-aa04-77f7-ccd545d2b7a0} {Suffrage:Voter ID:server-286bbf55-db71-bc61-6347-72349c32f537 Address:286bbf55-db71-bc61-6347-72349c32f537}]" - testing.go:149: server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: 2025-02-09T06:52:37.461Z [INFO] server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a4bd74c1-05c6-e416-9825-d5aba531a5ad Address:a4bd74c1-05c6-e416-9825-d5aba531a5ad} {Suffrage:Voter ID:server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0 Address:408d0d66-6ddc-aa04-77f7-ccd545d2b7a0} {Suffrage:Voter ID:server-286bbf55-db71-bc61-6347-72349c32f537 Address:286bbf55-db71-bc61-6347-72349c32f537}]" -testLoggerAdapter verbose: server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: 2025-02-09T06:52:37.461Z [INFO] server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: entering follower state: follower="Node at 408d0d66-6ddc-aa04-77f7-ccd545d2b7a0 [Follower]" leader= - testing.go:149: server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: 2025-02-09T06:52:37.461Z [INFO] server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: entering follower state: follower="Node at 408d0d66-6ddc-aa04-77f7-ccd545d2b7a0 [Follower]" leader= -testLoggerAdapter verbose: server-286bbf55-db71-bc61-6347-72349c32f537: 2025-02-09T06:52:37.461Z [INFO] server-286bbf55-db71-bc61-6347-72349c32f537: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a4bd74c1-05c6-e416-9825-d5aba531a5ad Address:a4bd74c1-05c6-e416-9825-d5aba531a5ad} {Suffrage:Voter ID:server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0 Address:408d0d66-6ddc-aa04-77f7-ccd545d2b7a0} {Suffrage:Voter ID:server-286bbf55-db71-bc61-6347-72349c32f537 Address:286bbf55-db71-bc61-6347-72349c32f537}]" - testing.go:149: server-286bbf55-db71-bc61-6347-72349c32f537: 2025-02-09T06:52:37.461Z [INFO] server-286bbf55-db71-bc61-6347-72349c32f537: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a4bd74c1-05c6-e416-9825-d5aba531a5ad Address:a4bd74c1-05c6-e416-9825-d5aba531a5ad} {Suffrage:Voter ID:server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0 Address:408d0d66-6ddc-aa04-77f7-ccd545d2b7a0} {Suffrage:Voter ID:server-286bbf55-db71-bc61-6347-72349c32f537 Address:286bbf55-db71-bc61-6347-72349c32f537}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:37.461Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:37.461Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-286bbf55-db71-bc61-6347-72349c32f537: 2025-02-09T06:52:37.461Z [INFO] server-286bbf55-db71-bc61-6347-72349c32f537: entering follower state: follower="Node at 286bbf55-db71-bc61-6347-72349c32f537 [Follower]" leader= - testing.go:149: server-286bbf55-db71-bc61-6347-72349c32f537: 2025-02-09T06:52:37.461Z [INFO] server-286bbf55-db71-bc61-6347-72349c32f537: entering follower state: follower="Node at 286bbf55-db71-bc61-6347-72349c32f537 [Follower]" leader= -testLoggerAdapter verbose: server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: 2025-02-09T06:52:37.514Z [WARN] server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: 2025-02-09T06:52:37.514Z [WARN] server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: 2025-02-09T06:52:37.514Z [INFO] server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: entering candidate state: node="Node at 408d0d66-6ddc-aa04-77f7-ccd545d2b7a0 [Candidate]" term=2 - testing.go:149: server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: 2025-02-09T06:52:37.514Z [INFO] server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: entering candidate state: node="Node at 408d0d66-6ddc-aa04-77f7-ccd545d2b7a0 [Candidate]" term=2 -testLoggerAdapter verbose: server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: 2025-02-09T06:52:37.514Z [INFO] server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: election won: tally=2 - testing.go:149: server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: 2025-02-09T06:52:37.514Z [INFO] server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: election won: tally=2 -testLoggerAdapter verbose: server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: 2025-02-09T06:52:37.514Z [INFO] server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: entering leader state: leader="Node at 408d0d66-6ddc-aa04-77f7-ccd545d2b7a0 [Leader]" - testing.go:149: server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: 2025-02-09T06:52:37.514Z [INFO] server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: entering leader state: leader="Node at 408d0d66-6ddc-aa04-77f7-ccd545d2b7a0 [Leader]" -testLoggerAdapter verbose: server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: 2025-02-09T06:52:37.514Z [INFO] server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: added peer, starting replication: peer=server-a4bd74c1-05c6-e416-9825-d5aba531a5ad - testing.go:149: server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: 2025-02-09T06:52:37.514Z [INFO] server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: added peer, starting replication: peer=server-a4bd74c1-05c6-e416-9825-d5aba531a5ad -testLoggerAdapter verbose: server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: 2025-02-09T06:52:37.514Z [INFO] server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: added peer, starting replication: peer=server-286bbf55-db71-bc61-6347-72349c32f537 - testing.go:149: server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: 2025-02-09T06:52:37.514Z [INFO] server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: added peer, starting replication: peer=server-286bbf55-db71-bc61-6347-72349c32f537 -testLoggerAdapter verbose: server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: 2025-02-09T06:52:37.514Z [INFO] server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: pipelining replication: peer="{Voter server-286bbf55-db71-bc61-6347-72349c32f537 286bbf55-db71-bc61-6347-72349c32f537}" - testing.go:149: server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: 2025-02-09T06:52:37.514Z [INFO] server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: pipelining replication: peer="{Voter server-286bbf55-db71-bc61-6347-72349c32f537 286bbf55-db71-bc61-6347-72349c32f537}" -testLoggerAdapter verbose: server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: 2025-02-09T06:52:37.514Z [INFO] server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: pipelining replication: peer="{Voter server-a4bd74c1-05c6-e416-9825-d5aba531a5ad a4bd74c1-05c6-e416-9825-d5aba531a5ad}" - testing.go:149: server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: 2025-02-09T06:52:37.514Z [INFO] server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: pipelining replication: peer="{Voter server-a4bd74c1-05c6-e416-9825-d5aba531a5ad a4bd74c1-05c6-e416-9825-d5aba531a5ad}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:37.620Z [INFO] stable state for Leader reached at 2025-02-09 06:52:37.514836443 +0000 UTC m=+18.493645223 (1 nodes), 52.899763ms from start of poll, 53.283114ms from cluster start. Timeout at 2025-02-09 06:52:37.620173734 +0000 UTC m=+18.598982494, 105.337271ms after stability - testing.go:149: cluster: 2025-02-09T06:52:37.620Z [INFO] stable state for Leader reached at 2025-02-09 06:52:37.514836443 +0000 UTC m=+18.493645223 (1 nodes), 52.899763ms from start of poll, 53.283114ms from cluster start. Timeout at 2025-02-09 06:52:37.620173734 +0000 UTC m=+18.598982494, 105.337271ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:37.620Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T06:52:37.620Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:37.725Z [INFO] stable state for Follower reached at 2025-02-09 06:52:37.620265578 +0000 UTC m=+18.599074341 (2 nodes), 825ns from start of poll, 158.712232ms from cluster start. Timeout at 2025-02-09 06:52:37.725445102 +0000 UTC m=+18.704253871, 105.17953ms after stability - testing.go:149: cluster: 2025-02-09T06:52:37.725Z [INFO] stable state for Follower reached at 2025-02-09 06:52:37.620265578 +0000 UTC m=+18.599074341 (2 nodes), 825ns from start of poll, 158.712232ms from cluster start. Timeout at 2025-02-09 06:52:37.725445102 +0000 UTC m=+18.704253871, 105.17953ms after stability -testLoggerAdapter verbose: server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: 2025-02-09T06:52:37.739Z [INFO] server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: entering candidate state: node="Node at a4bd74c1-05c6-e416-9825-d5aba531a5ad [Candidate]" term=3 - testing.go:149: server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: 2025-02-09T06:52:37.739Z [INFO] server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: entering candidate state: node="Node at a4bd74c1-05c6-e416-9825-d5aba531a5ad [Candidate]" term=3 -testLoggerAdapter verbose: server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: 2025-02-09T06:52:37.739Z [INFO] server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: election won: tally=2 - testing.go:149: server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: 2025-02-09T06:52:37.739Z [INFO] server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: election won: tally=2 -testLoggerAdapter verbose: server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: 2025-02-09T06:52:37.739Z [INFO] server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: entering leader state: leader="Node at a4bd74c1-05c6-e416-9825-d5aba531a5ad [Leader]" - testing.go:149: server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: 2025-02-09T06:52:37.739Z [INFO] server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: entering leader state: leader="Node at a4bd74c1-05c6-e416-9825-d5aba531a5ad [Leader]" -testLoggerAdapter verbose: server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: 2025-02-09T06:52:37.739Z [INFO] server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: added peer, starting replication: peer=server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0 -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:37.739Z [INFO] starting stability test: raft-state=Leader - testing.go:149: server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: 2025-02-09T06:52:37.739Z [INFO] server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: added peer, starting replication: peer=server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0 -testLoggerAdapter verbose: server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: 2025-02-09T06:52:37.739Z [INFO] server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: added peer, starting replication: peer=server-286bbf55-db71-bc61-6347-72349c32f537 -testLoggerAdapter verbose: server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: 2025-02-09T06:52:37.739Z [INFO] server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: entering follower state: follower="Node at 408d0d66-6ddc-aa04-77f7-ccd545d2b7a0 [Follower]" leader= - testing.go:149: server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: 2025-02-09T06:52:37.739Z [INFO] server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: added peer, starting replication: peer=server-286bbf55-db71-bc61-6347-72349c32f537 -testLoggerAdapter verbose: server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: 2025-02-09T06:52:37.739Z [INFO] server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: pipelining replication: peer="{Voter server-286bbf55-db71-bc61-6347-72349c32f537 286bbf55-db71-bc61-6347-72349c32f537}" - testing.go:149: server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: 2025-02-09T06:52:37.739Z [INFO] server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: entering follower state: follower="Node at 408d0d66-6ddc-aa04-77f7-ccd545d2b7a0 [Follower]" leader= -testLoggerAdapter verbose: server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: 2025-02-09T06:52:37.739Z [INFO] server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: aborting pipeline replication: peer="{Voter server-286bbf55-db71-bc61-6347-72349c32f537 286bbf55-db71-bc61-6347-72349c32f537}" - testing.go:149: cluster: 2025-02-09T06:52:37.739Z [INFO] starting stability test: raft-state=Leader - testing.go:149: server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: 2025-02-09T06:52:37.739Z [INFO] server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: pipelining replication: peer="{Voter server-286bbf55-db71-bc61-6347-72349c32f537 286bbf55-db71-bc61-6347-72349c32f537}" -testLoggerAdapter verbose: server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: 2025-02-09T06:52:37.739Z [INFO] server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: pipelining replication: peer="{Voter server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0 408d0d66-6ddc-aa04-77f7-ccd545d2b7a0}" - testing.go:149: server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: 2025-02-09T06:52:37.739Z [INFO] server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: aborting pipeline replication: peer="{Voter server-286bbf55-db71-bc61-6347-72349c32f537 286bbf55-db71-bc61-6347-72349c32f537}" -testLoggerAdapter verbose: server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: 2025-02-09T06:52:37.739Z [INFO] server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: aborting pipeline replication: peer="{Voter server-a4bd74c1-05c6-e416-9825-d5aba531a5ad a4bd74c1-05c6-e416-9825-d5aba531a5ad}" - testing.go:149: server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: 2025-02-09T06:52:37.739Z [INFO] server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: pipelining replication: peer="{Voter server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0 408d0d66-6ddc-aa04-77f7-ccd545d2b7a0}" - testing.go:149: server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: 2025-02-09T06:52:37.739Z [INFO] server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0: aborting pipeline replication: peer="{Voter server-a4bd74c1-05c6-e416-9825-d5aba531a5ad a4bd74c1-05c6-e416-9825-d5aba531a5ad}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:37.845Z [INFO] stable state for Leader reached at 2025-02-09 06:52:37.740020051 +0000 UTC m=+18.718828829 (1 nodes), 16.664µs from start of poll, 278.46672ms from cluster start. Timeout at 2025-02-09 06:52:37.845281148 +0000 UTC m=+18.824089938, 105.261109ms after stability - testing.go:149: cluster: 2025-02-09T06:52:37.845Z [INFO] stable state for Leader reached at 2025-02-09 06:52:37.740020051 +0000 UTC m=+18.718828829 (1 nodes), 16.664µs from start of poll, 278.46672ms from cluster start. Timeout at 2025-02-09 06:52:37.845281148 +0000 UTC m=+18.824089938, 105.261109ms after stability -testLoggerAdapter verbose: server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: 2025-02-09T06:52:37.845Z [INFO] server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: aborting pipeline replication: peer="{Voter server-286bbf55-db71-bc61-6347-72349c32f537 286bbf55-db71-bc61-6347-72349c32f537}" - testing.go:149: server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: 2025-02-09T06:52:37.845Z [INFO] server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: aborting pipeline replication: peer="{Voter server-286bbf55-db71-bc61-6347-72349c32f537 286bbf55-db71-bc61-6347-72349c32f537}" -testLoggerAdapter verbose: server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: 2025-02-09T06:52:37.845Z [INFO] server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: aborting pipeline replication: peer="{Voter server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0 408d0d66-6ddc-aa04-77f7-ccd545d2b7a0}" - testing.go:149: server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: 2025-02-09T06:52:37.845Z [INFO] server-a4bd74c1-05c6-e416-9825-d5aba531a5ad: aborting pipeline replication: peer="{Voter server-408d0d66-6ddc-aa04-77f7-ccd545d2b7a0 408d0d66-6ddc-aa04-77f7-ccd545d2b7a0}" ---- PASS: TestRaft_LeadershipTransferToBehindServer (0.38s) +testLoggerAdapter verbose: server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: 2024-01-08T00:31:12.928Z [INFO] server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923 Address:f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923} {Suffrage:Voter ID:server-73e1560a-14e4-b7ce-2078-a5c45305953d Address:73e1560a-14e4-b7ce-2078-a5c45305953d} {Suffrage:Voter ID:server-120b2497-465a-c4b2-0183-8928dbc276bb Address:120b2497-465a-c4b2-0183-8928dbc276bb}]" + testing.go:149: server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: 2024-01-08T00:31:12.928Z [INFO] server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923 Address:f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923} {Suffrage:Voter ID:server-73e1560a-14e4-b7ce-2078-a5c45305953d Address:73e1560a-14e4-b7ce-2078-a5c45305953d} {Suffrage:Voter ID:server-120b2497-465a-c4b2-0183-8928dbc276bb Address:120b2497-465a-c4b2-0183-8928dbc276bb}]" +testLoggerAdapter verbose: server-73e1560a-14e4-b7ce-2078-a5c45305953d: 2024-01-08T00:31:12.928Z [INFO] server-73e1560a-14e4-b7ce-2078-a5c45305953d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923 Address:f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923} {Suffrage:Voter ID:server-73e1560a-14e4-b7ce-2078-a5c45305953d Address:73e1560a-14e4-b7ce-2078-a5c45305953d} {Suffrage:Voter ID:server-120b2497-465a-c4b2-0183-8928dbc276bb Address:120b2497-465a-c4b2-0183-8928dbc276bb}]" + testing.go:149: server-73e1560a-14e4-b7ce-2078-a5c45305953d: 2024-01-08T00:31:12.928Z [INFO] server-73e1560a-14e4-b7ce-2078-a5c45305953d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923 Address:f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923} {Suffrage:Voter ID:server-73e1560a-14e4-b7ce-2078-a5c45305953d Address:73e1560a-14e4-b7ce-2078-a5c45305953d} {Suffrage:Voter ID:server-120b2497-465a-c4b2-0183-8928dbc276bb Address:120b2497-465a-c4b2-0183-8928dbc276bb}]" +testLoggerAdapter verbose: server-120b2497-465a-c4b2-0183-8928dbc276bb: 2024-01-08T00:31:12.928Z [INFO] server-120b2497-465a-c4b2-0183-8928dbc276bb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923 Address:f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923} {Suffrage:Voter ID:server-73e1560a-14e4-b7ce-2078-a5c45305953d Address:73e1560a-14e4-b7ce-2078-a5c45305953d} {Suffrage:Voter ID:server-120b2497-465a-c4b2-0183-8928dbc276bb Address:120b2497-465a-c4b2-0183-8928dbc276bb}]" + testing.go:149: server-120b2497-465a-c4b2-0183-8928dbc276bb: 2024-01-08T00:31:12.928Z [INFO] server-120b2497-465a-c4b2-0183-8928dbc276bb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923 Address:f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923} {Suffrage:Voter ID:server-73e1560a-14e4-b7ce-2078-a5c45305953d Address:73e1560a-14e4-b7ce-2078-a5c45305953d} {Suffrage:Voter ID:server-120b2497-465a-c4b2-0183-8928dbc276bb Address:120b2497-465a-c4b2-0183-8928dbc276bb}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:12.928Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:12.928Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: 2024-01-08T00:31:12.928Z [INFO] server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: entering follower state: follower="Node at f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923 [Follower]" leader= + testing.go:149: server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: 2024-01-08T00:31:12.928Z [INFO] server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: entering follower state: follower="Node at f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923 [Follower]" leader= +testLoggerAdapter verbose: server-73e1560a-14e4-b7ce-2078-a5c45305953d: 2024-01-08T00:31:12.928Z [INFO] server-73e1560a-14e4-b7ce-2078-a5c45305953d: entering follower state: follower="Node at 73e1560a-14e4-b7ce-2078-a5c45305953d [Follower]" leader= + testing.go:149: server-73e1560a-14e4-b7ce-2078-a5c45305953d: 2024-01-08T00:31:12.928Z [INFO] server-73e1560a-14e4-b7ce-2078-a5c45305953d: entering follower state: follower="Node at 73e1560a-14e4-b7ce-2078-a5c45305953d [Follower]" leader= +testLoggerAdapter verbose: server-120b2497-465a-c4b2-0183-8928dbc276bb: 2024-01-08T00:31:12.928Z [INFO] server-120b2497-465a-c4b2-0183-8928dbc276bb: entering follower state: follower="Node at 120b2497-465a-c4b2-0183-8928dbc276bb [Follower]" leader= + testing.go:149: server-120b2497-465a-c4b2-0183-8928dbc276bb: 2024-01-08T00:31:12.928Z [INFO] server-120b2497-465a-c4b2-0183-8928dbc276bb: entering follower state: follower="Node at 120b2497-465a-c4b2-0183-8928dbc276bb [Follower]" leader= +testLoggerAdapter verbose: server-73e1560a-14e4-b7ce-2078-a5c45305953d: 2024-01-08T00:31:12.980Z [WARN] server-73e1560a-14e4-b7ce-2078-a5c45305953d: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-73e1560a-14e4-b7ce-2078-a5c45305953d: 2024-01-08T00:31:12.980Z [WARN] server-73e1560a-14e4-b7ce-2078-a5c45305953d: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-73e1560a-14e4-b7ce-2078-a5c45305953d: 2024-01-08T00:31:12.980Z [INFO] server-73e1560a-14e4-b7ce-2078-a5c45305953d: entering candidate state: node="Node at 73e1560a-14e4-b7ce-2078-a5c45305953d [Candidate]" term=2 + testing.go:149: server-73e1560a-14e4-b7ce-2078-a5c45305953d: 2024-01-08T00:31:12.980Z [INFO] server-73e1560a-14e4-b7ce-2078-a5c45305953d: entering candidate state: node="Node at 73e1560a-14e4-b7ce-2078-a5c45305953d [Candidate]" term=2 +testLoggerAdapter verbose: server-73e1560a-14e4-b7ce-2078-a5c45305953d: 2024-01-08T00:31:12.980Z [INFO] server-73e1560a-14e4-b7ce-2078-a5c45305953d: election won: tally=2 + testing.go:149: server-73e1560a-14e4-b7ce-2078-a5c45305953d: 2024-01-08T00:31:12.980Z [INFO] server-73e1560a-14e4-b7ce-2078-a5c45305953d: election won: tally=2 +testLoggerAdapter verbose: server-73e1560a-14e4-b7ce-2078-a5c45305953d: 2024-01-08T00:31:12.980Z [INFO] server-73e1560a-14e4-b7ce-2078-a5c45305953d: entering leader state: leader="Node at 73e1560a-14e4-b7ce-2078-a5c45305953d [Leader]" + testing.go:149: server-73e1560a-14e4-b7ce-2078-a5c45305953d: 2024-01-08T00:31:12.980Z [INFO] server-73e1560a-14e4-b7ce-2078-a5c45305953d: entering leader state: leader="Node at 73e1560a-14e4-b7ce-2078-a5c45305953d [Leader]" +testLoggerAdapter verbose: server-73e1560a-14e4-b7ce-2078-a5c45305953d: 2024-01-08T00:31:12.980Z [INFO] server-73e1560a-14e4-b7ce-2078-a5c45305953d: added peer, starting replication: peer=server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923 + testing.go:149: server-73e1560a-14e4-b7ce-2078-a5c45305953d: 2024-01-08T00:31:12.980Z [INFO] server-73e1560a-14e4-b7ce-2078-a5c45305953d: added peer, starting replication: peer=server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923 +testLoggerAdapter verbose: server-73e1560a-14e4-b7ce-2078-a5c45305953d: 2024-01-08T00:31:12.980Z [INFO] server-73e1560a-14e4-b7ce-2078-a5c45305953d: added peer, starting replication: peer=server-120b2497-465a-c4b2-0183-8928dbc276bb + testing.go:149: server-73e1560a-14e4-b7ce-2078-a5c45305953d: 2024-01-08T00:31:12.980Z [INFO] server-73e1560a-14e4-b7ce-2078-a5c45305953d: added peer, starting replication: peer=server-120b2497-465a-c4b2-0183-8928dbc276bb +testLoggerAdapter verbose: server-73e1560a-14e4-b7ce-2078-a5c45305953d: 2024-01-08T00:31:12.981Z [INFO] server-73e1560a-14e4-b7ce-2078-a5c45305953d: pipelining replication: peer="{Voter server-120b2497-465a-c4b2-0183-8928dbc276bb 120b2497-465a-c4b2-0183-8928dbc276bb}" + testing.go:149: server-73e1560a-14e4-b7ce-2078-a5c45305953d: 2024-01-08T00:31:12.981Z [INFO] server-73e1560a-14e4-b7ce-2078-a5c45305953d: pipelining replication: peer="{Voter server-120b2497-465a-c4b2-0183-8928dbc276bb 120b2497-465a-c4b2-0183-8928dbc276bb}" +testLoggerAdapter verbose: server-73e1560a-14e4-b7ce-2078-a5c45305953d: 2024-01-08T00:31:12.981Z [INFO] server-73e1560a-14e4-b7ce-2078-a5c45305953d: pipelining replication: peer="{Voter server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923 f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923}" + testing.go:149: server-73e1560a-14e4-b7ce-2078-a5c45305953d: 2024-01-08T00:31:12.981Z [INFO] server-73e1560a-14e4-b7ce-2078-a5c45305953d: pipelining replication: peer="{Voter server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923 f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:13.087Z [INFO] stable state for Leader reached at 2024-01-08 00:31:12.981258051 +0000 UTC m=+19.028147108 (1 nodes), 52.566353ms from start of poll, 53.048673ms from cluster start. Timeout at 2024-01-08 00:31:13.087154202 +0000 UTC m=+19.134043318, 105.89621ms after stability + testing.go:149: cluster: 2024-01-08T00:31:13.087Z [INFO] stable state for Leader reached at 2024-01-08 00:31:12.981258051 +0000 UTC m=+19.028147108 (1 nodes), 52.566353ms from start of poll, 53.048673ms from cluster start. Timeout at 2024-01-08 00:31:13.087154202 +0000 UTC m=+19.134043318, 105.89621ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:13.087Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T00:31:13.087Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:13.192Z [INFO] stable state for Follower reached at 2024-01-08 00:31:13.087309318 +0000 UTC m=+19.134198445 (2 nodes), 2.454µs from start of poll, 159.10001ms from cluster start. Timeout at 2024-01-08 00:31:13.192347804 +0000 UTC m=+19.239236927, 105.038482ms after stability + testing.go:149: cluster: 2024-01-08T00:31:13.192Z [INFO] stable state for Follower reached at 2024-01-08 00:31:13.087309318 +0000 UTC m=+19.134198445 (2 nodes), 2.454µs from start of poll, 159.10001ms from cluster start. Timeout at 2024-01-08 00:31:13.192347804 +0000 UTC m=+19.239236927, 105.038482ms after stability +testLoggerAdapter verbose: server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: 2024-01-08T00:31:13.221Z [INFO] server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: entering candidate state: node="Node at f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923 [Candidate]" term=3 + testing.go:149: server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: 2024-01-08T00:31:13.221Z [INFO] server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: entering candidate state: node="Node at f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923 [Candidate]" term=3 +testLoggerAdapter verbose: server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: 2024-01-08T00:31:13.221Z [INFO] server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: election won: tally=2 + testing.go:149: server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: 2024-01-08T00:31:13.221Z [INFO] server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: election won: tally=2 +testLoggerAdapter verbose: server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: 2024-01-08T00:31:13.221Z [INFO] server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: entering leader state: leader="Node at f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923 [Leader]" + testing.go:149: server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: 2024-01-08T00:31:13.221Z [INFO] server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: entering leader state: leader="Node at f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923 [Leader]" +testLoggerAdapter verbose: server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: 2024-01-08T00:31:13.221Z [INFO] server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: added peer, starting replication: peer=server-73e1560a-14e4-b7ce-2078-a5c45305953d + testing.go:149: server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: 2024-01-08T00:31:13.221Z [INFO] server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: added peer, starting replication: peer=server-73e1560a-14e4-b7ce-2078-a5c45305953d +testLoggerAdapter verbose: server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: 2024-01-08T00:31:13.221Z [INFO] server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: added peer, starting replication: peer=server-120b2497-465a-c4b2-0183-8928dbc276bb + testing.go:149: server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: 2024-01-08T00:31:13.221Z [INFO] server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: added peer, starting replication: peer=server-120b2497-465a-c4b2-0183-8928dbc276bb +testLoggerAdapter verbose: server-120b2497-465a-c4b2-0183-8928dbc276bb: 2024-01-08T00:31:13.221Z [WARN] server-120b2497-465a-c4b2-0183-8928dbc276bb: failed to get previous log: previous-index=1002 last-index=822 error="log not found" + testing.go:149: server-120b2497-465a-c4b2-0183-8928dbc276bb: 2024-01-08T00:31:13.221Z [WARN] server-120b2497-465a-c4b2-0183-8928dbc276bb: failed to get previous log: previous-index=1002 last-index=822 error="log not found" +testLoggerAdapter verbose: server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: 2024-01-08T00:31:13.221Z [WARN] server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: appendEntries rejected, sending older logs: peer="{Voter server-120b2497-465a-c4b2-0183-8928dbc276bb 120b2497-465a-c4b2-0183-8928dbc276bb}" next=823 + testing.go:149: server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: 2024-01-08T00:31:13.221Z [WARN] server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: appendEntries rejected, sending older logs: peer="{Voter server-120b2497-465a-c4b2-0183-8928dbc276bb 120b2497-465a-c4b2-0183-8928dbc276bb}" next=823 +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:13.221Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:13.221Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: 2024-01-08T00:31:13.221Z [INFO] server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: pipelining replication: peer="{Voter server-120b2497-465a-c4b2-0183-8928dbc276bb 120b2497-465a-c4b2-0183-8928dbc276bb}" + testing.go:149: server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: 2024-01-08T00:31:13.221Z [INFO] server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: pipelining replication: peer="{Voter server-120b2497-465a-c4b2-0183-8928dbc276bb 120b2497-465a-c4b2-0183-8928dbc276bb}" +testLoggerAdapter verbose: server-73e1560a-14e4-b7ce-2078-a5c45305953d: 2024-01-08T00:31:13.222Z [INFO] server-73e1560a-14e4-b7ce-2078-a5c45305953d: entering follower state: follower="Node at 73e1560a-14e4-b7ce-2078-a5c45305953d [Follower]" leader= + testing.go:149: server-73e1560a-14e4-b7ce-2078-a5c45305953d: 2024-01-08T00:31:13.222Z [INFO] server-73e1560a-14e4-b7ce-2078-a5c45305953d: entering follower state: follower="Node at 73e1560a-14e4-b7ce-2078-a5c45305953d [Follower]" leader= +testLoggerAdapter verbose: server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: 2024-01-08T00:31:13.222Z [INFO] server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: pipelining replication: peer="{Voter server-73e1560a-14e4-b7ce-2078-a5c45305953d 73e1560a-14e4-b7ce-2078-a5c45305953d}" + testing.go:149: server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: 2024-01-08T00:31:13.222Z [INFO] server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: pipelining replication: peer="{Voter server-73e1560a-14e4-b7ce-2078-a5c45305953d 73e1560a-14e4-b7ce-2078-a5c45305953d}" +testLoggerAdapter verbose: server-73e1560a-14e4-b7ce-2078-a5c45305953d: 2024-01-08T00:31:13.223Z [INFO] server-73e1560a-14e4-b7ce-2078-a5c45305953d: aborting pipeline replication: peer="{Voter server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923 f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923}" + testing.go:149: server-73e1560a-14e4-b7ce-2078-a5c45305953d: 2024-01-08T00:31:13.223Z [INFO] server-73e1560a-14e4-b7ce-2078-a5c45305953d: aborting pipeline replication: peer="{Voter server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923 f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923}" +testLoggerAdapter verbose: server-73e1560a-14e4-b7ce-2078-a5c45305953d: 2024-01-08T00:31:13.223Z [INFO] server-73e1560a-14e4-b7ce-2078-a5c45305953d: aborting pipeline replication: peer="{Voter server-120b2497-465a-c4b2-0183-8928dbc276bb 120b2497-465a-c4b2-0183-8928dbc276bb}" + testing.go:149: server-73e1560a-14e4-b7ce-2078-a5c45305953d: 2024-01-08T00:31:13.223Z [INFO] server-73e1560a-14e4-b7ce-2078-a5c45305953d: aborting pipeline replication: peer="{Voter server-120b2497-465a-c4b2-0183-8928dbc276bb 120b2497-465a-c4b2-0183-8928dbc276bb}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:13.329Z [INFO] stable state for Leader reached at 2024-01-08 00:31:13.223024714 +0000 UTC m=+19.269913933 (1 nodes), 1.290673ms from start of poll, 294.815498ms from cluster start. Timeout at 2024-01-08 00:31:13.329202411 +0000 UTC m=+19.376091552, 106.177619ms after stability + testing.go:149: cluster: 2024-01-08T00:31:13.329Z [INFO] stable state for Leader reached at 2024-01-08 00:31:13.223024714 +0000 UTC m=+19.269913933 (1 nodes), 1.290673ms from start of poll, 294.815498ms from cluster start. Timeout at 2024-01-08 00:31:13.329202411 +0000 UTC m=+19.376091552, 106.177619ms after stability +testLoggerAdapter verbose: server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: 2024-01-08T00:31:13.329Z [INFO] server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: aborting pipeline replication: peer="{Voter server-73e1560a-14e4-b7ce-2078-a5c45305953d 73e1560a-14e4-b7ce-2078-a5c45305953d}" + testing.go:149: server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: 2024-01-08T00:31:13.329Z [INFO] server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: aborting pipeline replication: peer="{Voter server-73e1560a-14e4-b7ce-2078-a5c45305953d 73e1560a-14e4-b7ce-2078-a5c45305953d}" +testLoggerAdapter verbose: server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: 2024-01-08T00:31:13.329Z [INFO] server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: aborting pipeline replication: peer="{Voter server-120b2497-465a-c4b2-0183-8928dbc276bb 120b2497-465a-c4b2-0183-8928dbc276bb}" + testing.go:149: server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: 2024-01-08T00:31:13.329Z [INFO] server-f70c3fd0-bdc9-2c25-b3ed-e1fe3fa08923: aborting pipeline replication: peer="{Voter server-120b2497-465a-c4b2-0183-8928dbc276bb 120b2497-465a-c4b2-0183-8928dbc276bb}" +--- PASS: TestRaft_LeadershipTransferToBehindServer (0.40s) === RUN TestRaft_LeadershipTransferToItself -testLoggerAdapter verbose: server-53d5b798-497b-8eca-effc-e3da80dadfb5: 2025-02-09T06:52:37.846Z [INFO] server-53d5b798-497b-8eca-effc-e3da80dadfb5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53d5b798-497b-8eca-effc-e3da80dadfb5 Address:53d5b798-497b-8eca-effc-e3da80dadfb5} {Suffrage:Voter ID:server-9e5d54b2-92b2-1335-87b8-950f94e7c927 Address:9e5d54b2-92b2-1335-87b8-950f94e7c927} {Suffrage:Voter ID:server-f3695af8-c6ce-5ef1-5d0e-689a4594682b Address:f3695af8-c6ce-5ef1-5d0e-689a4594682b}]" - testing.go:149: server-53d5b798-497b-8eca-effc-e3da80dadfb5: 2025-02-09T06:52:37.846Z [INFO] server-53d5b798-497b-8eca-effc-e3da80dadfb5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53d5b798-497b-8eca-effc-e3da80dadfb5 Address:53d5b798-497b-8eca-effc-e3da80dadfb5} {Suffrage:Voter ID:server-9e5d54b2-92b2-1335-87b8-950f94e7c927 Address:9e5d54b2-92b2-1335-87b8-950f94e7c927} {Suffrage:Voter ID:server-f3695af8-c6ce-5ef1-5d0e-689a4594682b Address:f3695af8-c6ce-5ef1-5d0e-689a4594682b}]" -testLoggerAdapter verbose: server-53d5b798-497b-8eca-effc-e3da80dadfb5: 2025-02-09T06:52:37.846Z [INFO] server-53d5b798-497b-8eca-effc-e3da80dadfb5: entering follower state: follower="Node at 53d5b798-497b-8eca-effc-e3da80dadfb5 [Follower]" leader= - testing.go:149: server-53d5b798-497b-8eca-effc-e3da80dadfb5: 2025-02-09T06:52:37.846Z [INFO] server-53d5b798-497b-8eca-effc-e3da80dadfb5: entering follower state: follower="Node at 53d5b798-497b-8eca-effc-e3da80dadfb5 [Follower]" leader= -testLoggerAdapter verbose: server-9e5d54b2-92b2-1335-87b8-950f94e7c927: 2025-02-09T06:52:37.846Z [INFO] server-9e5d54b2-92b2-1335-87b8-950f94e7c927: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53d5b798-497b-8eca-effc-e3da80dadfb5 Address:53d5b798-497b-8eca-effc-e3da80dadfb5} {Suffrage:Voter ID:server-9e5d54b2-92b2-1335-87b8-950f94e7c927 Address:9e5d54b2-92b2-1335-87b8-950f94e7c927} {Suffrage:Voter ID:server-f3695af8-c6ce-5ef1-5d0e-689a4594682b Address:f3695af8-c6ce-5ef1-5d0e-689a4594682b}]" - testing.go:149: server-9e5d54b2-92b2-1335-87b8-950f94e7c927: 2025-02-09T06:52:37.846Z [INFO] server-9e5d54b2-92b2-1335-87b8-950f94e7c927: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53d5b798-497b-8eca-effc-e3da80dadfb5 Address:53d5b798-497b-8eca-effc-e3da80dadfb5} {Suffrage:Voter ID:server-9e5d54b2-92b2-1335-87b8-950f94e7c927 Address:9e5d54b2-92b2-1335-87b8-950f94e7c927} {Suffrage:Voter ID:server-f3695af8-c6ce-5ef1-5d0e-689a4594682b Address:f3695af8-c6ce-5ef1-5d0e-689a4594682b}]" -testLoggerAdapter verbose: server-9e5d54b2-92b2-1335-87b8-950f94e7c927: 2025-02-09T06:52:37.846Z [INFO] server-9e5d54b2-92b2-1335-87b8-950f94e7c927: entering follower state: follower="Node at 9e5d54b2-92b2-1335-87b8-950f94e7c927 [Follower]" leader= -testLoggerAdapter verbose: server-f3695af8-c6ce-5ef1-5d0e-689a4594682b: 2025-02-09T06:52:37.846Z [INFO] server-f3695af8-c6ce-5ef1-5d0e-689a4594682b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53d5b798-497b-8eca-effc-e3da80dadfb5 Address:53d5b798-497b-8eca-effc-e3da80dadfb5} {Suffrage:Voter ID:server-9e5d54b2-92b2-1335-87b8-950f94e7c927 Address:9e5d54b2-92b2-1335-87b8-950f94e7c927} {Suffrage:Voter ID:server-f3695af8-c6ce-5ef1-5d0e-689a4594682b Address:f3695af8-c6ce-5ef1-5d0e-689a4594682b}]" - testing.go:149: server-9e5d54b2-92b2-1335-87b8-950f94e7c927: 2025-02-09T06:52:37.846Z [INFO] server-9e5d54b2-92b2-1335-87b8-950f94e7c927: entering follower state: follower="Node at 9e5d54b2-92b2-1335-87b8-950f94e7c927 [Follower]" leader= - testing.go:149: server-f3695af8-c6ce-5ef1-5d0e-689a4594682b: 2025-02-09T06:52:37.846Z [INFO] server-f3695af8-c6ce-5ef1-5d0e-689a4594682b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53d5b798-497b-8eca-effc-e3da80dadfb5 Address:53d5b798-497b-8eca-effc-e3da80dadfb5} {Suffrage:Voter ID:server-9e5d54b2-92b2-1335-87b8-950f94e7c927 Address:9e5d54b2-92b2-1335-87b8-950f94e7c927} {Suffrage:Voter ID:server-f3695af8-c6ce-5ef1-5d0e-689a4594682b Address:f3695af8-c6ce-5ef1-5d0e-689a4594682b}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:37.846Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:37.846Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-f3695af8-c6ce-5ef1-5d0e-689a4594682b: 2025-02-09T06:52:37.846Z [INFO] server-f3695af8-c6ce-5ef1-5d0e-689a4594682b: entering follower state: follower="Node at f3695af8-c6ce-5ef1-5d0e-689a4594682b [Follower]" leader= - testing.go:149: server-f3695af8-c6ce-5ef1-5d0e-689a4594682b: 2025-02-09T06:52:37.846Z [INFO] server-f3695af8-c6ce-5ef1-5d0e-689a4594682b: entering follower state: follower="Node at f3695af8-c6ce-5ef1-5d0e-689a4594682b [Follower]" leader= -testLoggerAdapter verbose: server-9e5d54b2-92b2-1335-87b8-950f94e7c927: 2025-02-09T06:52:37.905Z [WARN] server-9e5d54b2-92b2-1335-87b8-950f94e7c927: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-9e5d54b2-92b2-1335-87b8-950f94e7c927: 2025-02-09T06:52:37.905Z [WARN] server-9e5d54b2-92b2-1335-87b8-950f94e7c927: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-9e5d54b2-92b2-1335-87b8-950f94e7c927: 2025-02-09T06:52:37.905Z [INFO] server-9e5d54b2-92b2-1335-87b8-950f94e7c927: entering candidate state: node="Node at 9e5d54b2-92b2-1335-87b8-950f94e7c927 [Candidate]" term=2 - testing.go:149: server-9e5d54b2-92b2-1335-87b8-950f94e7c927: 2025-02-09T06:52:37.905Z [INFO] server-9e5d54b2-92b2-1335-87b8-950f94e7c927: entering candidate state: node="Node at 9e5d54b2-92b2-1335-87b8-950f94e7c927 [Candidate]" term=2 -testLoggerAdapter verbose: server-9e5d54b2-92b2-1335-87b8-950f94e7c927: 2025-02-09T06:52:37.905Z [INFO] server-9e5d54b2-92b2-1335-87b8-950f94e7c927: election won: tally=2 - testing.go:149: server-9e5d54b2-92b2-1335-87b8-950f94e7c927: 2025-02-09T06:52:37.905Z [INFO] server-9e5d54b2-92b2-1335-87b8-950f94e7c927: election won: tally=2 -testLoggerAdapter verbose: server-9e5d54b2-92b2-1335-87b8-950f94e7c927: 2025-02-09T06:52:37.906Z [INFO] server-9e5d54b2-92b2-1335-87b8-950f94e7c927: entering leader state: leader="Node at 9e5d54b2-92b2-1335-87b8-950f94e7c927 [Leader]" - testing.go:149: server-9e5d54b2-92b2-1335-87b8-950f94e7c927: 2025-02-09T06:52:37.906Z [INFO] server-9e5d54b2-92b2-1335-87b8-950f94e7c927: entering leader state: leader="Node at 9e5d54b2-92b2-1335-87b8-950f94e7c927 [Leader]" -testLoggerAdapter verbose: server-9e5d54b2-92b2-1335-87b8-950f94e7c927: 2025-02-09T06:52:37.906Z [INFO] server-9e5d54b2-92b2-1335-87b8-950f94e7c927: added peer, starting replication: peer=server-53d5b798-497b-8eca-effc-e3da80dadfb5 - testing.go:149: server-9e5d54b2-92b2-1335-87b8-950f94e7c927: 2025-02-09T06:52:37.906Z [INFO] server-9e5d54b2-92b2-1335-87b8-950f94e7c927: added peer, starting replication: peer=server-53d5b798-497b-8eca-effc-e3da80dadfb5 -testLoggerAdapter verbose: server-9e5d54b2-92b2-1335-87b8-950f94e7c927: 2025-02-09T06:52:37.906Z [INFO] server-9e5d54b2-92b2-1335-87b8-950f94e7c927: added peer, starting replication: peer=server-f3695af8-c6ce-5ef1-5d0e-689a4594682b - testing.go:149: server-9e5d54b2-92b2-1335-87b8-950f94e7c927: 2025-02-09T06:52:37.906Z [INFO] server-9e5d54b2-92b2-1335-87b8-950f94e7c927: added peer, starting replication: peer=server-f3695af8-c6ce-5ef1-5d0e-689a4594682b -testLoggerAdapter verbose: server-9e5d54b2-92b2-1335-87b8-950f94e7c927: 2025-02-09T06:52:37.906Z [INFO] server-9e5d54b2-92b2-1335-87b8-950f94e7c927: pipelining replication: peer="{Voter server-53d5b798-497b-8eca-effc-e3da80dadfb5 53d5b798-497b-8eca-effc-e3da80dadfb5}" - testing.go:149: server-9e5d54b2-92b2-1335-87b8-950f94e7c927: 2025-02-09T06:52:37.906Z [INFO] server-9e5d54b2-92b2-1335-87b8-950f94e7c927: pipelining replication: peer="{Voter server-53d5b798-497b-8eca-effc-e3da80dadfb5 53d5b798-497b-8eca-effc-e3da80dadfb5}" -testLoggerAdapter verbose: server-9e5d54b2-92b2-1335-87b8-950f94e7c927: 2025-02-09T06:52:37.906Z [INFO] server-9e5d54b2-92b2-1335-87b8-950f94e7c927: pipelining replication: peer="{Voter server-f3695af8-c6ce-5ef1-5d0e-689a4594682b f3695af8-c6ce-5ef1-5d0e-689a4594682b}" - testing.go:149: server-9e5d54b2-92b2-1335-87b8-950f94e7c927: 2025-02-09T06:52:37.906Z [INFO] server-9e5d54b2-92b2-1335-87b8-950f94e7c927: pipelining replication: peer="{Voter server-f3695af8-c6ce-5ef1-5d0e-689a4594682b f3695af8-c6ce-5ef1-5d0e-689a4594682b}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:38.011Z [INFO] stable state for Leader reached at 2025-02-09 06:52:37.906052933 +0000 UTC m=+18.884861711 (1 nodes), 59.27485ms from start of poll, 59.682457ms from cluster start. Timeout at 2025-02-09 06:52:38.011621996 +0000 UTC m=+18.990430764, 105.569053ms after stability - testing.go:149: cluster: 2025-02-09T06:52:38.011Z [INFO] stable state for Leader reached at 2025-02-09 06:52:37.906052933 +0000 UTC m=+18.884861711 (1 nodes), 59.27485ms from start of poll, 59.682457ms from cluster start. Timeout at 2025-02-09 06:52:38.011621996 +0000 UTC m=+18.990430764, 105.569053ms after stability -testLoggerAdapter verbose: server-9e5d54b2-92b2-1335-87b8-950f94e7c927: 2025-02-09T06:52:38.011Z [INFO] server-9e5d54b2-92b2-1335-87b8-950f94e7c927: cannot transfer leadership to itself - testing.go:149: server-9e5d54b2-92b2-1335-87b8-950f94e7c927: 2025-02-09T06:52:38.011Z [INFO] server-9e5d54b2-92b2-1335-87b8-950f94e7c927: cannot transfer leadership to itself -testLoggerAdapter verbose: server-9e5d54b2-92b2-1335-87b8-950f94e7c927: 2025-02-09T06:52:38.011Z [INFO] server-9e5d54b2-92b2-1335-87b8-950f94e7c927: aborting pipeline replication: peer="{Voter server-f3695af8-c6ce-5ef1-5d0e-689a4594682b f3695af8-c6ce-5ef1-5d0e-689a4594682b}" - testing.go:149: server-9e5d54b2-92b2-1335-87b8-950f94e7c927: 2025-02-09T06:52:38.011Z [INFO] server-9e5d54b2-92b2-1335-87b8-950f94e7c927: aborting pipeline replication: peer="{Voter server-f3695af8-c6ce-5ef1-5d0e-689a4594682b f3695af8-c6ce-5ef1-5d0e-689a4594682b}" -testLoggerAdapter verbose: server-9e5d54b2-92b2-1335-87b8-950f94e7c927: 2025-02-09T06:52:38.011Z [INFO] server-9e5d54b2-92b2-1335-87b8-950f94e7c927: aborting pipeline replication: peer="{Voter server-53d5b798-497b-8eca-effc-e3da80dadfb5 53d5b798-497b-8eca-effc-e3da80dadfb5}" - testing.go:149: server-9e5d54b2-92b2-1335-87b8-950f94e7c927: 2025-02-09T06:52:38.011Z [INFO] server-9e5d54b2-92b2-1335-87b8-950f94e7c927: aborting pipeline replication: peer="{Voter server-53d5b798-497b-8eca-effc-e3da80dadfb5 53d5b798-497b-8eca-effc-e3da80dadfb5}" ---- PASS: TestRaft_LeadershipTransferToItself (0.17s) +testLoggerAdapter verbose: server-9f315898-dfb9-f03d-4e0a-5ad8dc1a928e: 2024-01-08T00:31:13.330Z [INFO] server-9f315898-dfb9-f03d-4e0a-5ad8dc1a928e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9f315898-dfb9-f03d-4e0a-5ad8dc1a928e Address:9f315898-dfb9-f03d-4e0a-5ad8dc1a928e} {Suffrage:Voter ID:server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6 Address:6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6} {Suffrage:Voter ID:server-eabf3825-ed24-b923-ae5c-dbc482dc87b0 Address:eabf3825-ed24-b923-ae5c-dbc482dc87b0}]" + testing.go:149: server-9f315898-dfb9-f03d-4e0a-5ad8dc1a928e: 2024-01-08T00:31:13.330Z [INFO] server-9f315898-dfb9-f03d-4e0a-5ad8dc1a928e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9f315898-dfb9-f03d-4e0a-5ad8dc1a928e Address:9f315898-dfb9-f03d-4e0a-5ad8dc1a928e} {Suffrage:Voter ID:server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6 Address:6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6} {Suffrage:Voter ID:server-eabf3825-ed24-b923-ae5c-dbc482dc87b0 Address:eabf3825-ed24-b923-ae5c-dbc482dc87b0}]" +testLoggerAdapter verbose: server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: 2024-01-08T00:31:13.330Z [INFO] server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9f315898-dfb9-f03d-4e0a-5ad8dc1a928e Address:9f315898-dfb9-f03d-4e0a-5ad8dc1a928e} {Suffrage:Voter ID:server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6 Address:6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6} {Suffrage:Voter ID:server-eabf3825-ed24-b923-ae5c-dbc482dc87b0 Address:eabf3825-ed24-b923-ae5c-dbc482dc87b0}]" + testing.go:149: server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: 2024-01-08T00:31:13.330Z [INFO] server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9f315898-dfb9-f03d-4e0a-5ad8dc1a928e Address:9f315898-dfb9-f03d-4e0a-5ad8dc1a928e} {Suffrage:Voter ID:server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6 Address:6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6} {Suffrage:Voter ID:server-eabf3825-ed24-b923-ae5c-dbc482dc87b0 Address:eabf3825-ed24-b923-ae5c-dbc482dc87b0}]" +testLoggerAdapter verbose: server-eabf3825-ed24-b923-ae5c-dbc482dc87b0: 2024-01-08T00:31:13.331Z [INFO] server-eabf3825-ed24-b923-ae5c-dbc482dc87b0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9f315898-dfb9-f03d-4e0a-5ad8dc1a928e Address:9f315898-dfb9-f03d-4e0a-5ad8dc1a928e} {Suffrage:Voter ID:server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6 Address:6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6} {Suffrage:Voter ID:server-eabf3825-ed24-b923-ae5c-dbc482dc87b0 Address:eabf3825-ed24-b923-ae5c-dbc482dc87b0}]" + testing.go:149: server-eabf3825-ed24-b923-ae5c-dbc482dc87b0: 2024-01-08T00:31:13.331Z [INFO] server-eabf3825-ed24-b923-ae5c-dbc482dc87b0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9f315898-dfb9-f03d-4e0a-5ad8dc1a928e Address:9f315898-dfb9-f03d-4e0a-5ad8dc1a928e} {Suffrage:Voter ID:server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6 Address:6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6} {Suffrage:Voter ID:server-eabf3825-ed24-b923-ae5c-dbc482dc87b0 Address:eabf3825-ed24-b923-ae5c-dbc482dc87b0}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:13.331Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:13.331Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-9f315898-dfb9-f03d-4e0a-5ad8dc1a928e: 2024-01-08T00:31:13.331Z [INFO] server-9f315898-dfb9-f03d-4e0a-5ad8dc1a928e: entering follower state: follower="Node at 9f315898-dfb9-f03d-4e0a-5ad8dc1a928e [Follower]" leader= + testing.go:149: server-9f315898-dfb9-f03d-4e0a-5ad8dc1a928e: 2024-01-08T00:31:13.331Z [INFO] server-9f315898-dfb9-f03d-4e0a-5ad8dc1a928e: entering follower state: follower="Node at 9f315898-dfb9-f03d-4e0a-5ad8dc1a928e [Follower]" leader= +testLoggerAdapter verbose: server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: 2024-01-08T00:31:13.331Z [INFO] server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: entering follower state: follower="Node at 6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6 [Follower]" leader= + testing.go:149: server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: 2024-01-08T00:31:13.331Z [INFO] server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: entering follower state: follower="Node at 6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6 [Follower]" leader= +testLoggerAdapter verbose: server-eabf3825-ed24-b923-ae5c-dbc482dc87b0: 2024-01-08T00:31:13.331Z [INFO] server-eabf3825-ed24-b923-ae5c-dbc482dc87b0: entering follower state: follower="Node at eabf3825-ed24-b923-ae5c-dbc482dc87b0 [Follower]" leader= + testing.go:149: server-eabf3825-ed24-b923-ae5c-dbc482dc87b0: 2024-01-08T00:31:13.331Z [INFO] server-eabf3825-ed24-b923-ae5c-dbc482dc87b0: entering follower state: follower="Node at eabf3825-ed24-b923-ae5c-dbc482dc87b0 [Follower]" leader= +testLoggerAdapter verbose: server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: 2024-01-08T00:31:13.385Z [WARN] server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: 2024-01-08T00:31:13.385Z [WARN] server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: 2024-01-08T00:31:13.386Z [INFO] server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: entering candidate state: node="Node at 6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6 [Candidate]" term=2 + testing.go:149: server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: 2024-01-08T00:31:13.386Z [INFO] server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: entering candidate state: node="Node at 6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6 [Candidate]" term=2 +testLoggerAdapter verbose: server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: 2024-01-08T00:31:13.386Z [INFO] server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: election won: tally=2 + testing.go:149: server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: 2024-01-08T00:31:13.386Z [INFO] server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: election won: tally=2 +testLoggerAdapter verbose: server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: 2024-01-08T00:31:13.386Z [INFO] server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: entering leader state: leader="Node at 6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6 [Leader]" + testing.go:149: server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: 2024-01-08T00:31:13.386Z [INFO] server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: entering leader state: leader="Node at 6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6 [Leader]" +testLoggerAdapter verbose: server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: 2024-01-08T00:31:13.386Z [INFO] server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: added peer, starting replication: peer=server-9f315898-dfb9-f03d-4e0a-5ad8dc1a928e + testing.go:149: server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: 2024-01-08T00:31:13.386Z [INFO] server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: added peer, starting replication: peer=server-9f315898-dfb9-f03d-4e0a-5ad8dc1a928e +testLoggerAdapter verbose: server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: 2024-01-08T00:31:13.386Z [INFO] server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: added peer, starting replication: peer=server-eabf3825-ed24-b923-ae5c-dbc482dc87b0 + testing.go:149: server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: 2024-01-08T00:31:13.386Z [INFO] server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: added peer, starting replication: peer=server-eabf3825-ed24-b923-ae5c-dbc482dc87b0 +testLoggerAdapter verbose: server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: 2024-01-08T00:31:13.386Z [INFO] server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: pipelining replication: peer="{Voter server-eabf3825-ed24-b923-ae5c-dbc482dc87b0 eabf3825-ed24-b923-ae5c-dbc482dc87b0}" + testing.go:149: server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: 2024-01-08T00:31:13.386Z [INFO] server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: pipelining replication: peer="{Voter server-eabf3825-ed24-b923-ae5c-dbc482dc87b0 eabf3825-ed24-b923-ae5c-dbc482dc87b0}" +testLoggerAdapter verbose: server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: 2024-01-08T00:31:13.386Z [INFO] server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: pipelining replication: peer="{Voter server-9f315898-dfb9-f03d-4e0a-5ad8dc1a928e 9f315898-dfb9-f03d-4e0a-5ad8dc1a928e}" + testing.go:149: server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: 2024-01-08T00:31:13.386Z [INFO] server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: pipelining replication: peer="{Voter server-9f315898-dfb9-f03d-4e0a-5ad8dc1a928e 9f315898-dfb9-f03d-4e0a-5ad8dc1a928e}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:13.492Z [INFO] stable state for Leader reached at 2024-01-08 00:31:13.386677732 +0000 UTC m=+19.433566968 (1 nodes), 55.474344ms from start of poll, 55.98127ms from cluster start. Timeout at 2024-01-08 00:31:13.492774381 +0000 UTC m=+19.539663605, 106.096637ms after stability + testing.go:149: cluster: 2024-01-08T00:31:13.492Z [INFO] stable state for Leader reached at 2024-01-08 00:31:13.386677732 +0000 UTC m=+19.433566968 (1 nodes), 55.474344ms from start of poll, 55.98127ms from cluster start. Timeout at 2024-01-08 00:31:13.492774381 +0000 UTC m=+19.539663605, 106.096637ms after stability +testLoggerAdapter verbose: server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: 2024-01-08T00:31:13.492Z [INFO] server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: cannot transfer leadership to itself + testing.go:149: server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: 2024-01-08T00:31:13.492Z [INFO] server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: cannot transfer leadership to itself +testLoggerAdapter verbose: server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: 2024-01-08T00:31:13.492Z [INFO] server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: aborting pipeline replication: peer="{Voter server-eabf3825-ed24-b923-ae5c-dbc482dc87b0 eabf3825-ed24-b923-ae5c-dbc482dc87b0}" + testing.go:149: server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: 2024-01-08T00:31:13.492Z [INFO] server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: aborting pipeline replication: peer="{Voter server-eabf3825-ed24-b923-ae5c-dbc482dc87b0 eabf3825-ed24-b923-ae5c-dbc482dc87b0}" +testLoggerAdapter verbose: server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: 2024-01-08T00:31:13.492Z [INFO] server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: aborting pipeline replication: peer="{Voter server-9f315898-dfb9-f03d-4e0a-5ad8dc1a928e 9f315898-dfb9-f03d-4e0a-5ad8dc1a928e}" + testing.go:149: server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: 2024-01-08T00:31:13.492Z [INFO] server-6c84fc72-995f-88ff-4a04-4c8b5bb6bfa6: aborting pipeline replication: peer="{Voter server-9f315898-dfb9-f03d-4e0a-5ad8dc1a928e 9f315898-dfb9-f03d-4e0a-5ad8dc1a928e}" +--- PASS: TestRaft_LeadershipTransferToItself (0.16s) === RUN TestRaft_LeadershipTransferLeaderRejectsClientRequests raft_test.go:2064: Disabled unreliable test --- SKIP: TestRaft_LeadershipTransferLeaderRejectsClientRequests (0.00s) === RUN TestRaft_LeadershipTransferLeaderReplicationTimeout -testLoggerAdapter verbose: server-2942ab34-c24a-f504-08d5-837689be6a7d: 2025-02-09T06:52:38.012Z [INFO] server-2942ab34-c24a-f504-08d5-837689be6a7d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2942ab34-c24a-f504-08d5-837689be6a7d Address:2942ab34-c24a-f504-08d5-837689be6a7d} {Suffrage:Voter ID:server-2d521933-f9a6-0b6f-d886-3626d358a11c Address:2d521933-f9a6-0b6f-d886-3626d358a11c} {Suffrage:Voter ID:server-c1de82a1-a2ba-9cfc-47da-4efa687b0e8e Address:c1de82a1-a2ba-9cfc-47da-4efa687b0e8e}]" - testing.go:149: server-2942ab34-c24a-f504-08d5-837689be6a7d: 2025-02-09T06:52:38.012Z [INFO] server-2942ab34-c24a-f504-08d5-837689be6a7d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2942ab34-c24a-f504-08d5-837689be6a7d Address:2942ab34-c24a-f504-08d5-837689be6a7d} {Suffrage:Voter ID:server-2d521933-f9a6-0b6f-d886-3626d358a11c Address:2d521933-f9a6-0b6f-d886-3626d358a11c} {Suffrage:Voter ID:server-c1de82a1-a2ba-9cfc-47da-4efa687b0e8e Address:c1de82a1-a2ba-9cfc-47da-4efa687b0e8e}]" -testLoggerAdapter verbose: server-2942ab34-c24a-f504-08d5-837689be6a7d: 2025-02-09T06:52:38.012Z [INFO] server-2942ab34-c24a-f504-08d5-837689be6a7d: entering follower state: follower="Node at 2942ab34-c24a-f504-08d5-837689be6a7d [Follower]" leader= -testLoggerAdapter verbose: server-2d521933-f9a6-0b6f-d886-3626d358a11c: 2025-02-09T06:52:38.012Z [INFO] server-2d521933-f9a6-0b6f-d886-3626d358a11c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2942ab34-c24a-f504-08d5-837689be6a7d Address:2942ab34-c24a-f504-08d5-837689be6a7d} {Suffrage:Voter ID:server-2d521933-f9a6-0b6f-d886-3626d358a11c Address:2d521933-f9a6-0b6f-d886-3626d358a11c} {Suffrage:Voter ID:server-c1de82a1-a2ba-9cfc-47da-4efa687b0e8e Address:c1de82a1-a2ba-9cfc-47da-4efa687b0e8e}]" - testing.go:149: server-2942ab34-c24a-f504-08d5-837689be6a7d: 2025-02-09T06:52:38.012Z [INFO] server-2942ab34-c24a-f504-08d5-837689be6a7d: entering follower state: follower="Node at 2942ab34-c24a-f504-08d5-837689be6a7d [Follower]" leader= - testing.go:149: server-2d521933-f9a6-0b6f-d886-3626d358a11c: 2025-02-09T06:52:38.012Z [INFO] server-2d521933-f9a6-0b6f-d886-3626d358a11c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2942ab34-c24a-f504-08d5-837689be6a7d Address:2942ab34-c24a-f504-08d5-837689be6a7d} {Suffrage:Voter ID:server-2d521933-f9a6-0b6f-d886-3626d358a11c Address:2d521933-f9a6-0b6f-d886-3626d358a11c} {Suffrage:Voter ID:server-c1de82a1-a2ba-9cfc-47da-4efa687b0e8e Address:c1de82a1-a2ba-9cfc-47da-4efa687b0e8e}]" -testLoggerAdapter verbose: server-2d521933-f9a6-0b6f-d886-3626d358a11c: 2025-02-09T06:52:38.013Z [INFO] server-2d521933-f9a6-0b6f-d886-3626d358a11c: entering follower state: follower="Node at 2d521933-f9a6-0b6f-d886-3626d358a11c [Follower]" leader= - testing.go:149: server-2d521933-f9a6-0b6f-d886-3626d358a11c: 2025-02-09T06:52:38.013Z [INFO] server-2d521933-f9a6-0b6f-d886-3626d358a11c: entering follower state: follower="Node at 2d521933-f9a6-0b6f-d886-3626d358a11c [Follower]" leader= -testLoggerAdapter verbose: server-c1de82a1-a2ba-9cfc-47da-4efa687b0e8e: 2025-02-09T06:52:38.013Z [INFO] server-c1de82a1-a2ba-9cfc-47da-4efa687b0e8e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2942ab34-c24a-f504-08d5-837689be6a7d Address:2942ab34-c24a-f504-08d5-837689be6a7d} {Suffrage:Voter ID:server-2d521933-f9a6-0b6f-d886-3626d358a11c Address:2d521933-f9a6-0b6f-d886-3626d358a11c} {Suffrage:Voter ID:server-c1de82a1-a2ba-9cfc-47da-4efa687b0e8e Address:c1de82a1-a2ba-9cfc-47da-4efa687b0e8e}]" - testing.go:149: server-c1de82a1-a2ba-9cfc-47da-4efa687b0e8e: 2025-02-09T06:52:38.013Z [INFO] server-c1de82a1-a2ba-9cfc-47da-4efa687b0e8e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2942ab34-c24a-f504-08d5-837689be6a7d Address:2942ab34-c24a-f504-08d5-837689be6a7d} {Suffrage:Voter ID:server-2d521933-f9a6-0b6f-d886-3626d358a11c Address:2d521933-f9a6-0b6f-d886-3626d358a11c} {Suffrage:Voter ID:server-c1de82a1-a2ba-9cfc-47da-4efa687b0e8e Address:c1de82a1-a2ba-9cfc-47da-4efa687b0e8e}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:38.013Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:38.013Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-c1de82a1-a2ba-9cfc-47da-4efa687b0e8e: 2025-02-09T06:52:38.013Z [INFO] server-c1de82a1-a2ba-9cfc-47da-4efa687b0e8e: entering follower state: follower="Node at c1de82a1-a2ba-9cfc-47da-4efa687b0e8e [Follower]" leader= - testing.go:149: server-c1de82a1-a2ba-9cfc-47da-4efa687b0e8e: 2025-02-09T06:52:38.013Z [INFO] server-c1de82a1-a2ba-9cfc-47da-4efa687b0e8e: entering follower state: follower="Node at c1de82a1-a2ba-9cfc-47da-4efa687b0e8e [Follower]" leader= -testLoggerAdapter verbose: server-2d521933-f9a6-0b6f-d886-3626d358a11c: 2025-02-09T06:52:38.085Z [WARN] server-2d521933-f9a6-0b6f-d886-3626d358a11c: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-2d521933-f9a6-0b6f-d886-3626d358a11c: 2025-02-09T06:52:38.085Z [WARN] server-2d521933-f9a6-0b6f-d886-3626d358a11c: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-2d521933-f9a6-0b6f-d886-3626d358a11c: 2025-02-09T06:52:38.085Z [INFO] server-2d521933-f9a6-0b6f-d886-3626d358a11c: entering candidate state: node="Node at 2d521933-f9a6-0b6f-d886-3626d358a11c [Candidate]" term=2 - testing.go:149: server-2d521933-f9a6-0b6f-d886-3626d358a11c: 2025-02-09T06:52:38.085Z [INFO] server-2d521933-f9a6-0b6f-d886-3626d358a11c: entering candidate state: node="Node at 2d521933-f9a6-0b6f-d886-3626d358a11c [Candidate]" term=2 -testLoggerAdapter verbose: server-2d521933-f9a6-0b6f-d886-3626d358a11c: 2025-02-09T06:52:38.085Z [INFO] server-2d521933-f9a6-0b6f-d886-3626d358a11c: election won: tally=2 - testing.go:149: server-2d521933-f9a6-0b6f-d886-3626d358a11c: 2025-02-09T06:52:38.085Z [INFO] server-2d521933-f9a6-0b6f-d886-3626d358a11c: election won: tally=2 -testLoggerAdapter verbose: server-2d521933-f9a6-0b6f-d886-3626d358a11c: 2025-02-09T06:52:38.085Z [INFO] server-2d521933-f9a6-0b6f-d886-3626d358a11c: entering leader state: leader="Node at 2d521933-f9a6-0b6f-d886-3626d358a11c [Leader]" - testing.go:149: server-2d521933-f9a6-0b6f-d886-3626d358a11c: 2025-02-09T06:52:38.085Z [INFO] server-2d521933-f9a6-0b6f-d886-3626d358a11c: entering leader state: leader="Node at 2d521933-f9a6-0b6f-d886-3626d358a11c [Leader]" -testLoggerAdapter verbose: server-2d521933-f9a6-0b6f-d886-3626d358a11c: 2025-02-09T06:52:38.085Z [INFO] server-2d521933-f9a6-0b6f-d886-3626d358a11c: added peer, starting replication: peer=server-2942ab34-c24a-f504-08d5-837689be6a7d - testing.go:149: server-2d521933-f9a6-0b6f-d886-3626d358a11c: 2025-02-09T06:52:38.085Z [INFO] server-2d521933-f9a6-0b6f-d886-3626d358a11c: added peer, starting replication: peer=server-2942ab34-c24a-f504-08d5-837689be6a7d -testLoggerAdapter verbose: server-2d521933-f9a6-0b6f-d886-3626d358a11c: 2025-02-09T06:52:38.085Z [INFO] server-2d521933-f9a6-0b6f-d886-3626d358a11c: added peer, starting replication: peer=server-c1de82a1-a2ba-9cfc-47da-4efa687b0e8e - testing.go:149: server-2d521933-f9a6-0b6f-d886-3626d358a11c: 2025-02-09T06:52:38.085Z [INFO] server-2d521933-f9a6-0b6f-d886-3626d358a11c: added peer, starting replication: peer=server-c1de82a1-a2ba-9cfc-47da-4efa687b0e8e -testLoggerAdapter verbose: server-2d521933-f9a6-0b6f-d886-3626d358a11c: 2025-02-09T06:52:38.085Z [INFO] server-2d521933-f9a6-0b6f-d886-3626d358a11c: pipelining replication: peer="{Voter server-2942ab34-c24a-f504-08d5-837689be6a7d 2942ab34-c24a-f504-08d5-837689be6a7d}" - testing.go:149: server-2d521933-f9a6-0b6f-d886-3626d358a11c: 2025-02-09T06:52:38.085Z [INFO] server-2d521933-f9a6-0b6f-d886-3626d358a11c: pipelining replication: peer="{Voter server-2942ab34-c24a-f504-08d5-837689be6a7d 2942ab34-c24a-f504-08d5-837689be6a7d}" -testLoggerAdapter verbose: server-2d521933-f9a6-0b6f-d886-3626d358a11c: 2025-02-09T06:52:38.085Z [INFO] server-2d521933-f9a6-0b6f-d886-3626d358a11c: pipelining replication: peer="{Voter server-c1de82a1-a2ba-9cfc-47da-4efa687b0e8e c1de82a1-a2ba-9cfc-47da-4efa687b0e8e}" - testing.go:149: server-2d521933-f9a6-0b6f-d886-3626d358a11c: 2025-02-09T06:52:38.085Z [INFO] server-2d521933-f9a6-0b6f-d886-3626d358a11c: pipelining replication: peer="{Voter server-c1de82a1-a2ba-9cfc-47da-4efa687b0e8e c1de82a1-a2ba-9cfc-47da-4efa687b0e8e}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:38.191Z [INFO] stable state for Leader reached at 2025-02-09 06:52:38.085504524 +0000 UTC m=+19.064313302 (1 nodes), 72.267395ms from start of poll, 72.783123ms from cluster start. Timeout at 2025-02-09 06:52:38.191080707 +0000 UTC m=+19.169889467, 105.576165ms after stability - testing.go:149: cluster: 2025-02-09T06:52:38.191Z [INFO] stable state for Leader reached at 2025-02-09 06:52:38.085504524 +0000 UTC m=+19.064313302 (1 nodes), 72.267395ms from start of poll, 72.783123ms from cluster start. Timeout at 2025-02-09 06:52:38.191080707 +0000 UTC m=+19.169889467, 105.576165ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:38.191Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T06:52:38.191Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:38.296Z [INFO] stable state for Follower reached at 2025-02-09 06:52:38.19117293 +0000 UTC m=+19.169981710 (2 nodes), 878ns from start of poll, 178.451531ms from cluster start. Timeout at 2025-02-09 06:52:38.296342086 +0000 UTC m=+19.275150869, 105.169159ms after stability - testing.go:149: cluster: 2025-02-09T06:52:38.296Z [INFO] stable state for Follower reached at 2025-02-09 06:52:38.19117293 +0000 UTC m=+19.169981710 (2 nodes), 878ns from start of poll, 178.451531ms from cluster start. Timeout at 2025-02-09 06:52:38.296342086 +0000 UTC m=+19.275150869, 105.169159ms after stability -testLoggerAdapter verbose: server-2d521933-f9a6-0b6f-d886-3626d358a11c: 2025-02-09T06:52:38.439Z [INFO] server-2d521933-f9a6-0b6f-d886-3626d358a11c: aborting pipeline replication: peer="{Voter server-c1de82a1-a2ba-9cfc-47da-4efa687b0e8e c1de82a1-a2ba-9cfc-47da-4efa687b0e8e}" - testing.go:149: server-2d521933-f9a6-0b6f-d886-3626d358a11c: 2025-02-09T06:52:38.439Z [INFO] server-2d521933-f9a6-0b6f-d886-3626d358a11c: aborting pipeline replication: peer="{Voter server-c1de82a1-a2ba-9cfc-47da-4efa687b0e8e c1de82a1-a2ba-9cfc-47da-4efa687b0e8e}" -testLoggerAdapter verbose: server-2d521933-f9a6-0b6f-d886-3626d358a11c: 2025-02-09T06:52:38.439Z [INFO] server-2d521933-f9a6-0b6f-d886-3626d358a11c: aborting pipeline replication: peer="{Voter server-2942ab34-c24a-f504-08d5-837689be6a7d 2942ab34-c24a-f504-08d5-837689be6a7d}" - testing.go:149: server-2d521933-f9a6-0b6f-d886-3626d358a11c: 2025-02-09T06:52:38.439Z [INFO] server-2d521933-f9a6-0b6f-d886-3626d358a11c: aborting pipeline replication: peer="{Voter server-2942ab34-c24a-f504-08d5-837689be6a7d 2942ab34-c24a-f504-08d5-837689be6a7d}" ---- PASS: TestRaft_LeadershipTransferLeaderReplicationTimeout (0.43s) +testLoggerAdapter verbose: server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: 2024-01-08T00:31:13.494Z [INFO] server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1 Address:0a0f1ed7-2d77-7717-673e-e0ecee9910a1} {Suffrage:Voter ID:server-cf8aab85-5f7f-3026-60cc-5df9dc5aa44e Address:cf8aab85-5f7f-3026-60cc-5df9dc5aa44e} {Suffrage:Voter ID:server-259137cf-5d3d-38bf-3995-e1d74d8c4293 Address:259137cf-5d3d-38bf-3995-e1d74d8c4293}]" + testing.go:149: server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: 2024-01-08T00:31:13.494Z [INFO] server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1 Address:0a0f1ed7-2d77-7717-673e-e0ecee9910a1} {Suffrage:Voter ID:server-cf8aab85-5f7f-3026-60cc-5df9dc5aa44e Address:cf8aab85-5f7f-3026-60cc-5df9dc5aa44e} {Suffrage:Voter ID:server-259137cf-5d3d-38bf-3995-e1d74d8c4293 Address:259137cf-5d3d-38bf-3995-e1d74d8c4293}]" +testLoggerAdapter verbose: server-cf8aab85-5f7f-3026-60cc-5df9dc5aa44e: 2024-01-08T00:31:13.494Z [INFO] server-cf8aab85-5f7f-3026-60cc-5df9dc5aa44e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1 Address:0a0f1ed7-2d77-7717-673e-e0ecee9910a1} {Suffrage:Voter ID:server-cf8aab85-5f7f-3026-60cc-5df9dc5aa44e Address:cf8aab85-5f7f-3026-60cc-5df9dc5aa44e} {Suffrage:Voter ID:server-259137cf-5d3d-38bf-3995-e1d74d8c4293 Address:259137cf-5d3d-38bf-3995-e1d74d8c4293}]" + testing.go:149: server-cf8aab85-5f7f-3026-60cc-5df9dc5aa44e: 2024-01-08T00:31:13.494Z [INFO] server-cf8aab85-5f7f-3026-60cc-5df9dc5aa44e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1 Address:0a0f1ed7-2d77-7717-673e-e0ecee9910a1} {Suffrage:Voter ID:server-cf8aab85-5f7f-3026-60cc-5df9dc5aa44e Address:cf8aab85-5f7f-3026-60cc-5df9dc5aa44e} {Suffrage:Voter ID:server-259137cf-5d3d-38bf-3995-e1d74d8c4293 Address:259137cf-5d3d-38bf-3995-e1d74d8c4293}]" +testLoggerAdapter verbose: server-259137cf-5d3d-38bf-3995-e1d74d8c4293: 2024-01-08T00:31:13.494Z [INFO] server-259137cf-5d3d-38bf-3995-e1d74d8c4293: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1 Address:0a0f1ed7-2d77-7717-673e-e0ecee9910a1} {Suffrage:Voter ID:server-cf8aab85-5f7f-3026-60cc-5df9dc5aa44e Address:cf8aab85-5f7f-3026-60cc-5df9dc5aa44e} {Suffrage:Voter ID:server-259137cf-5d3d-38bf-3995-e1d74d8c4293 Address:259137cf-5d3d-38bf-3995-e1d74d8c4293}]" + testing.go:149: server-259137cf-5d3d-38bf-3995-e1d74d8c4293: 2024-01-08T00:31:13.494Z [INFO] server-259137cf-5d3d-38bf-3995-e1d74d8c4293: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1 Address:0a0f1ed7-2d77-7717-673e-e0ecee9910a1} {Suffrage:Voter ID:server-cf8aab85-5f7f-3026-60cc-5df9dc5aa44e Address:cf8aab85-5f7f-3026-60cc-5df9dc5aa44e} {Suffrage:Voter ID:server-259137cf-5d3d-38bf-3995-e1d74d8c4293 Address:259137cf-5d3d-38bf-3995-e1d74d8c4293}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:13.494Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:13.494Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: 2024-01-08T00:31:13.494Z [INFO] server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: entering follower state: follower="Node at 0a0f1ed7-2d77-7717-673e-e0ecee9910a1 [Follower]" leader= + testing.go:149: server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: 2024-01-08T00:31:13.494Z [INFO] server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: entering follower state: follower="Node at 0a0f1ed7-2d77-7717-673e-e0ecee9910a1 [Follower]" leader= +testLoggerAdapter verbose: server-cf8aab85-5f7f-3026-60cc-5df9dc5aa44e: 2024-01-08T00:31:13.494Z [INFO] server-cf8aab85-5f7f-3026-60cc-5df9dc5aa44e: entering follower state: follower="Node at cf8aab85-5f7f-3026-60cc-5df9dc5aa44e [Follower]" leader= + testing.go:149: server-cf8aab85-5f7f-3026-60cc-5df9dc5aa44e: 2024-01-08T00:31:13.494Z [INFO] server-cf8aab85-5f7f-3026-60cc-5df9dc5aa44e: entering follower state: follower="Node at cf8aab85-5f7f-3026-60cc-5df9dc5aa44e [Follower]" leader= +testLoggerAdapter verbose: server-259137cf-5d3d-38bf-3995-e1d74d8c4293: 2024-01-08T00:31:13.494Z [INFO] server-259137cf-5d3d-38bf-3995-e1d74d8c4293: entering follower state: follower="Node at 259137cf-5d3d-38bf-3995-e1d74d8c4293 [Follower]" leader= + testing.go:149: server-259137cf-5d3d-38bf-3995-e1d74d8c4293: 2024-01-08T00:31:13.494Z [INFO] server-259137cf-5d3d-38bf-3995-e1d74d8c4293: entering follower state: follower="Node at 259137cf-5d3d-38bf-3995-e1d74d8c4293 [Follower]" leader= +testLoggerAdapter verbose: server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: 2024-01-08T00:31:13.550Z [WARN] server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: 2024-01-08T00:31:13.550Z [WARN] server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: 2024-01-08T00:31:13.550Z [INFO] server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: entering candidate state: node="Node at 0a0f1ed7-2d77-7717-673e-e0ecee9910a1 [Candidate]" term=2 + testing.go:149: server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: 2024-01-08T00:31:13.550Z [INFO] server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: entering candidate state: node="Node at 0a0f1ed7-2d77-7717-673e-e0ecee9910a1 [Candidate]" term=2 +testLoggerAdapter verbose: server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: 2024-01-08T00:31:13.550Z [INFO] server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: election won: tally=2 + testing.go:149: server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: 2024-01-08T00:31:13.550Z [INFO] server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: election won: tally=2 +testLoggerAdapter verbose: server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: 2024-01-08T00:31:13.550Z [INFO] server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: entering leader state: leader="Node at 0a0f1ed7-2d77-7717-673e-e0ecee9910a1 [Leader]" + testing.go:149: server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: 2024-01-08T00:31:13.550Z [INFO] server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: entering leader state: leader="Node at 0a0f1ed7-2d77-7717-673e-e0ecee9910a1 [Leader]" +testLoggerAdapter verbose: server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: 2024-01-08T00:31:13.550Z [INFO] server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: added peer, starting replication: peer=server-cf8aab85-5f7f-3026-60cc-5df9dc5aa44e + testing.go:149: server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: 2024-01-08T00:31:13.550Z [INFO] server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: added peer, starting replication: peer=server-cf8aab85-5f7f-3026-60cc-5df9dc5aa44e +testLoggerAdapter verbose: server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: 2024-01-08T00:31:13.550Z [INFO] server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: added peer, starting replication: peer=server-259137cf-5d3d-38bf-3995-e1d74d8c4293 + testing.go:149: server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: 2024-01-08T00:31:13.550Z [INFO] server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: added peer, starting replication: peer=server-259137cf-5d3d-38bf-3995-e1d74d8c4293 +testLoggerAdapter verbose: server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: 2024-01-08T00:31:13.551Z [INFO] server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: pipelining replication: peer="{Voter server-259137cf-5d3d-38bf-3995-e1d74d8c4293 259137cf-5d3d-38bf-3995-e1d74d8c4293}" + testing.go:149: server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: 2024-01-08T00:31:13.551Z [INFO] server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: pipelining replication: peer="{Voter server-259137cf-5d3d-38bf-3995-e1d74d8c4293 259137cf-5d3d-38bf-3995-e1d74d8c4293}" +testLoggerAdapter verbose: server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: 2024-01-08T00:31:13.551Z [INFO] server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: pipelining replication: peer="{Voter server-cf8aab85-5f7f-3026-60cc-5df9dc5aa44e cf8aab85-5f7f-3026-60cc-5df9dc5aa44e}" + testing.go:149: server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: 2024-01-08T00:31:13.551Z [INFO] server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: pipelining replication: peer="{Voter server-cf8aab85-5f7f-3026-60cc-5df9dc5aa44e cf8aab85-5f7f-3026-60cc-5df9dc5aa44e}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:13.657Z [INFO] stable state for Leader reached at 2024-01-08 00:31:13.551315809 +0000 UTC m=+19.598204972 (1 nodes), 56.943839ms from start of poll, 57.312961ms from cluster start. Timeout at 2024-01-08 00:31:13.657153156 +0000 UTC m=+19.704042244, 105.837272ms after stability + testing.go:149: cluster: 2024-01-08T00:31:13.657Z [INFO] stable state for Leader reached at 2024-01-08 00:31:13.551315809 +0000 UTC m=+19.598204972 (1 nodes), 56.943839ms from start of poll, 57.312961ms from cluster start. Timeout at 2024-01-08 00:31:13.657153156 +0000 UTC m=+19.704042244, 105.837272ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:13.657Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T00:31:13.657Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:13.762Z [INFO] stable state for Follower reached at 2024-01-08 00:31:13.657314862 +0000 UTC m=+19.704203974 (2 nodes), 2.551µs from start of poll, 163.311963ms from cluster start. Timeout at 2024-01-08 00:31:13.76292298 +0000 UTC m=+19.809812855, 105.608881ms after stability + testing.go:149: cluster: 2024-01-08T00:31:13.762Z [INFO] stable state for Follower reached at 2024-01-08 00:31:13.657314862 +0000 UTC m=+19.704203974 (2 nodes), 2.551µs from start of poll, 163.311963ms from cluster start. Timeout at 2024-01-08 00:31:13.76292298 +0000 UTC m=+19.809812855, 105.608881ms after stability +testLoggerAdapter verbose: server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: 2024-01-08T00:31:14.144Z [INFO] server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: aborting pipeline replication: peer="{Voter server-259137cf-5d3d-38bf-3995-e1d74d8c4293 259137cf-5d3d-38bf-3995-e1d74d8c4293}" + testing.go:149: server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: 2024-01-08T00:31:14.144Z [INFO] server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: aborting pipeline replication: peer="{Voter server-259137cf-5d3d-38bf-3995-e1d74d8c4293 259137cf-5d3d-38bf-3995-e1d74d8c4293}" +testLoggerAdapter verbose: server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: 2024-01-08T00:31:14.144Z [INFO] server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: aborting pipeline replication: peer="{Voter server-cf8aab85-5f7f-3026-60cc-5df9dc5aa44e cf8aab85-5f7f-3026-60cc-5df9dc5aa44e}" + testing.go:149: server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: 2024-01-08T00:31:14.144Z [INFO] server-0a0f1ed7-2d77-7717-673e-e0ecee9910a1: aborting pipeline replication: peer="{Voter server-cf8aab85-5f7f-3026-60cc-5df9dc5aa44e cf8aab85-5f7f-3026-60cc-5df9dc5aa44e}" +--- PASS: TestRaft_LeadershipTransferLeaderReplicationTimeout (0.65s) === RUN TestRaft_LeadershipTransferStopRightAway --- PASS: TestRaft_LeadershipTransferStopRightAway (0.00s) === RUN TestRaft_GetConfigurationNoBootstrap -testLoggerAdapter verbose: server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: 2025-02-09T06:52:38.440Z [INFO] server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c97ed00-795e-25e0-a0d3-6328b9991ac6 Address:0c97ed00-795e-25e0-a0d3-6328b9991ac6} {Suffrage:Voter ID:server-e64640f2-9154-c4ed-a882-cc366d5dd235 Address:e64640f2-9154-c4ed-a882-cc366d5dd235}]" - testing.go:149: server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: 2025-02-09T06:52:38.440Z [INFO] server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c97ed00-795e-25e0-a0d3-6328b9991ac6 Address:0c97ed00-795e-25e0-a0d3-6328b9991ac6} {Suffrage:Voter ID:server-e64640f2-9154-c4ed-a882-cc366d5dd235 Address:e64640f2-9154-c4ed-a882-cc366d5dd235}]" -testLoggerAdapter verbose: server-e64640f2-9154-c4ed-a882-cc366d5dd235: 2025-02-09T06:52:38.440Z [INFO] server-e64640f2-9154-c4ed-a882-cc366d5dd235: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c97ed00-795e-25e0-a0d3-6328b9991ac6 Address:0c97ed00-795e-25e0-a0d3-6328b9991ac6} {Suffrage:Voter ID:server-e64640f2-9154-c4ed-a882-cc366d5dd235 Address:e64640f2-9154-c4ed-a882-cc366d5dd235}]" - testing.go:149: server-e64640f2-9154-c4ed-a882-cc366d5dd235: 2025-02-09T06:52:38.440Z [INFO] server-e64640f2-9154-c4ed-a882-cc366d5dd235: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c97ed00-795e-25e0-a0d3-6328b9991ac6 Address:0c97ed00-795e-25e0-a0d3-6328b9991ac6} {Suffrage:Voter ID:server-e64640f2-9154-c4ed-a882-cc366d5dd235 Address:e64640f2-9154-c4ed-a882-cc366d5dd235}]" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:38.440Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: 2025-02-09T06:52:38.440Z [INFO] server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: entering follower state: follower="Node at 0c97ed00-795e-25e0-a0d3-6328b9991ac6 [Follower]" leader= - testing.go:149: cluster: 2025-02-09T06:52:38.440Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-e64640f2-9154-c4ed-a882-cc366d5dd235: 2025-02-09T06:52:38.440Z [INFO] server-e64640f2-9154-c4ed-a882-cc366d5dd235: entering follower state: follower="Node at e64640f2-9154-c4ed-a882-cc366d5dd235 [Follower]" leader= - testing.go:149: server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: 2025-02-09T06:52:38.440Z [INFO] server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: entering follower state: follower="Node at 0c97ed00-795e-25e0-a0d3-6328b9991ac6 [Follower]" leader= - testing.go:149: server-e64640f2-9154-c4ed-a882-cc366d5dd235: 2025-02-09T06:52:38.440Z [INFO] server-e64640f2-9154-c4ed-a882-cc366d5dd235: entering follower state: follower="Node at e64640f2-9154-c4ed-a882-cc366d5dd235 [Follower]" leader= -testLoggerAdapter verbose: server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: 2025-02-09T06:52:38.497Z [WARN] server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: 2025-02-09T06:52:38.497Z [WARN] server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: 2025-02-09T06:52:38.497Z [INFO] server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: entering candidate state: node="Node at 0c97ed00-795e-25e0-a0d3-6328b9991ac6 [Candidate]" term=2 - testing.go:149: server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: 2025-02-09T06:52:38.497Z [INFO] server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: entering candidate state: node="Node at 0c97ed00-795e-25e0-a0d3-6328b9991ac6 [Candidate]" term=2 -testLoggerAdapter verbose: server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: 2025-02-09T06:52:38.497Z [INFO] server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: election won: tally=2 - testing.go:149: server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: 2025-02-09T06:52:38.497Z [INFO] server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: election won: tally=2 -testLoggerAdapter verbose: server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: 2025-02-09T06:52:38.497Z [INFO] server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: entering leader state: leader="Node at 0c97ed00-795e-25e0-a0d3-6328b9991ac6 [Leader]" - testing.go:149: server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: 2025-02-09T06:52:38.497Z [INFO] server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: entering leader state: leader="Node at 0c97ed00-795e-25e0-a0d3-6328b9991ac6 [Leader]" -testLoggerAdapter verbose: server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: 2025-02-09T06:52:38.497Z [INFO] server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: added peer, starting replication: peer=server-e64640f2-9154-c4ed-a882-cc366d5dd235 - testing.go:149: server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: 2025-02-09T06:52:38.497Z [INFO] server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: added peer, starting replication: peer=server-e64640f2-9154-c4ed-a882-cc366d5dd235 -testLoggerAdapter verbose: server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: 2025-02-09T06:52:38.498Z [INFO] server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: pipelining replication: peer="{Voter server-e64640f2-9154-c4ed-a882-cc366d5dd235 e64640f2-9154-c4ed-a882-cc366d5dd235}" - testing.go:149: server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: 2025-02-09T06:52:38.498Z [INFO] server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: pipelining replication: peer="{Voter server-e64640f2-9154-c4ed-a882-cc366d5dd235 e64640f2-9154-c4ed-a882-cc366d5dd235}" -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:38.603Z [INFO] stable state for Follower reached at 2025-02-09 06:52:38.497977027 +0000 UTC m=+19.476785805 (1 nodes), 57.66602ms from start of poll, 57.923086ms from cluster start. Timeout at 2025-02-09 06:52:38.603748504 +0000 UTC m=+19.582557261, 105.771456ms after stability - testing.go:149: cluster: 2025-02-09T06:52:38.603Z [INFO] stable state for Follower reached at 2025-02-09 06:52:38.497977027 +0000 UTC m=+19.476785805 (1 nodes), 57.66602ms from start of poll, 57.923086ms from cluster start. Timeout at 2025-02-09 06:52:38.603748504 +0000 UTC m=+19.582557261, 105.771456ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:38.603Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T06:52:38.603Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T06:52:38.709Z [INFO] stable state for Leader reached at 2025-02-09 06:52:38.603907039 +0000 UTC m=+19.582715815 (1 nodes), 474ns from start of poll, 163.853096ms from cluster start. Timeout at 2025-02-09 06:52:38.709503554 +0000 UTC m=+19.688312324, 105.596509ms after stability - testing.go:149: cluster: 2025-02-09T06:52:38.709Z [INFO] stable state for Leader reached at 2025-02-09 06:52:38.603907039 +0000 UTC m=+19.582715815 (1 nodes), 474ns from start of poll, 163.853096ms from cluster start. Timeout at 2025-02-09 06:52:38.709503554 +0000 UTC m=+19.688312324, 105.596509ms after stability -testLoggerAdapter verbose: server-e64640f2-9154-c4ed-a882-cc366d5dd235: 2025-02-09T06:52:38.720Z [INFO] server-e64640f2-9154-c4ed-a882-cc366d5dd235: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c97ed00-795e-25e0-a0d3-6328b9991ac6 Address:0c97ed00-795e-25e0-a0d3-6328b9991ac6} {Suffrage:Voter ID:server-e64640f2-9154-c4ed-a882-cc366d5dd235 Address:e64640f2-9154-c4ed-a882-cc366d5dd235}]" - testing.go:149: server-e64640f2-9154-c4ed-a882-cc366d5dd235: 2025-02-09T06:52:38.720Z [INFO] server-e64640f2-9154-c4ed-a882-cc366d5dd235: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0c97ed00-795e-25e0-a0d3-6328b9991ac6 Address:0c97ed00-795e-25e0-a0d3-6328b9991ac6} {Suffrage:Voter ID:server-e64640f2-9154-c4ed-a882-cc366d5dd235 Address:e64640f2-9154-c4ed-a882-cc366d5dd235}]" -testLoggerAdapter verbose: server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: 2025-02-09T06:52:38.720Z [INFO] server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: aborting pipeline replication: peer="{Voter server-e64640f2-9154-c4ed-a882-cc366d5dd235 e64640f2-9154-c4ed-a882-cc366d5dd235}" - testing.go:149: server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: 2025-02-09T06:52:38.720Z [INFO] server-0c97ed00-795e-25e0-a0d3-6328b9991ac6: aborting pipeline replication: peer="{Voter server-e64640f2-9154-c4ed-a882-cc366d5dd235 e64640f2-9154-c4ed-a882-cc366d5dd235}" ---- PASS: TestRaft_GetConfigurationNoBootstrap (0.28s) +testLoggerAdapter verbose: server-583c32b5-8e2e-809d-cc03-feff3264cb4d: 2024-01-08T00:31:14.146Z [INFO] server-583c32b5-8e2e-809d-cc03-feff3264cb4d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-583c32b5-8e2e-809d-cc03-feff3264cb4d Address:583c32b5-8e2e-809d-cc03-feff3264cb4d} {Suffrage:Voter ID:server-8942f1da-0549-78a5-321a-3384b25815a3 Address:8942f1da-0549-78a5-321a-3384b25815a3}]" + testing.go:149: server-583c32b5-8e2e-809d-cc03-feff3264cb4d: 2024-01-08T00:31:14.146Z [INFO] server-583c32b5-8e2e-809d-cc03-feff3264cb4d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-583c32b5-8e2e-809d-cc03-feff3264cb4d Address:583c32b5-8e2e-809d-cc03-feff3264cb4d} {Suffrage:Voter ID:server-8942f1da-0549-78a5-321a-3384b25815a3 Address:8942f1da-0549-78a5-321a-3384b25815a3}]" +testLoggerAdapter verbose: server-583c32b5-8e2e-809d-cc03-feff3264cb4d: 2024-01-08T00:31:14.146Z [INFO] server-583c32b5-8e2e-809d-cc03-feff3264cb4d: entering follower state: follower="Node at 583c32b5-8e2e-809d-cc03-feff3264cb4d [Follower]" leader= + testing.go:149: server-583c32b5-8e2e-809d-cc03-feff3264cb4d: 2024-01-08T00:31:14.146Z [INFO] server-583c32b5-8e2e-809d-cc03-feff3264cb4d: entering follower state: follower="Node at 583c32b5-8e2e-809d-cc03-feff3264cb4d [Follower]" leader= +testLoggerAdapter verbose: server-8942f1da-0549-78a5-321a-3384b25815a3: 2024-01-08T00:31:14.146Z [INFO] server-8942f1da-0549-78a5-321a-3384b25815a3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-583c32b5-8e2e-809d-cc03-feff3264cb4d Address:583c32b5-8e2e-809d-cc03-feff3264cb4d} {Suffrage:Voter ID:server-8942f1da-0549-78a5-321a-3384b25815a3 Address:8942f1da-0549-78a5-321a-3384b25815a3}]" + testing.go:149: server-8942f1da-0549-78a5-321a-3384b25815a3: 2024-01-08T00:31:14.146Z [INFO] server-8942f1da-0549-78a5-321a-3384b25815a3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-583c32b5-8e2e-809d-cc03-feff3264cb4d Address:583c32b5-8e2e-809d-cc03-feff3264cb4d} {Suffrage:Voter ID:server-8942f1da-0549-78a5-321a-3384b25815a3 Address:8942f1da-0549-78a5-321a-3384b25815a3}]" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:14.146Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T00:31:14.146Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: server-8942f1da-0549-78a5-321a-3384b25815a3: 2024-01-08T00:31:14.146Z [INFO] server-8942f1da-0549-78a5-321a-3384b25815a3: entering follower state: follower="Node at 8942f1da-0549-78a5-321a-3384b25815a3 [Follower]" leader= + testing.go:149: server-8942f1da-0549-78a5-321a-3384b25815a3: 2024-01-08T00:31:14.146Z [INFO] server-8942f1da-0549-78a5-321a-3384b25815a3: entering follower state: follower="Node at 8942f1da-0549-78a5-321a-3384b25815a3 [Follower]" leader= +testLoggerAdapter verbose: server-8942f1da-0549-78a5-321a-3384b25815a3: 2024-01-08T00:31:14.224Z [WARN] server-8942f1da-0549-78a5-321a-3384b25815a3: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-8942f1da-0549-78a5-321a-3384b25815a3: 2024-01-08T00:31:14.224Z [WARN] server-8942f1da-0549-78a5-321a-3384b25815a3: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-8942f1da-0549-78a5-321a-3384b25815a3: 2024-01-08T00:31:14.224Z [INFO] server-8942f1da-0549-78a5-321a-3384b25815a3: entering candidate state: node="Node at 8942f1da-0549-78a5-321a-3384b25815a3 [Candidate]" term=2 + testing.go:149: server-8942f1da-0549-78a5-321a-3384b25815a3: 2024-01-08T00:31:14.224Z [INFO] server-8942f1da-0549-78a5-321a-3384b25815a3: entering candidate state: node="Node at 8942f1da-0549-78a5-321a-3384b25815a3 [Candidate]" term=2 +testLoggerAdapter verbose: server-8942f1da-0549-78a5-321a-3384b25815a3: 2024-01-08T00:31:14.224Z [INFO] server-8942f1da-0549-78a5-321a-3384b25815a3: election won: tally=2 + testing.go:149: server-8942f1da-0549-78a5-321a-3384b25815a3: 2024-01-08T00:31:14.224Z [INFO] server-8942f1da-0549-78a5-321a-3384b25815a3: election won: tally=2 +testLoggerAdapter verbose: server-8942f1da-0549-78a5-321a-3384b25815a3: 2024-01-08T00:31:14.224Z [INFO] server-8942f1da-0549-78a5-321a-3384b25815a3: entering leader state: leader="Node at 8942f1da-0549-78a5-321a-3384b25815a3 [Leader]" + testing.go:149: server-8942f1da-0549-78a5-321a-3384b25815a3: 2024-01-08T00:31:14.224Z [INFO] server-8942f1da-0549-78a5-321a-3384b25815a3: entering leader state: leader="Node at 8942f1da-0549-78a5-321a-3384b25815a3 [Leader]" +testLoggerAdapter verbose: server-8942f1da-0549-78a5-321a-3384b25815a3: 2024-01-08T00:31:14.224Z [INFO] server-8942f1da-0549-78a5-321a-3384b25815a3: added peer, starting replication: peer=server-583c32b5-8e2e-809d-cc03-feff3264cb4d + testing.go:149: server-8942f1da-0549-78a5-321a-3384b25815a3: 2024-01-08T00:31:14.224Z [INFO] server-8942f1da-0549-78a5-321a-3384b25815a3: added peer, starting replication: peer=server-583c32b5-8e2e-809d-cc03-feff3264cb4d +testLoggerAdapter verbose: server-8942f1da-0549-78a5-321a-3384b25815a3: 2024-01-08T00:31:14.224Z [INFO] server-8942f1da-0549-78a5-321a-3384b25815a3: pipelining replication: peer="{Voter server-583c32b5-8e2e-809d-cc03-feff3264cb4d 583c32b5-8e2e-809d-cc03-feff3264cb4d}" + testing.go:149: server-8942f1da-0549-78a5-321a-3384b25815a3: 2024-01-08T00:31:14.224Z [INFO] server-8942f1da-0549-78a5-321a-3384b25815a3: pipelining replication: peer="{Voter server-583c32b5-8e2e-809d-cc03-feff3264cb4d 583c32b5-8e2e-809d-cc03-feff3264cb4d}" +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:14.330Z [INFO] stable state for Follower reached at 2024-01-08 00:31:14.22477102 +0000 UTC m=+20.271660107 (1 nodes), 78.189388ms from start of poll, 78.662146ms from cluster start. Timeout at 2024-01-08 00:31:14.330707984 +0000 UTC m=+20.377597221, 105.937114ms after stability + testing.go:149: cluster: 2024-01-08T00:31:14.330Z [INFO] stable state for Follower reached at 2024-01-08 00:31:14.22477102 +0000 UTC m=+20.271660107 (1 nodes), 78.189388ms from start of poll, 78.662146ms from cluster start. Timeout at 2024-01-08 00:31:14.330707984 +0000 UTC m=+20.377597221, 105.937114ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:14.330Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T00:31:14.330Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T00:31:14.436Z [INFO] stable state for Leader reached at 2024-01-08 00:31:14.330922676 +0000 UTC m=+20.377811867 (1 nodes), 2.64µs from start of poll, 184.813906ms from cluster start. Timeout at 2024-01-08 00:31:14.436783734 +0000 UTC m=+20.483672847, 105.86098ms after stability + testing.go:149: cluster: 2024-01-08T00:31:14.436Z [INFO] stable state for Leader reached at 2024-01-08 00:31:14.330922676 +0000 UTC m=+20.377811867 (1 nodes), 2.64µs from start of poll, 184.813906ms from cluster start. Timeout at 2024-01-08 00:31:14.436783734 +0000 UTC m=+20.483672847, 105.86098ms after stability +testLoggerAdapter verbose: server-8942f1da-0549-78a5-321a-3384b25815a3: 2024-01-08T00:31:14.448Z [INFO] server-8942f1da-0549-78a5-321a-3384b25815a3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-583c32b5-8e2e-809d-cc03-feff3264cb4d Address:583c32b5-8e2e-809d-cc03-feff3264cb4d} {Suffrage:Voter ID:server-8942f1da-0549-78a5-321a-3384b25815a3 Address:8942f1da-0549-78a5-321a-3384b25815a3}]" + testing.go:149: server-8942f1da-0549-78a5-321a-3384b25815a3: 2024-01-08T00:31:14.448Z [INFO] server-8942f1da-0549-78a5-321a-3384b25815a3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-583c32b5-8e2e-809d-cc03-feff3264cb4d Address:583c32b5-8e2e-809d-cc03-feff3264cb4d} {Suffrage:Voter ID:server-8942f1da-0549-78a5-321a-3384b25815a3 Address:8942f1da-0549-78a5-321a-3384b25815a3}]" +testLoggerAdapter verbose: server-8942f1da-0549-78a5-321a-3384b25815a3: 2024-01-08T00:31:14.448Z [INFO] server-8942f1da-0549-78a5-321a-3384b25815a3: aborting pipeline replication: peer="{Voter server-583c32b5-8e2e-809d-cc03-feff3264cb4d 583c32b5-8e2e-809d-cc03-feff3264cb4d}" + testing.go:149: server-8942f1da-0549-78a5-321a-3384b25815a3: 2024-01-08T00:31:14.448Z [INFO] server-8942f1da-0549-78a5-321a-3384b25815a3: aborting pipeline replication: peer="{Voter server-583c32b5-8e2e-809d-cc03-feff3264cb4d 583c32b5-8e2e-809d-cc03-feff3264cb4d}" +--- PASS: TestRaft_GetConfigurationNoBootstrap (0.30s) === RUN TestTCPTransport_BadAddr --- PASS: TestTCPTransport_BadAddr (0.00s) === RUN TestTCPTransport_WithAdvertise @@ -3926,11 +4092,11 @@ === RUN TestMax --- PASS: TestMax (0.00s) === RUN TestGenerateUUID ---- PASS: TestGenerateUUID (0.00s) +--- PASS: TestGenerateUUID (0.01s) === RUN TestBackoff --- PASS: TestBackoff (0.00s) PASS -ok github.com/hashicorp/raft 19.713s +ok github.com/hashicorp/raft 20.524s ? github.com/hashicorp/raft/bench [no test files] create-stamp debian/debhelper-build-stamp fakeroot debian/rules binary @@ -3962,12 +4128,14 @@ dpkg-buildpackage: info: binary-only upload (no source included) dpkg-genchanges: info: including full source code in upload I: copying local configuration +I: user script /srv/workspace/pbuilder/15535/tmp/hooks/B01_cleanup starting +I: user script /srv/workspace/pbuilder/15535/tmp/hooks/B01_cleanup finished I: unmounting dev/ptmx filesystem I: unmounting dev/pts filesystem I: unmounting dev/shm filesystem I: unmounting proc filesystem I: unmounting sys filesystem I: cleaning the build env -I: removing directory /srv/workspace/pbuilder/849 and its subdirectories -I: Current time: Sat Feb 8 18:52:43 -12 2025 -I: pbuilder-time-stamp: 1739083963 +I: removing directory /srv/workspace/pbuilder/15535 and its subdirectories +I: Current time: Mon Jan 8 14:31:23 +14 2024 +I: pbuilder-time-stamp: 1704673883