Diff of the two buildlogs: -- --- b1/build.log 2022-03-18 11:27:44.158541859 +0000 +++ b2/build.log 2022-03-18 11:29:23.103748818 +0000 @@ -1,6 +1,6 @@ I: pbuilder: network access will be disabled during build -I: Current time: Thu Mar 17 23:18:16 -12 2022 -I: pbuilder-time-stamp: 1647602296 +I: Current time: Fri Apr 21 07:50:46 +14 2023 +I: pbuilder-time-stamp: 1682013046 I: Building the build Environment I: extracting base tarball [/var/cache/pbuilder/buster-reproducible-base.tgz] I: copying local configuration @@ -17,7 +17,7 @@ I: Extracting source gpgv: unknown type of key resource 'trustedkeys.kbx' gpgv: keyblock resource '/root/.gnupg/trustedkeys.kbx': General error -gpgv: Signature made Thu Aug 2 20:35:10 2018 -12 +gpgv: Signature made Fri Aug 3 22:35:10 2018 +14 gpgv: using RSA key 0740778A2DFC4A39C0C8BC8C8F2B113C6535C5A7 gpgv: Can't check signature: No public key dpkg-source: warning: failed to verify signature on ./golang-github-canonicalltd-raft-test_0.0~git20180628.c3345b5-1.dsc @@ -26,136 +26,170 @@ dpkg-source: info: unpacking golang-github-canonicalltd-raft-test_0.0~git20180628.c3345b5-1.debian.tar.xz I: using fakeroot in build. I: Installing the build-deps -I: user script /srv/workspace/pbuilder/1789904/tmp/hooks/D02_print_environment starting +I: user script /srv/workspace/pbuilder/274583/tmp/hooks/D01_modify_environment starting +debug: Running on ionos5-amd64. +I: Changing host+domainname to test build reproducibility +I: Adding a custom variable just for the fun of it... +I: Changing /bin/sh to bash +Removing 'diversion of /bin/sh to /bin/sh.distrib by dash' +Adding 'diversion of /bin/sh to /bin/sh.distrib by bash' +Removing 'diversion of /usr/share/man/man1/sh.1.gz to /usr/share/man/man1/sh.distrib.1.gz by dash' +Adding 'diversion of /usr/share/man/man1/sh.1.gz to /usr/share/man/man1/sh.distrib.1.gz by bash' +I: Setting pbuilder2's login shell to /bin/bash +I: Setting pbuilder2's GECOS to second user,second room,second work-phone,second home-phone,second other +I: user script /srv/workspace/pbuilder/274583/tmp/hooks/D01_modify_environment finished +I: user script /srv/workspace/pbuilder/274583/tmp/hooks/D02_print_environment starting I: set - BUILDDIR='/build' - BUILDUSERGECOS='first user,first room,first work-phone,first home-phone,first other' - BUILDUSERNAME='pbuilder1' - BUILD_ARCH='amd64' - DEBIAN_FRONTEND='noninteractive' - DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all parallel=15' - DISTRIBUTION='' - HOME='/root' - HOST_ARCH='amd64' + BASH=/bin/sh + BASHOPTS=checkwinsize:cmdhist:complete_fullquote:extquote:force_fignore:globasciiranges:hostcomplete:interactive_comments:progcomp:promptvars:sourcepath + BASH_ALIASES=() + BASH_ARGC=() + BASH_ARGV=() + BASH_CMDS=() + BASH_LINENO=([0]="12" [1]="0") + BASH_SOURCE=([0]="/tmp/hooks/D02_print_environment" [1]="/tmp/hooks/D02_print_environment") + BASH_VERSINFO=([0]="5" [1]="0" [2]="3" [3]="1" [4]="release" [5]="x86_64-pc-linux-gnu") + BASH_VERSION='5.0.3(1)-release' + BUILDDIR=/build + BUILDUSERGECOS='second user,second room,second work-phone,second home-phone,second other' + BUILDUSERNAME=pbuilder2 + BUILD_ARCH=amd64 + DEBIAN_FRONTEND=noninteractive + DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all parallel=16' + DIRSTACK=() + DISTRIBUTION= + EUID=0 + FUNCNAME=([0]="Echo" [1]="main") + GROUPS=() + HOME=/root + HOSTNAME=i-capture-the-hostname + HOSTTYPE=x86_64 + HOST_ARCH=amd64 IFS=' ' - INVOCATION_ID='31dc0540489649e2820c64aec64810ce' - LANG='C' - LANGUAGE='en_US:en' - LC_ALL='C' - MAIL='/var/mail/root' - OPTIND='1' - PATH='/usr/sbin:/usr/bin:/sbin:/bin:/usr/games' - PBCURRENTCOMMANDLINEOPERATION='build' - PBUILDER_OPERATION='build' - PBUILDER_PKGDATADIR='/usr/share/pbuilder' - PBUILDER_PKGLIBDIR='/usr/lib/pbuilder' - PBUILDER_SYSCONFDIR='/etc' - PPID='1789904' - PS1='# ' - PS2='> ' + INVOCATION_ID=43b0a2ba44da4444aa5a9bf888d621cb + LANG=C + LANGUAGE=et_EE:et + LC_ALL=C + MACHTYPE=x86_64-pc-linux-gnu + MAIL=/var/mail/root + OPTERR=1 + OPTIND=1 + OSTYPE=linux-gnu + PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path + PBCURRENTCOMMANDLINEOPERATION=build + PBUILDER_OPERATION=build + PBUILDER_PKGDATADIR=/usr/share/pbuilder + PBUILDER_PKGLIBDIR=/usr/lib/pbuilder + PBUILDER_SYSCONFDIR=/etc + PIPESTATUS=([0]="0") + POSIXLY_CORRECT=y + PPID=274583 PS4='+ ' - PWD='/' - SHELL='/bin/bash' - SHLVL='2' - SUDO_COMMAND='/usr/bin/timeout -k 18.1h 18h /usr/bin/ionice -c 3 /usr/bin/nice /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/tmp.cYbDoQfjPE/pbuilderrc_emY3 --hookdir /etc/pbuilder/first-build-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/buster-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/tmp.cYbDoQfjPE/b1 --logfile b1/build.log golang-github-canonicalltd-raft-test_0.0~git20180628.c3345b5-1.dsc' - SUDO_GID='111' - SUDO_UID='106' - SUDO_USER='jenkins' - TERM='unknown' - TZ='/usr/share/zoneinfo/Etc/GMT+12' - USER='root' - _='/usr/bin/systemd-run' - http_proxy='http://78.137.99.97:3128' + PWD=/ + SHELL=/bin/bash + SHELLOPTS=braceexpand:errexit:hashall:interactive-comments:posix + SHLVL=3 + SUDO_COMMAND='/usr/bin/timeout -k 24.1h 24h /usr/bin/ionice -c 3 /usr/bin/nice -n 11 /usr/bin/unshare --uts -- /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/tmp.cYbDoQfjPE/pbuilderrc_2BG9 --hookdir /etc/pbuilder/rebuild-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/buster-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/tmp.cYbDoQfjPE/b2 --logfile b2/build.log golang-github-canonicalltd-raft-test_0.0~git20180628.c3345b5-1.dsc' + SUDO_GID=110 + SUDO_UID=105 + SUDO_USER=jenkins + TERM=unknown + TZ=/usr/share/zoneinfo/Etc/GMT-14 + UID=0 + USER=root + _='I: set' + http_proxy=http://85.184.249.68:3128 I: uname -a - Linux ionos11-amd64 5.10.0-12-amd64 #1 SMP Debian 5.10.103-1 (2022-03-07) x86_64 GNU/Linux + Linux i-capture-the-hostname 5.16.0-0.bpo.3-amd64 #1 SMP PREEMPT Debian 5.16.11-1~bpo11+1 (2022-03-02) x86_64 GNU/Linux I: ls -l /bin total 5116 - -rwxr-xr-x 1 root root 1168776 Apr 17 2019 bash - -rwxr-xr-x 3 root root 38984 Jul 10 2019 bunzip2 - -rwxr-xr-x 3 root root 38984 Jul 10 2019 bzcat - lrwxrwxrwx 1 root root 6 Jul 10 2019 bzcmp -> bzdiff - -rwxr-xr-x 1 root root 2227 Jul 10 2019 bzdiff - lrwxrwxrwx 1 root root 6 Jul 10 2019 bzegrep -> bzgrep - -rwxr-xr-x 1 root root 4877 Jun 24 2019 bzexe - lrwxrwxrwx 1 root root 6 Jul 10 2019 bzfgrep -> bzgrep - -rwxr-xr-x 1 root root 3641 Jul 10 2019 bzgrep - -rwxr-xr-x 3 root root 38984 Jul 10 2019 bzip2 - -rwxr-xr-x 1 root root 14328 Jul 10 2019 bzip2recover - lrwxrwxrwx 1 root root 6 Jul 10 2019 bzless -> bzmore - -rwxr-xr-x 1 root root 1297 Jul 10 2019 bzmore - -rwxr-xr-x 1 root root 43744 Feb 28 2019 cat - -rwxr-xr-x 1 root root 64320 Feb 28 2019 chgrp - -rwxr-xr-x 1 root root 64288 Feb 28 2019 chmod - -rwxr-xr-x 1 root root 72512 Feb 28 2019 chown - -rwxr-xr-x 1 root root 146880 Feb 28 2019 cp - -rwxr-xr-x 1 root root 121464 Jan 17 2019 dash - -rwxr-xr-x 1 root root 109408 Feb 28 2019 date - -rwxr-xr-x 1 root root 76712 Feb 28 2019 dd - -rwxr-xr-x 1 root root 93744 Feb 28 2019 df - -rwxr-xr-x 1 root root 138856 Feb 28 2019 dir - -rwxr-xr-x 1 root root 84288 Jan 9 2019 dmesg - lrwxrwxrwx 1 root root 8 Sep 26 2018 dnsdomainname -> hostname - lrwxrwxrwx 1 root root 8 Sep 26 2018 domainname -> hostname - -rwxr-xr-x 1 root root 39520 Feb 28 2019 echo - -rwxr-xr-x 1 root root 28 Jan 7 2019 egrep - -rwxr-xr-x 1 root root 35424 Feb 28 2019 false - -rwxr-xr-x 1 root root 28 Jan 7 2019 fgrep - -rwxr-xr-x 1 root root 68880 Jan 9 2019 findmnt - -rwsr-xr-x 1 root root 34896 Apr 22 2020 fusermount - -rwxr-xr-x 1 root root 198976 Jan 7 2019 grep - -rwxr-xr-x 2 root root 2345 Jan 5 2019 gunzip - -rwxr-xr-x 1 root root 6375 Jan 5 2019 gzexe - -rwxr-xr-x 1 root root 98048 Jan 5 2019 gzip - -rwxr-xr-x 1 root root 26696 Sep 26 2018 hostname - -rwxr-xr-x 1 root root 68552 Feb 28 2019 ln - -rwxr-xr-x 1 root root 56760 Jul 26 2018 login - -rwxr-xr-x 1 root root 138856 Feb 28 2019 ls - -rwxr-xr-x 1 root root 108624 Jan 9 2019 lsblk - -rwxr-xr-x 1 root root 89088 Feb 28 2019 mkdir - -rwxr-xr-x 1 root root 68544 Feb 28 2019 mknod - -rwxr-xr-x 1 root root 43808 Feb 28 2019 mktemp - -rwxr-xr-x 1 root root 43008 Jan 9 2019 more - -rwsr-xr-x 1 root root 51280 Jan 9 2019 mount - -rwxr-xr-x 1 root root 14408 Jan 9 2019 mountpoint - -rwxr-xr-x 1 root root 138728 Feb 28 2019 mv - lrwxrwxrwx 1 root root 8 Sep 26 2018 nisdomainname -> hostname - lrwxrwxrwx 1 root root 14 Feb 14 2019 pidof -> /sbin/killall5 - -rwxr-xr-x 1 root root 39616 Feb 28 2019 pwd - lrwxrwxrwx 1 root root 4 Apr 17 2019 rbash -> bash - -rwxr-xr-x 1 root root 47776 Feb 28 2019 readlink - -rwxr-xr-x 1 root root 68416 Feb 28 2019 rm - -rwxr-xr-x 1 root root 47776 Feb 28 2019 rmdir - -rwxr-xr-x 1 root root 23312 Jan 21 2019 run-parts - -rwxr-xr-x 1 root root 122224 Dec 22 2018 sed - lrwxrwxrwx 1 root root 4 Mar 6 20:25 sh -> dash - -rwxr-xr-x 1 root root 39552 Feb 28 2019 sleep - -rwxr-xr-x 1 root root 80672 Feb 28 2019 stty - -rwsr-xr-x 1 root root 63568 Jan 9 2019 su - -rwxr-xr-x 1 root root 35488 Feb 28 2019 sync - -rwxr-xr-x 1 root root 445560 Apr 23 2019 tar - -rwxr-xr-x 1 root root 14440 Jan 21 2019 tempfile - -rwxr-xr-x 1 root root 97152 Feb 28 2019 touch - -rwxr-xr-x 1 root root 35424 Feb 28 2019 true - -rwxr-xr-x 1 root root 14328 Apr 22 2020 ulockmgr_server - -rwsr-xr-x 1 root root 34888 Jan 9 2019 umount - -rwxr-xr-x 1 root root 39584 Feb 28 2019 uname - -rwxr-xr-x 2 root root 2345 Jan 5 2019 uncompress - -rwxr-xr-x 1 root root 138856 Feb 28 2019 vdir - -rwxr-xr-x 1 root root 34896 Jan 9 2019 wdctl - -rwxr-xr-x 1 root root 946 Jan 21 2019 which - lrwxrwxrwx 1 root root 8 Sep 26 2018 ypdomainname -> hostname - -rwxr-xr-x 1 root root 1983 Jan 5 2019 zcat - -rwxr-xr-x 1 root root 1677 Jan 5 2019 zcmp - -rwxr-xr-x 1 root root 5879 Jan 5 2019 zdiff - -rwxr-xr-x 1 root root 29 Jan 5 2019 zegrep - -rwxr-xr-x 1 root root 29 Jan 5 2019 zfgrep - -rwxr-xr-x 1 root root 2080 Jan 5 2019 zforce - -rwxr-xr-x 1 root root 7584 Jan 5 2019 zgrep - -rwxr-xr-x 1 root root 2205 Jan 5 2019 zless - -rwxr-xr-x 1 root root 1841 Jan 5 2019 zmore - -rwxr-xr-x 1 root root 4552 Jan 5 2019 znew -I: user script /srv/workspace/pbuilder/1789904/tmp/hooks/D02_print_environment finished + -rwxr-xr-x 1 root root 1168776 Apr 18 2019 bash + -rwxr-xr-x 3 root root 38984 Jul 11 2019 bunzip2 + -rwxr-xr-x 3 root root 38984 Jul 11 2019 bzcat + lrwxrwxrwx 1 root root 6 Jul 11 2019 bzcmp -> bzdiff + -rwxr-xr-x 1 root root 2227 Jul 11 2019 bzdiff + lrwxrwxrwx 1 root root 6 Jul 11 2019 bzegrep -> bzgrep + -rwxr-xr-x 1 root root 4877 Jun 25 2019 bzexe + lrwxrwxrwx 1 root root 6 Jul 11 2019 bzfgrep -> bzgrep + -rwxr-xr-x 1 root root 3641 Jul 11 2019 bzgrep + -rwxr-xr-x 3 root root 38984 Jul 11 2019 bzip2 + -rwxr-xr-x 1 root root 14328 Jul 11 2019 bzip2recover + lrwxrwxrwx 1 root root 6 Jul 11 2019 bzless -> bzmore + -rwxr-xr-x 1 root root 1297 Jul 11 2019 bzmore + -rwxr-xr-x 1 root root 43744 Mar 1 2019 cat + -rwxr-xr-x 1 root root 64320 Mar 1 2019 chgrp + -rwxr-xr-x 1 root root 64288 Mar 1 2019 chmod + -rwxr-xr-x 1 root root 72512 Mar 1 2019 chown + -rwxr-xr-x 1 root root 146880 Mar 1 2019 cp + -rwxr-xr-x 1 root root 121464 Jan 18 2019 dash + -rwxr-xr-x 1 root root 109408 Mar 1 2019 date + -rwxr-xr-x 1 root root 76712 Mar 1 2019 dd + -rwxr-xr-x 1 root root 93744 Mar 1 2019 df + -rwxr-xr-x 1 root root 138856 Mar 1 2019 dir + -rwxr-xr-x 1 root root 84288 Jan 10 2019 dmesg + lrwxrwxrwx 1 root root 8 Sep 27 2018 dnsdomainname -> hostname + lrwxrwxrwx 1 root root 8 Sep 27 2018 domainname -> hostname + -rwxr-xr-x 1 root root 39520 Mar 1 2019 echo + -rwxr-xr-x 1 root root 28 Jan 8 2019 egrep + -rwxr-xr-x 1 root root 35424 Mar 1 2019 false + -rwxr-xr-x 1 root root 28 Jan 8 2019 fgrep + -rwxr-xr-x 1 root root 68880 Jan 10 2019 findmnt + -rwsr-xr-x 1 root root 34896 Apr 23 2020 fusermount + -rwxr-xr-x 1 root root 198976 Jan 8 2019 grep + -rwxr-xr-x 2 root root 2345 Jan 6 2019 gunzip + -rwxr-xr-x 1 root root 6375 Jan 6 2019 gzexe + -rwxr-xr-x 1 root root 98048 Jan 6 2019 gzip + -rwxr-xr-x 1 root root 26696 Sep 27 2018 hostname + -rwxr-xr-x 1 root root 68552 Mar 1 2019 ln + -rwxr-xr-x 1 root root 56760 Jul 27 2018 login + -rwxr-xr-x 1 root root 138856 Mar 1 2019 ls + -rwxr-xr-x 1 root root 108624 Jan 10 2019 lsblk + -rwxr-xr-x 1 root root 89088 Mar 1 2019 mkdir + -rwxr-xr-x 1 root root 68544 Mar 1 2019 mknod + -rwxr-xr-x 1 root root 43808 Mar 1 2019 mktemp + -rwxr-xr-x 1 root root 43008 Jan 10 2019 more + -rwsr-xr-x 1 root root 51280 Jan 10 2019 mount + -rwxr-xr-x 1 root root 14408 Jan 10 2019 mountpoint + -rwxr-xr-x 1 root root 138728 Mar 1 2019 mv + lrwxrwxrwx 1 root root 8 Sep 27 2018 nisdomainname -> hostname + lrwxrwxrwx 1 root root 14 Feb 15 2019 pidof -> /sbin/killall5 + -rwxr-xr-x 1 root root 39616 Mar 1 2019 pwd + lrwxrwxrwx 1 root root 4 Apr 18 2019 rbash -> bash + -rwxr-xr-x 1 root root 47776 Mar 1 2019 readlink + -rwxr-xr-x 1 root root 68416 Mar 1 2019 rm + -rwxr-xr-x 1 root root 47776 Mar 1 2019 rmdir + -rwxr-xr-x 1 root root 23312 Jan 22 2019 run-parts + -rwxr-xr-x 1 root root 122224 Dec 23 2018 sed + lrwxrwxrwx 1 root root 4 Apr 21 07:50 sh -> bash + lrwxrwxrwx 1 root root 4 Apr 10 04:47 sh.distrib -> dash + -rwxr-xr-x 1 root root 39552 Mar 1 2019 sleep + -rwxr-xr-x 1 root root 80672 Mar 1 2019 stty + -rwsr-xr-x 1 root root 63568 Jan 10 2019 su + -rwxr-xr-x 1 root root 35488 Mar 1 2019 sync + -rwxr-xr-x 1 root root 445560 Apr 24 2019 tar + -rwxr-xr-x 1 root root 14440 Jan 22 2019 tempfile + -rwxr-xr-x 1 root root 97152 Mar 1 2019 touch + -rwxr-xr-x 1 root root 35424 Mar 1 2019 true + -rwxr-xr-x 1 root root 14328 Apr 23 2020 ulockmgr_server + -rwsr-xr-x 1 root root 34888 Jan 10 2019 umount + -rwxr-xr-x 1 root root 39584 Mar 1 2019 uname + -rwxr-xr-x 2 root root 2345 Jan 6 2019 uncompress + -rwxr-xr-x 1 root root 138856 Mar 1 2019 vdir + -rwxr-xr-x 1 root root 34896 Jan 10 2019 wdctl + -rwxr-xr-x 1 root root 946 Jan 22 2019 which + lrwxrwxrwx 1 root root 8 Sep 27 2018 ypdomainname -> hostname + -rwxr-xr-x 1 root root 1983 Jan 6 2019 zcat + -rwxr-xr-x 1 root root 1677 Jan 6 2019 zcmp + -rwxr-xr-x 1 root root 5879 Jan 6 2019 zdiff + -rwxr-xr-x 1 root root 29 Jan 6 2019 zegrep + -rwxr-xr-x 1 root root 29 Jan 6 2019 zfgrep + -rwxr-xr-x 1 root root 2080 Jan 6 2019 zforce + -rwxr-xr-x 1 root root 7584 Jan 6 2019 zgrep + -rwxr-xr-x 1 root root 2205 Jan 6 2019 zless + -rwxr-xr-x 1 root root 1841 Jan 6 2019 zmore + -rwxr-xr-x 1 root root 4552 Jan 6 2019 znew +I: user script /srv/workspace/pbuilder/274583/tmp/hooks/D02_print_environment finished -> Attempting to satisfy build-dependencies -> Creating pbuilder-satisfydepends-dummy package Package: pbuilder-satisfydepends-dummy @@ -312,7 +346,7 @@ Get: 107 http://deb.debian.org/debian buster/main amd64 golang-github-hashicorp-go-msgpack-dev all 0.0~git20150518.0.fa3f638-6 [42.7 kB] Get: 108 http://deb.debian.org/debian buster/main amd64 golang-github-hashicorp-logutils-dev all 0.0~git20150609.0.0dc08b1-1 [8150 B] Get: 109 http://deb.debian.org/debian buster/main amd64 golang-github-hashicorp-raft-dev all 1.0.0+git20180823.82694fb-3 [85.9 kB] -Fetched 90.2 MB in 19s (4838 kB/s) +Fetched 90.2 MB in 2s (55.5 MB/s) debconf: delaying package configuration, since apt-utils is not installed Selecting previously unselected package libbsd0:amd64. (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 19195 files and directories currently installed.) @@ -771,7 +805,8 @@ fakeroot is already the newest version (1.23-1). 0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded. I: Building the package -I: Running cd /build/golang-github-canonicalltd-raft-test-0.0~git20180628.c3345b5/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-genchanges -S > ../golang-github-canonicalltd-raft-test_0.0~git20180628.c3345b5-1_source.changes +hostname: Name or service not known +I: Running cd /build/golang-github-canonicalltd-raft-test-0.0~git20180628.c3345b5/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-genchanges -S > ../golang-github-canonicalltd-raft-test_0.0~git20180628.c3345b5-1_source.changes dpkg-buildpackage: info: source package golang-github-canonicalltd-raft-test dpkg-buildpackage: info: source version 0.0~git20180628.c3345b5-1 dpkg-buildpackage: info: source distribution unstable @@ -789,30 +824,30 @@ dh_autoreconf -O--buildsystem=golang dh_auto_configure -O--buildsystem=golang dh_auto_build -O--buildsystem=golang - cd obj-x86_64-linux-gnu && go install -gcflags=all=\"-trimpath=/build/golang-github-canonicalltd-raft-test-0.0\~git20180628.c3345b5/obj-x86_64-linux-gnu/src\" -asmflags=all=\"-trimpath=/build/golang-github-canonicalltd-raft-test-0.0\~git20180628.c3345b5/obj-x86_64-linux-gnu/src\" -v -p 15 github.com/CanonicalLtd/raft-test github.com/CanonicalLtd/raft-test/internal/election github.com/CanonicalLtd/raft-test/internal/event github.com/CanonicalLtd/raft-test/internal/fsms github.com/CanonicalLtd/raft-test/internal/logging github.com/CanonicalLtd/raft-test/internal/network + cd obj-x86_64-linux-gnu && go install -gcflags=all=\"-trimpath=/build/golang-github-canonicalltd-raft-test-0.0\~git20180628.c3345b5/obj-x86_64-linux-gnu/src\" -asmflags=all=\"-trimpath=/build/golang-github-canonicalltd-raft-test-0.0\~git20180628.c3345b5/obj-x86_64-linux-gnu/src\" -v -p 16 github.com/CanonicalLtd/raft-test github.com/CanonicalLtd/raft-test/internal/election github.com/CanonicalLtd/raft-test/internal/event github.com/CanonicalLtd/raft-test/internal/fsms github.com/CanonicalLtd/raft-test/internal/logging github.com/CanonicalLtd/raft-test/internal/network errors -runtime/internal/sys unicode/utf8 +internal/race +runtime/internal/sys math/bits +unicode +crypto/subtle crypto/internal/subtle container/list -unicode -internal/race -runtime/internal/atomic vendor/golang_org/x/net/dns/dnsmessage -crypto/subtle -unicode/utf16 -internal/nettrace -encoding -vendor/golang_org/x/crypto/cryptobyte/asn1 sync/atomic +vendor/golang_org/x/crypto/cryptobyte/asn1 internal/cpu -github.com/CanonicalLtd/raft-test/internal/event -vendor/golang_org/x/crypto/curve25519 +runtime/internal/atomic +encoding +internal/nettrace runtime/cgo -github.com/hashicorp/golang-lru/simplelru -internal/testlog +unicode/utf16 +vendor/golang_org/x/crypto/curve25519 +github.com/CanonicalLtd/raft-test/internal/event internal/bytealg +internal/testlog +github.com/hashicorp/golang-lru/simplelru math runtime strconv @@ -821,8 +856,8 @@ io syscall math/rand -internal/singleflight reflect +internal/singleflight bytes crypto/cipher strings @@ -845,40 +880,40 @@ internal/syscall/unix time internal/poll -os -os/signal -fmt encoding/binary sort +os github.com/hashicorp/go-immutable-radix -runtime/debug -path/filepath encoding/base64 crypto/des vendor/golang_org/x/crypto/internal/chacha20 vendor/golang_org/x/crypto/poly1305 -io/ioutil -encoding/pem vendor/golang_org/x/crypto/chacha20poly1305 +encoding/pem +path/filepath +os/signal +runtime/debug +fmt +io/ioutil context +math/big log -encoding/json compress/flate +encoding/hex net/url +encoding/json vendor/golang_org/x/net/http2/hpack -math/big -mime mime/quotedprintable -vendor/golang_org/x/text/unicode/norm -encoding/hex net/http/internal -text/template/parse +mime +vendor/golang_org/x/text/unicode/norm +flag github.com/pkg/errors encoding/gob +text/template/parse vendor/golang_org/x/text/unicode/bidi -flag -runtime/trace net +runtime/trace testing compress/gzip vendor/golang_org/x/text/secure/bidirule @@ -887,16 +922,16 @@ vendor/golang_org/x/net/idna crypto/rand encoding/asn1 -crypto/dsa crypto/elliptic -html/template +crypto/dsa crypto/rsa +html/template crypto/x509/pkix vendor/golang_org/x/crypto/cryptobyte crypto/ecdsa -vendor/golang_org/x/net/http/httpproxy net/textproto crypto/x509 +vendor/golang_org/x/net/http/httpproxy vendor/golang_org/x/net/http/httpguts mime/multipart crypto/tls @@ -911,1566 +946,1715 @@ github.com/CanonicalLtd/raft-test/internal/network github.com/CanonicalLtd/raft-test dh_auto_test -O--buildsystem=golang - cd obj-x86_64-linux-gnu && go test -vet=off -v -p 15 github.com/CanonicalLtd/raft-test github.com/CanonicalLtd/raft-test/internal/election github.com/CanonicalLtd/raft-test/internal/event github.com/CanonicalLtd/raft-test/internal/fsms github.com/CanonicalLtd/raft-test/internal/logging github.com/CanonicalLtd/raft-test/internal/network + cd obj-x86_64-linux-gnu && go test -vet=off -v -p 16 github.com/CanonicalLtd/raft-test github.com/CanonicalLtd/raft-test/internal/election github.com/CanonicalLtd/raft-test/internal/event github.com/CanonicalLtd/raft-test/internal/fsms github.com/CanonicalLtd/raft-test/internal/logging github.com/CanonicalLtd/raft-test/internal/network === RUN TestCluster_Default --- PASS: TestCluster_Default (0.00s) - logger.go:44: 11:26:55.429169 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 11:26:55.429380 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 11:26:55.429516 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 11:26:55.429579 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 11:26:55.429608 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 11:26:55.429799 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:55.429837 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 11:26:55.429887 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:55.429906 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 11:26:55.429956 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:55.429985 [DEBUG] raft-test: setup: done - logger.go:44: 11:26:55.430040 [DEBUG] raft-test: close: start - logger.go:44: 11:26:55.430275 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:55.430364 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 11:26:55.430395 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:55.430429 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 11:26:55.430930 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 11:26:55.430967 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 11:26:55.430982 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 11:26:55.431002 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 11:26:55.431013 [DEBUG] raft-test: close: done + logger.go:44: 17:52:12.721859 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 17:52:12.722041 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 17:52:12.722155 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 17:52:12.722186 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 17:52:12.722211 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 17:52:12.722340 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:12.722364 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 17:52:12.722405 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:12.722422 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 17:52:12.722458 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:12.722473 [DEBUG] raft-test: setup: done + logger.go:44: 17:52:12.722509 [DEBUG] raft-test: close: start + logger.go:44: 17:52:12.722723 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:12.722800 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:12.722849 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 17:52:12.722886 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 17:52:12.722919 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 17:52:12.722935 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 17:52:12.722948 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 17:52:12.722964 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 17:52:12.722975 [DEBUG] raft-test: close: done === RUN TestControl_ElectAndDepose ---- PASS: TestControl_ElectAndDepose (0.13s) - logger.go:44: 11:26:55.431130 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 11:26:55.431187 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 11:26:55.431257 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 11:26:55.431280 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 11:26:55.431661 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 11:26:55.431748 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 11:26:55.431897 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 11:26:55.431952 [DEBUG] raft-test: setup: done - logger.go:44: 11:26:55.431995 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 11:26:55.432028 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 11:26:55.432052 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 11:26:55.552049 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=6,index=2] - logger.go:44: 11:26:55.552245 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 11:26:55.552268 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 11:26:55.552308 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=6,index=2] - logger.go:44: 11:26:55.552436 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 11:26:55.552450 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 11:26:55.552460 [DEBUG] raft-test: elect: done - logger.go:44: 11:26:55.552523 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 11:26:55.552537 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 11:26:55.555440 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 11:26:55.555500 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 11:26:55.555567 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 11:26:55.555617 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected - logger.go:44: 11:26:55.555627 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:55.555694 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:55.557202 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:55.557233 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:55.557509 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:55.557526 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:55.563641 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:55.563740 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:55.563648 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 11:26:55.563819 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 11:26:55.563859 [DEBUG] raft-test: close: start - logger.go:44: 11:26:55.563913 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:55.563928 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:55.563963 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:55.563976 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:55.564009 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 11:26:55.564054 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 11:26:55.564082 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 11:26:55.564095 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 11:26:55.564120 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 11:26:55.564140 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 11:26:55.564164 [DEBUG] raft-test: close: done +--- PASS: TestControl_ElectAndDepose (0.03s) + logger.go:44: 17:52:12.723050 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 17:52:12.723104 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 17:52:12.723158 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 17:52:12.723185 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 17:52:12.723212 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 17:52:12.723261 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 17:52:12.723308 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 17:52:12.723350 [DEBUG] raft-test: setup: done + logger.go:44: 17:52:12.723365 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 17:52:12.723387 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 17:52:12.723421 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 17:52:12.739315 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] + logger.go:44: 17:52:12.739457 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 17:52:12.739480 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 17:52:12.739517 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] + logger.go:44: 17:52:12.739615 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 17:52:12.739631 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 17:52:12.739642 [DEBUG] raft-test: elect: done + logger.go:44: 17:52:12.739679 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 17:52:12.739697 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 17:52:12.741148 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 17:52:12.741175 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected + logger.go:44: 17:52:12.741494 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:12.742442 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:12.742854 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 17:52:12.742873 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 17:52:12.743334 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:12.743407 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:12.745622 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:12.745678 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:12.751227 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 17:52:12.751268 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 17:52:12.751294 [DEBUG] raft-test: close: start + logger.go:44: 17:52:12.751338 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:12.751371 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:12.751415 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:12.751430 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:12.751479 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 17:52:12.751535 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 17:52:12.751572 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 17:52:12.751587 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 17:52:12.751615 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 17:52:12.751634 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 17:52:12.751645 [DEBUG] raft-test: close: done === RUN TestControl_DeposeAfterCommandEnqueued ---- PASS: TestControl_DeposeAfterCommandEnqueued (0.04s) - logger.go:44: 11:26:55.564274 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 11:26:55.564367 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 11:26:55.564457 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 11:26:55.564494 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 11:26:55.564526 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 11:26:55.564630 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:55.564647 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 11:26:55.564716 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:55.564731 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 11:26:55.564766 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:55.564781 [DEBUG] raft-test: setup: done - logger.go:44: 11:26:55.564796 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 11:26:55.564822 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 11:26:55.564848 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 11:26:55.564878 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:55.564906 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:55.564942 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:55.586543 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 11:26:55.586641 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 11:26:55.586733 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:55.586750 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 11:26:55.586813 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 2 - logger.go:44: 11:26:55.586830 [INFO] raft: Election won. Tally: 2 - logger.go:44: 11:26:55.586844 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 11:26:55.586867 [INFO] raft: Added peer 1, starting replication - logger.go:44: 11:26:55.586882 [INFO] raft: Added peer 2, starting replication - logger.go:44: 11:26:55.586952 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] - logger.go:44: 11:26:55.587029 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 11:26:55.587091 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 11:26:55.587110 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 11:26:55.587165 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] - logger.go:44: 11:26:55.587234 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 11:26:55.587366 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 11:26:55.587390 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 11:26:55.587415 [DEBUG] raft-test: elect: done - logger.go:44: 11:26:55.587468 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] - logger.go:44: 11:26:55.587662 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] - logger.go:44: 11:26:55.587779 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 11:26:55.587825 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 11:26:55.587831 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=4] - logger.go:44: 11:26:55.587854 [DEBUG] raft-test: server 0: pipeline: append to: 1: enqueue fault: command 2 - logger.go:44: 11:26:55.587838 [DEBUG] raft-test: server 0: pipeline: append to: 2: enqueue fault: command 2 - logger.go:44: 11:26:55.587946 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 11:26:55.587965 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 11:26:55.588025 [DEBUG] raft-test: fsm 2: applied 1 - logger.go:44: 11:26:55.588052 [DEBUG] raft-test: fsm 1: applied 1 - logger.go:44: 11:26:55.590458 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:55.590496 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:55.590510 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 11:26:55.595458 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:55.595510 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:55.595527 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 11:26:55.605814 [WARN] raft: Failed to contact 1 in 17.759175ms - logger.go:44: 11:26:55.605869 [WARN] raft: Failed to contact 2 in 17.700619ms - logger.go:44: 11:26:55.605903 [WARN] raft: Failed to contact quorum of nodes, stepping down - logger.go:44: 11:26:55.605952 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:55.606001 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 11:26:55.606019 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 11:26:55.606045 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 11:26:55.606075 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 11:26:55.606130 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 11:26:55.606178 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 11:26:55.606242 [DEBUG] raft-test: close: start - logger.go:44: 11:26:55.606628 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 11:26:55.606728 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 11:26:55.606989 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 11:26:55.607007 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 11:26:55.607069 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 11:26:55.607090 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 11:26:55.607101 [DEBUG] raft-test: close: done +--- PASS: TestControl_DeposeAfterCommandEnqueued (0.11s) + logger.go:44: 17:52:12.751751 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 17:52:12.751827 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 17:52:12.751900 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 17:52:12.751930 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 17:52:12.751956 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 17:52:12.752018 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:12.752035 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 17:52:12.752075 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:12.752091 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 17:52:12.752138 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:12.752152 [DEBUG] raft-test: setup: done + logger.go:44: 17:52:12.752168 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 17:52:12.752188 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 17:52:12.752208 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 17:52:12.752233 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:12.752260 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:12.752287 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:12.775847 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 17:52:12.775907 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 2 + logger.go:44: 17:52:12.775971 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:12.775998 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 1 + logger.go:44: 17:52:12.776041 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 17:52:12.776042 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:12.780077 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 17:52:12.780111 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 2 + logger.go:44: 17:52:12.780166 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:12.780180 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 1 + logger.go:44: 17:52:12.780212 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 17:52:12.780240 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:12.780264 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 17:52:12.780278 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 17:52:12.780316 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:12.780328 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 17:52:12.780368 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 17:52:12.780404 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 17:52:12.800631 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:12.800691 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 3 + logger.go:44: 17:52:12.800734 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:12.800771 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 1 + logger.go:44: 17:52:12.800801 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:12.800828 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 17:52:12.809443 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:12.809516 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 + logger.go:44: 17:52:12.809565 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:12.809580 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 + logger.go:44: 17:52:12.809634 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 17:52:12.809661 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:12.809846 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:12.809905 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 3 + logger.go:44: 17:52:12.809948 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:12.809962 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 1 + logger.go:44: 17:52:12.810014 [INFO] raft: Duplicate RequestVote for same term: 3 + logger.go:44: 17:52:12.810055 [INFO] raft: Duplicate RequestVote for same term: 3 + logger.go:44: 17:52:12.833859 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:12.833917 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 4 + logger.go:44: 17:52:12.833982 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:12.833997 [DEBUG] raft: Vote granted from 0 in term 4. Tally: 1 + logger.go:44: 17:52:12.834055 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:12.834073 [DEBUG] raft: Vote granted from 2 in term 4. Tally: 2 + logger.go:44: 17:52:12.834083 [INFO] raft: Election won. Tally: 2 + logger.go:44: 17:52:12.834096 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 17:52:12.834113 [INFO] raft: Added peer 1, starting replication + logger.go:44: 17:52:12.834127 [INFO] raft: Added peer 2, starting replication + logger.go:44: 17:52:12.834179 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=4,index=2] + logger.go:44: 17:52:12.834246 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 17:52:12.834293 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 17:52:12.834311 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 17:52:12.834345 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=4,index=2] + logger.go:44: 17:52:12.834423 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:12.834436 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 4 + logger.go:44: 17:52:12.834458 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:12.834471 [INFO] raft: Duplicate RequestVote for same term: 4 + logger.go:44: 17:52:12.834494 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "0") + logger.go:44: 17:52:12.834517 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 17:52:12.834549 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:12.834571 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 17:52:12.839886 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 17:52:12.839966 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 17:52:12.839980 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 17:52:12.839990 [DEBUG] raft-test: elect: done + logger.go:44: 17:52:12.840049 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=4,index=3] + logger.go:44: 17:52:12.840123 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 17:52:12.840161 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=4,index=4] + logger.go:44: 17:52:12.840173 [DEBUG] raft-test: server 0: pipeline: append to: 1: enqueue fault: command 2 + logger.go:44: 17:52:12.840214 [DEBUG] raft-test: fsm 1: applied 1 + logger.go:44: 17:52:12.840234 [DEBUG] raft-test: server 0: pipeline: append to 2: 2 entries [Command:term=4,index=3 Command:term=4,index=4] + logger.go:44: 17:52:12.840246 [DEBUG] raft-test: server 0: pipeline: append to: 2: enqueue fault: command 2 + logger.go:44: 17:52:12.840273 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 17:52:12.840289 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 17:52:12.840323 [DEBUG] raft-test: fsm 2: applied 1 + logger.go:44: 17:52:12.840348 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:12.840361 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:12.840374 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 17:52:12.840388 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:12.840400 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:12.840410 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 17:52:12.857835 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:12.858914 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:12.858947 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 17:52:12.857837 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 17:52:12.859022 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 5 + logger.go:44: 17:52:12.859051 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:12.859063 [DEBUG] raft: Vote granted from 1 in term 5. Tally: 1 + logger.go:44: 17:52:12.859100 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 17:52:12.857900 [WARN] raft: Failed to contact 1 in 17.611889ms + logger.go:44: 17:52:12.859130 [WARN] raft: Failed to contact 2 in 17.472207ms + logger.go:44: 17:52:12.859138 [WARN] raft: Failed to contact quorum of nodes, stepping down + logger.go:44: 17:52:12.859157 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:12.859173 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 17:52:12.859185 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 17:52:12.859198 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 17:52:12.859212 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 17:52:12.859239 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:12.857914 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:12.859260 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:12.859268 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 17:52:12.859303 [DEBUG] raft-test: close: start + logger.go:44: 17:52:12.859325 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 17:52:12.859336 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 17:52:12.859365 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 17:52:12.859388 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 17:52:12.859411 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 17:52:12.859421 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 17:52:12.859437 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 17:52:12.859451 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 17:52:12.859458 [DEBUG] raft-test: close: done === RUN TestControl_DeposeAfterCommandEnqueuedThenElect --- PASS: TestControl_DeposeAfterCommandEnqueuedThenElect (0.00s) === RUN TestControl_DeposeAfterCommandEnqueuedThenElectSame ---- PASS: TestControl_DeposeAfterCommandEnqueuedThenElectSame (0.17s) - logger.go:44: 11:26:55.607434 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 11:26:55.607983 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 11:26:55.608097 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 11:26:55.608174 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 11:26:55.608198 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 11:26:55.608301 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 11:26:55.608391 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 11:26:55.608443 [DEBUG] raft-test: setup: done - logger.go:44: 11:26:55.608456 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 11:26:55.608494 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 11:26:55.608510 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 11:26:55.711102 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=6,index=2] - logger.go:44: 11:26:55.711181 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 11:26:55.711200 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 11:26:55.711275 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=6,index=2] - logger.go:44: 11:26:55.713958 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 11:26:55.714087 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 11:26:55.714109 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 11:26:55.714121 [DEBUG] raft-test: elect: done - logger.go:44: 11:26:55.714209 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=6,index=3] - logger.go:44: 11:26:55.714411 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=6,index=3] - logger.go:44: 11:26:55.714452 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:55.714486 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 11:26:55.714412 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 11:26:55.714550 [DEBUG] raft-test: fsm 1: applied 1 - logger.go:44: 11:26:55.714553 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:55.714607 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=6,index=4] - logger.go:44: 11:26:55.714622 [DEBUG] raft-test: server 0: pipeline: append to: 2: enqueue fault: command 2 - logger.go:44: 11:26:55.714651 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=6,index=4] - logger.go:44: 11:26:55.714666 [DEBUG] raft-test: server 0: pipeline: append to: 1: enqueue fault: command 2 - logger.go:44: 11:26:55.714694 [DEBUG] raft-test: fsm 2: applied 1 - logger.go:44: 11:26:55.714736 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 11:26:55.714756 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 11:26:55.719417 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:55.719470 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:55.719566 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:55.719668 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:55.732722 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 11:26:55.732755 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 11:26:55.732878 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:55.732975 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:55.732937 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 11:26:55.733028 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 11:26:55.733049 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 11:26:55.779761 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=9,index=5] - logger.go:44: 11:26:55.779874 [DEBUG] raft-test: server 0: transport: append to 2: 2 entries [Command:term=6,index=4 Noop:term=9,index=5] - logger.go:44: 11:26:55.779893 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 11:26:55.779956 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=9,index=5] - logger.go:44: 11:26:55.780013 [DEBUG] raft-test: server 0: transport: append to 1: 2 entries [Command:term=6,index=4 Noop:term=9,index=5] - logger.go:44: 11:26:55.779973 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 11:26:55.780083 [DEBUG] raft-test: fsm 0: applied 2 - logger.go:44: 11:26:55.780098 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 11:26:55.780121 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 11:26:55.780130 [DEBUG] raft-test: elect: done - logger.go:44: 11:26:55.780159 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=9,index=6] - logger.go:44: 11:26:55.780239 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=9,index=6] - logger.go:44: 11:26:55.780294 [DEBUG] raft-test: fsm 0: applied 3 - logger.go:44: 11:26:55.780302 [DEBUG] raft-test: fsm 2: applied 2 - logger.go:44: 11:26:55.780317 [DEBUG] raft-test: fsm 1: applied 2 - logger.go:44: 11:26:55.780342 [DEBUG] raft-test: close: start - logger.go:44: 11:26:55.780430 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 11:26:55.780463 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 11:26:55.780495 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 11:26:55.780508 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 11:26:55.780529 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 11:26:55.780552 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 11:26:55.780560 [DEBUG] raft-test: close: done +--- PASS: TestControl_DeposeAfterCommandEnqueuedThenElectSame (0.11s) + logger.go:44: 17:52:12.859573 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 17:52:12.859644 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 17:52:12.859709 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 17:52:12.859731 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 17:52:12.859750 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 17:52:12.859805 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 17:52:12.859839 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 17:52:12.859868 [DEBUG] raft-test: setup: done + logger.go:44: 17:52:12.859878 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 17:52:12.859891 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 17:52:12.859906 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 17:52:12.937989 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=4,index=2] + logger.go:44: 17:52:12.938101 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 17:52:12.938119 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 17:52:12.938152 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=4,index=2] + logger.go:44: 17:52:12.938240 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 17:52:12.938252 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 17:52:12.938261 [DEBUG] raft-test: elect: done + logger.go:44: 17:52:12.938304 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=4,index=3] + logger.go:44: 17:52:12.938363 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 17:52:12.938401 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=4,index=4] + logger.go:44: 17:52:12.938424 [DEBUG] raft-test: server 0: pipeline: append to: 2: enqueue fault: command 2 + logger.go:44: 17:52:12.938467 [DEBUG] raft-test: server 0: pipeline: append to 1: 2 entries [Command:term=4,index=3 Command:term=4,index=4] + logger.go:44: 17:52:12.938482 [DEBUG] raft-test: server 0: pipeline: append to: 1: enqueue fault: command 2 + logger.go:44: 17:52:12.938505 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 17:52:12.938522 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 17:52:12.938558 [DEBUG] raft-test: fsm 1: applied 1 + logger.go:44: 17:52:12.938579 [DEBUG] raft-test: fsm 2: applied 1 + logger.go:44: 17:52:12.941955 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:12.942010 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:12.941989 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:12.942063 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:12.951288 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 17:52:12.951330 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 17:52:12.951394 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 17:52:12.951410 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 17:52:12.951427 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 17:52:12.951463 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=4,index=4] + logger.go:44: 17:52:12.973234 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=5,index=5] + logger.go:44: 17:52:12.973367 [DEBUG] raft-test: server 0: transport: append to 2: 2 entries [Command:term=4,index=4 Noop:term=5,index=5] + logger.go:44: 17:52:12.973428 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 17:52:12.973446 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 17:52:12.973478 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=5,index=5] + logger.go:44: 17:52:12.973566 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 17:52:12.973578 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 17:52:12.973587 [DEBUG] raft-test: elect: done + logger.go:44: 17:52:12.973615 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=5,index=6] + logger.go:44: 17:52:12.973645 [DEBUG] raft-test: fsm 2: applied 2 + logger.go:44: 17:52:12.973698 [DEBUG] raft-test: fsm 0: applied 2 + logger.go:44: 17:52:12.973710 [DEBUG] raft-test: fsm 0: applied 3 + logger.go:44: 17:52:12.973736 [DEBUG] raft-test: close: start + logger.go:44: 17:52:12.973792 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=5,index=6] + logger.go:44: 17:52:12.973845 [DEBUG] raft-test: fsm 1: applied 2 + logger.go:44: 17:52:12.973858 [DEBUG] raft-test: fsm 1: applied 3 + logger.go:44: 17:52:12.973912 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 17:52:12.973987 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 17:52:12.974028 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 17:52:12.974047 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 17:52:12.974078 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 17:52:12.974092 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 17:52:12.974102 [DEBUG] raft-test: close: done === RUN TestControl_DeposeAfterCommandAppended ---- PASS: TestControl_DeposeAfterCommandAppended (0.05s) - logger.go:44: 11:26:55.780649 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 11:26:55.780744 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 11:26:55.780809 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 11:26:55.780832 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 11:26:55.780861 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 11:26:55.780923 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 11:26:55.780983 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 11:26:55.781039 [DEBUG] raft-test: setup: done - logger.go:44: 11:26:55.781051 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 11:26:55.781064 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 11:26:55.781087 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 11:26:55.805834 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] - logger.go:44: 11:26:55.805960 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 11:26:55.805991 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 11:26:55.806068 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] - logger.go:44: 11:26:55.809870 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:55.811431 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 11:26:55.811507 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 11:26:55.811525 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 11:26:55.811536 [DEBUG] raft-test: elect: done - logger.go:44: 11:26:55.811588 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 11:26:55.811756 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] - logger.go:44: 11:26:55.811772 [DEBUG] raft-test: server 0: pipeline: append to 2: append fault: command 1 - logger.go:44: 11:26:55.811555 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:55.811881 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] - logger.go:44: 11:26:55.811904 [DEBUG] raft-test: server 0: pipeline: append to 1: append fault: command 1 - logger.go:44: 11:26:55.811965 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 11:26:55.811983 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 11:26:55.813517 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 11:26:55.813538 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected - logger.go:44: 11:26:55.813594 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:55.813607 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:55.813624 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:55.813643 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:55.813666 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 11:26:55.813686 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 11:26:55.815355 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=3] - logger.go:44: 11:26:55.815382 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:55.815413 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=2,index=3] - logger.go:44: 11:26:55.815451 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:55.827274 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=2,index=3] - logger.go:44: 11:26:55.827312 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:55.827508 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 11:26:55.827557 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 11:26:55.827612 [DEBUG] raft-test: close: start - logger.go:44: 11:26:55.827661 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 11:26:55.827694 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=3] - logger.go:44: 11:26:55.827711 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:55.827765 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 11:26:55.827795 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 11:26:55.827815 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 11:26:55.827850 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 11:26:55.827885 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 11:26:55.827894 [DEBUG] raft-test: close: done +--- PASS: TestControl_DeposeAfterCommandAppended (0.10s) + logger.go:44: 17:52:12.974185 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 17:52:12.974241 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 17:52:12.974308 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 17:52:12.974332 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 17:52:12.974375 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 17:52:12.974433 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 17:52:12.974470 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 17:52:12.974527 [DEBUG] raft-test: setup: done + logger.go:44: 17:52:12.974543 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 17:52:12.974563 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 17:52:12.974579 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 17:52:13.046115 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=4,index=2] + logger.go:44: 17:52:13.046213 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 17:52:13.046230 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 17:52:13.046259 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=4,index=2] + logger.go:44: 17:52:13.049900 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 17:52:13.049967 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 17:52:13.049983 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 17:52:13.049992 [DEBUG] raft-test: elect: done + logger.go:44: 17:52:13.050054 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=4,index=3] + logger.go:44: 17:52:13.050078 [DEBUG] raft-test: server 0: pipeline: append to 2: append fault: command 1 + logger.go:44: 17:52:13.050133 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=4,index=3] + logger.go:44: 17:52:13.050145 [DEBUG] raft-test: server 0: pipeline: append to 1: append fault: command 1 + logger.go:44: 17:52:13.050164 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 17:52:13.050215 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 17:52:13.050247 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 17:52:13.050272 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:13.050284 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:13.050309 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:13.050320 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:13.051976 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 17:52:13.052034 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected + logger.go:44: 17:52:13.052101 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 17:52:13.052131 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 17:52:13.053808 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=4,index=3] + logger.go:44: 17:52:13.053832 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:13.054045 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=4,index=3] + logger.go:44: 17:52:13.054063 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:13.068065 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:13.068134 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:13.068177 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:13.068190 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:13.069854 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=4,index=3] + logger.go:44: 17:52:13.069918 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:13.069932 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 17:52:13.069955 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 17:52:13.069855 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=4,index=3] + logger.go:44: 17:52:13.069997 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:13.070074 [DEBUG] raft-test: close: start + logger.go:44: 17:52:13.070146 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 17:52:13.070175 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 17:52:13.070200 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 17:52:13.070214 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 17:52:13.070240 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 17:52:13.070259 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 17:52:13.070269 [DEBUG] raft-test: close: done === RUN TestControl_DeposeAfterCommandAppendedThenElect === RUN TestControl_DeposeAfterCommandAppendedThenElect/0 === RUN TestControl_DeposeAfterCommandAppendedThenElect/1 === RUN TestControl_DeposeAfterCommandAppendedThenElect/2 ---- PASS: TestControl_DeposeAfterCommandAppendedThenElect (0.32s) - --- PASS: TestControl_DeposeAfterCommandAppendedThenElect/0 (0.10s) - logger.go:44: 11:26:55.828029 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 11:26:55.828100 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 11:26:55.828178 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 11:26:55.828213 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 11:26:55.828241 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 11:26:55.828310 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:55.828326 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 11:26:55.828369 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:55.828384 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 11:26:55.828421 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:55.828435 [DEBUG] raft-test: setup: done - logger.go:44: 11:26:55.828448 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 11:26:55.828480 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 11:26:55.828507 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 11:26:55.828533 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:55.828570 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:55.828595 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:55.850232 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 11:26:55.850312 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 11:26:55.850389 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:55.850405 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 11:26:55.850478 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 2 - logger.go:44: 11:26:55.850488 [INFO] raft: Election won. Tally: 2 - logger.go:44: 11:26:55.850510 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 11:26:55.850539 [INFO] raft: Added peer 1, starting replication - logger.go:44: 11:26:55.850558 [INFO] raft: Added peer 2, starting replication - logger.go:44: 11:26:55.850617 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] - logger.go:44: 11:26:55.850678 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 11:26:55.850811 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 11:26:55.850842 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 11:26:55.850846 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] - logger.go:44: 11:26:55.850927 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 11:26:55.851998 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 11:26:55.852020 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 11:26:55.852031 [DEBUG] raft-test: elect: done - logger.go:44: 11:26:55.852122 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] - logger.go:44: 11:26:55.852171 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] - logger.go:44: 11:26:55.852243 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 11:26:55.852293 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 11:26:55.852307 [DEBUG] raft-test: server 0: pipeline: append to 2: append fault: command 2 - logger.go:44: 11:26:55.852357 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=4] - logger.go:44: 11:26:55.852371 [DEBUG] raft-test: server 0: pipeline: append to 1: append fault: command 2 - logger.go:44: 11:26:55.852441 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 11:26:55.852458 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 11:26:55.852502 [DEBUG] raft-test: fsm 2: applied 1 - logger.go:44: 11:26:55.852528 [DEBUG] raft-test: fsm 1: applied 1 - logger.go:44: 11:26:55.854329 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:55.854349 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 11:26:55.854407 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 11:26:55.854429 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 11:26:55.854450 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 11:26:55.854468 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:55.854508 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:55.854521 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 11:26:55.854340 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 11:26:55.854557 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected - logger.go:44: 11:26:55.854575 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 11:26:55.854592 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 11:26:55.854369 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:55.854636 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 11:26:55.856752 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 11:26:55.856783 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:55.856827 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 11:26:55.856947 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=2,index=4] - logger.go:44: 11:26:55.856964 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:55.856978 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 11:26:55.869594 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 11:26:55.869649 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:55.869684 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 11:26:55.871506 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:55.871545 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:55.871558 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 11:26:55.871576 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:55.871597 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:55.871608 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 11:26:55.873921 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 11:26:55.873978 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=2,index=4] - logger.go:44: 11:26:55.874006 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:55.874027 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 11:26:55.873980 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 - logger.go:44: 11:26:55.874096 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:55.874123 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 - logger.go:44: 11:26:55.874158 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 11:26:55.873927 [WARN] raft: Failed to contact 1 in 21.552776ms - logger.go:44: 11:26:55.874192 [WARN] raft: Failed to contact 2 in 21.686249ms - logger.go:44: 11:26:55.874209 [WARN] raft: Failed to contact quorum of nodes, stepping down - logger.go:44: 11:26:55.874248 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:55.874268 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 11:26:55.874295 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 11:26:55.874325 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 11:26:55.874393 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 11:26:55.874413 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 11:26:55.874431 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 11:26:55.875462 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 11:26:55.875506 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 3 - logger.go:44: 11:26:55.875540 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:55.875577 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 1 - logger.go:44: 11:26:55.875604 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 11:26:55.875630 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 11:26:55.881277 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 11:26:55.881354 [DEBUG] raft-test: server 0: transport: append to 2: newer term - logger.go:44: 11:26:55.881376 [ERR] raft: peer {Voter 2 2} has newer term, stopping replication - logger.go:44: 11:26:55.890059 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 11:26:55.890142 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 4 - logger.go:44: 11:26:55.890196 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:55.890212 [DEBUG] raft: Vote granted from 2 in term 4. Tally: 1 - logger.go:44: 11:26:55.890275 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 11:26:55.890309 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 11:26:55.903722 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 11:26:55.903788 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 3 - logger.go:44: 11:26:55.903843 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:55.903874 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 1 - logger.go:44: 11:26:55.904000 [DEBUG] raft: Newer term discovered, fallback to follower - logger.go:44: 11:26:55.904019 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:55.904051 [INFO] raft: Duplicate RequestVote for same term: 3 - logger.go:44: 11:26:55.904953 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 11:26:55.904983 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 4 - logger.go:44: 11:26:55.905010 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:55.905022 [DEBUG] raft: Vote granted from 1 in term 4. Tally: 1 - logger.go:44: 11:26:55.905059 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 11:26:55.905084 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 11:26:55.907407 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 11:26:55.907463 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 5 - logger.go:44: 11:26:55.907512 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:55.907533 [DEBUG] raft: Vote granted from 2 in term 5. Tally: 1 - logger.go:44: 11:26:55.907566 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 11:26:55.907608 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 11:26:55.923677 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 11:26:55.923741 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 5 - logger.go:44: 11:26:55.923782 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:55.923799 [DEBUG] raft: Vote granted from 0 in term 5. Tally: 1 - logger.go:44: 11:26:55.923854 [INFO] raft: Duplicate RequestVote for same term: 5 - logger.go:44: 11:26:55.923911 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:55.923934 [DEBUG] raft: Vote granted from 1 in term 5. Tally: 2 - logger.go:44: 11:26:55.923945 [INFO] raft: Election won. Tally: 2 - logger.go:44: 11:26:55.923958 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 11:26:55.924001 [INFO] raft: Added peer 1, starting replication - logger.go:44: 11:26:55.924023 [INFO] raft: Added peer 2, starting replication - logger.go:44: 11:26:55.924105 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=5,index=5] - logger.go:44: 11:26:55.924153 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "0") - logger.go:44: 11:26:55.924183 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 11:26:55.924207 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 11:26:55.924224 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 11:26:55.924254 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=5,index=5] - logger.go:44: 11:26:55.924310 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 11:26:55.924346 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 11:26:55.924356 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 11:26:55.924365 [DEBUG] raft-test: elect: done - logger.go:44: 11:26:55.924405 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=5,index=6] - logger.go:44: 11:26:55.924440 [DEBUG] raft-test: fsm 2: applied 2 - logger.go:44: 11:26:55.924535 [DEBUG] raft-test: fsm 0: applied 2 - logger.go:44: 11:26:55.924548 [DEBUG] raft-test: fsm 0: applied 3 - logger.go:44: 11:26:55.924582 [DEBUG] raft-test: close: start - logger.go:44: 11:26:55.924611 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=5,index=6] - logger.go:44: 11:26:55.924663 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 11:26:55.924680 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 11:26:55.924699 [DEBUG] raft-test: fsm 1: applied 2 - logger.go:44: 11:26:55.924716 [DEBUG] raft-test: fsm 1: applied 3 - logger.go:44: 11:26:55.924735 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 11:26:55.924759 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 11:26:55.924781 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 11:26:55.924803 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 11:26:55.924820 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 11:26:55.924830 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 11:26:55.924840 [DEBUG] raft-test: close: done - --- PASS: TestControl_DeposeAfterCommandAppendedThenElect/1 (0.07s) - logger.go:44: 11:26:55.925092 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 11:26:55.925164 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 11:26:55.925227 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 11:26:55.925249 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 11:26:55.925290 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 11:26:55.925362 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:55.925376 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 11:26:55.925413 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:55.925428 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 11:26:55.925458 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:55.925478 [DEBUG] raft-test: setup: done - logger.go:44: 11:26:55.925491 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 11:26:55.925508 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 11:26:55.925524 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 11:26:55.925546 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:55.925570 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:55.925591 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:55.947464 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 11:26:55.947532 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 2 - logger.go:44: 11:26:55.947586 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:55.947621 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 1 - logger.go:44: 11:26:55.947592 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 11:26:55.947651 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 11:26:55.947669 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 11:26:55.947773 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:55.947787 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 11:26:55.947828 [INFO] raft: Duplicate RequestVote for same term: 2 - logger.go:44: 11:26:55.947696 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 11:26:55.947897 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 2 - logger.go:44: 11:26:55.947906 [INFO] raft: Election won. Tally: 2 - logger.go:44: 11:26:55.947926 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 11:26:55.947943 [INFO] raft: Added peer 1, starting replication - logger.go:44: 11:26:55.947963 [INFO] raft: Added peer 2, starting replication - logger.go:44: 11:26:55.948033 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] - logger.go:44: 11:26:55.948099 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "0") - logger.go:44: 11:26:55.948128 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 11:26:55.948157 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 11:26:55.948175 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 11:26:55.948214 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] - logger.go:44: 11:26:55.948266 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 11:26:55.948317 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 11:26:55.948340 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 11:26:55.948352 [DEBUG] raft-test: elect: done - logger.go:44: 11:26:55.948397 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] - logger.go:44: 11:26:55.948466 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 11:26:55.948500 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 11:26:55.948511 [DEBUG] raft-test: server 0: pipeline: append to 2: append fault: command 2 - logger.go:44: 11:26:55.948558 [DEBUG] raft-test: server 0: pipeline: append to 1: 2 entries [Command:term=2,index=3 Command:term=2,index=4] - logger.go:44: 11:26:55.948570 [DEBUG] raft-test: server 0: pipeline: append to 1: append fault: command 2 - logger.go:44: 11:26:55.948598 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 11:26:55.948665 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 11:26:55.948677 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 11:26:55.948699 [DEBUG] raft-test: fsm 2: applied 1 - logger.go:44: 11:26:55.948722 [DEBUG] raft-test: fsm 1: applied 1 - logger.go:44: 11:26:55.951485 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 11:26:55.951566 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 11:26:55.951585 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 11:26:55.951605 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:55.951644 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:55.951658 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 11:26:55.951605 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 11:26:55.951615 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 11:26:55.951718 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected - logger.go:44: 11:26:55.951756 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 11:26:55.951780 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 11:26:55.951628 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:55.951812 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:55.951826 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 11:26:55.959445 [DEBUG] raft-test: server 0: transport: append to 1: 2 entries [Command:term=2,index=3 Command:term=2,index=4] - logger.go:44: 11:26:55.959511 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:55.959534 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 11:26:55.959602 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 11:26:55.959675 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:55.959700 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 11:26:55.967468 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:55.967540 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:55.967549 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:55.967603 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:55.967620 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 11:26:55.967557 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 11:26:55.971601 [WARN] raft: Failed to contact 1 in 23.324015ms - logger.go:44: 11:26:55.971643 [WARN] raft: Failed to contact 2 in 23.137869ms - logger.go:44: 11:26:55.971678 [WARN] raft: Failed to contact quorum of nodes, stepping down - logger.go:44: 11:26:55.971736 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:55.971759 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 11:26:55.971775 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 11:26:55.971810 [DEBUG] raft-test: elect: start (server 1) - logger.go:44: 11:26:55.971828 [DEBUG] raft-test: server 1: establish outbound connection to all other nodes - logger.go:44: 11:26:55.971855 [DEBUG] raft-test: elect: server 1: wait to become leader within 1.5s - logger.go:44: 11:26:55.973500 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 11:26:55.973544 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 - logger.go:44: 11:26:55.973569 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:55.973585 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 - logger.go:44: 11:26:55.973628 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 11:26:55.973659 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 11:26:55.975797 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 11:26:55.975852 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:55.975878 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 11:26:55.975928 [DEBUG] raft-test: server 0: transport: append to 1: 2 entries [Command:term=2,index=3 Command:term=2,index=4] - logger.go:44: 11:26:55.975959 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:55.975978 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 11:26:55.991486 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 11:26:55.991555 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 3 - logger.go:44: 11:26:55.991640 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:55.991658 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 1 - logger.go:44: 11:26:55.991724 [INFO] raft: Duplicate RequestVote for same term: 3 - logger.go:44: 11:26:55.991783 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 2 - logger.go:44: 11:26:55.991794 [INFO] raft: Election won. Tally: 2 - logger.go:44: 11:26:55.991807 [INFO] raft: Node at 1 [Leader] entering Leader state - logger.go:44: 11:26:55.991826 [INFO] raft: Added peer 0, starting replication - logger.go:44: 11:26:55.991847 [INFO] raft: Added peer 2, starting replication - logger.go:44: 11:26:55.991947 [DEBUG] raft-test: server 1: transport: append to 2: 1 entry [Noop:term=3,index=5] - logger.go:44: 11:26:55.992022 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "1") - logger.go:44: 11:26:55.992060 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 11:26:55.992094 [DEBUG] raft-test: server 1: leadership: acquired - logger.go:44: 11:26:55.992117 [DEBUG] raft-test: elect: server 1: wait for other servers to become followers within 40ms - logger.go:44: 11:26:55.992164 [DEBUG] raft-test: server 1: transport: append to 0: 1 entry [Noop:term=3,index=5] - logger.go:44: 11:26:55.992234 [INFO] raft: pipelining replication to peer {Voter 0 0} - logger.go:44: 11:26:55.992300 [DEBUG] raft-test: elect: server 0: became follower - logger.go:44: 11:26:55.992333 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 11:26:55.992350 [DEBUG] raft-test: elect: done - logger.go:44: 11:26:55.992380 [DEBUG] raft-test: server 1: pipeline: append to 2: 1 entry [Command:term=3,index=6] - logger.go:44: 11:26:55.992420 [DEBUG] raft-test: fsm 2: applied 2 - logger.go:44: 11:26:55.992474 [DEBUG] raft-test: fsm 1: applied 2 - logger.go:44: 11:26:55.992490 [DEBUG] raft-test: fsm 1: applied 3 - logger.go:44: 11:26:55.992532 [DEBUG] raft-test: close: start - logger.go:44: 11:26:55.992590 [DEBUG] raft-test: server 1: pipeline: append to 0: 1 entry [Command:term=3,index=6] - logger.go:44: 11:26:55.992638 [INFO] raft: aborting pipeline replication to peer {Voter 0 0} - logger.go:44: 11:26:55.992654 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 11:26:55.992673 [DEBUG] raft-test: fsm 0: applied 2 - logger.go:44: 11:26:55.992684 [DEBUG] raft-test: fsm 0: applied 3 - logger.go:44: 11:26:55.992712 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 11:26:55.992737 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 11:26:55.992760 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 11:26:55.992772 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 11:26:55.992782 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 11:26:55.992799 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 11:26:55.992807 [DEBUG] raft-test: close: done - --- PASS: TestControl_DeposeAfterCommandAppendedThenElect/2 (0.06s) - logger.go:44: 11:26:56.083637 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 11:26:56.083857 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 11:26:56.083981 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 11:26:56.084041 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 11:26:56.084126 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 11:26:56.084272 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:56.084305 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 11:26:56.084414 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:56.084445 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 11:26:56.084537 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:56.084569 [DEBUG] raft-test: setup: done - logger.go:44: 11:26:56.084599 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 11:26:56.084666 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 11:26:56.084720 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 11:26:56.084811 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.084903 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.095660 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.103623 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 11:26:56.103731 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 2 - logger.go:44: 11:26:56.103769 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:56.103785 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 1 - logger.go:44: 11:26:56.103822 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 11:26:56.103908 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 11:26:56.110707 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 11:26:56.110766 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 11:26:56.110807 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:56.110828 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 11:26:56.110910 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 2 - logger.go:44: 11:26:56.110921 [INFO] raft: Election won. Tally: 2 - logger.go:44: 11:26:56.110936 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 11:26:56.110958 [INFO] raft: Added peer 1, starting replication - logger.go:44: 11:26:56.110976 [INFO] raft: Added peer 2, starting replication - logger.go:44: 11:26:56.111047 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] - logger.go:44: 11:26:56.111118 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 11:26:56.111164 [INFO] raft: Duplicate RequestVote for same term: 2 - logger.go:44: 11:26:56.111197 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 11:26:56.111218 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 11:26:56.111258 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] - logger.go:44: 11:26:56.111304 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "0") - logger.go:44: 11:26:56.111352 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 11:26:56.111397 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 11:26:56.111411 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 11:26:56.111425 [DEBUG] raft-test: elect: done - logger.go:44: 11:26:56.111483 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] - logger.go:44: 11:26:56.111550 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 11:26:56.111598 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 11:26:56.111612 [DEBUG] raft-test: server 0: pipeline: append to 2: append fault: command 2 - logger.go:44: 11:26:56.111648 [DEBUG] raft-test: server 0: pipeline: append to 1: 2 entries [Command:term=2,index=3 Command:term=2,index=4] - logger.go:44: 11:26:56.111663 [DEBUG] raft-test: server 0: pipeline: append to 1: append fault: command 2 - logger.go:44: 11:26:56.111686 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 11:26:56.111772 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 11:26:56.111790 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 11:26:56.111815 [DEBUG] raft-test: fsm 2: applied 1 - logger.go:44: 11:26:56.111841 [DEBUG] raft-test: fsm 1: applied 1 - logger.go:44: 11:26:56.114037 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 11:26:56.114076 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected - logger.go:44: 11:26:56.114096 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 11:26:56.114115 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 11:26:56.117776 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 11:26:56.117837 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:56.117863 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 11:26:56.117924 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:56.117950 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:56.117964 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 11:26:56.117987 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 11:26:56.118006 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 11:26:56.118023 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 11:26:56.118045 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 11:26:56.118075 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:56.118111 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:56.118123 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 11:26:56.120100 [DEBUG] raft-test: server 0: transport: append to 1: 2 entries [Command:term=2,index=3 Command:term=2,index=4] - logger.go:44: 11:26:56.120168 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:56.120188 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 11:26:56.130395 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 11:26:56.130439 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:56.130462 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 11:26:56.131375 [WARN] raft: Failed to contact 1 in 20.01985ms - logger.go:44: 11:26:56.131411 [WARN] raft: Failed to contact 2 in 19.838421ms - logger.go:44: 11:26:56.131423 [WARN] raft: Failed to contact quorum of nodes, stepping down - logger.go:44: 11:26:56.131464 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.131487 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 11:26:56.131501 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 11:26:56.131554 [DEBUG] raft-test: elect: start (server 2) - logger.go:44: 11:26:56.131585 [DEBUG] raft-test: server 2: establish outbound connection to all other nodes - logger.go:44: 11:26:56.131603 [DEBUG] raft-test: elect: server 2: wait to become leader within 1.5s - logger.go:44: 11:26:56.134331 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:56.134390 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:56.134414 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 11:26:56.134346 [DEBUG] raft-test: server 0: transport: append to 1: 2 entries [Command:term=2,index=3 Command:term=2,index=4] - logger.go:44: 11:26:56.134465 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:56.134484 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 11:26:56.143644 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 11:26:56.143742 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 3 - logger.go:44: 11:26:56.143773 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:56.143788 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 1 - logger.go:44: 11:26:56.143859 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 11:26:56.143889 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 11:26:56.144065 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 11:26:56.144086 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 - logger.go:44: 11:26:56.144123 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:56.144136 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 - logger.go:44: 11:26:56.144176 [INFO] raft: Duplicate RequestVote for same term: 3 - logger.go:44: 11:26:56.144224 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 2 - logger.go:44: 11:26:56.144236 [INFO] raft: Election won. Tally: 2 - logger.go:44: 11:26:56.144248 [INFO] raft: Node at 2 [Leader] entering Leader state - logger.go:44: 11:26:56.144275 [INFO] raft: Added peer 0, starting replication - logger.go:44: 11:26:56.144300 [INFO] raft: Added peer 1, starting replication - logger.go:44: 11:26:56.144358 [DEBUG] raft-test: server 2: transport: append to 1: 1 entry [Noop:term=3,index=5] - logger.go:44: 11:26:56.144403 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "2") - logger.go:44: 11:26:56.144463 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 11:26:56.144500 [DEBUG] raft-test: server 2: leadership: acquired - logger.go:44: 11:26:56.144517 [DEBUG] raft-test: elect: server 2: wait for other servers to become followers within 40ms - logger.go:44: 11:26:56.144548 [DEBUG] raft-test: server 2: transport: append to 0: 1 entry [Noop:term=3,index=5] - logger.go:44: 11:26:56.144592 [INFO] raft: pipelining replication to peer {Voter 0 0} - logger.go:44: 11:26:56.144647 [DEBUG] raft-test: elect: server 0: became follower - logger.go:44: 11:26:56.144658 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 11:26:56.144669 [DEBUG] raft-test: elect: done - logger.go:44: 11:26:56.144700 [DEBUG] raft-test: server 2: pipeline: append to 1: 1 entry [Command:term=3,index=6] - logger.go:44: 11:26:56.144734 [DEBUG] raft-test: fsm 1: applied 2 - logger.go:44: 11:26:56.144795 [DEBUG] raft-test: fsm 2: applied 2 - logger.go:44: 11:26:56.144806 [DEBUG] raft-test: fsm 2: applied 3 - logger.go:44: 11:26:56.144831 [DEBUG] raft-test: close: start - logger.go:44: 11:26:56.144859 [DEBUG] raft-test: server 2: pipeline: append to 0: 1 entry [Command:term=3,index=6] - logger.go:44: 11:26:56.144909 [INFO] raft: aborting pipeline replication to peer {Voter 0 0} - logger.go:44: 11:26:56.144932 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 11:26:56.144966 [DEBUG] raft-test: fsm 0: applied 2 - logger.go:44: 11:26:56.144979 [DEBUG] raft-test: fsm 0: applied 3 - logger.go:44: 11:26:56.144998 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 11:26:56.145028 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 11:26:56.145048 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 11:26:56.145060 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 11:26:56.145079 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 11:26:56.145088 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 11:26:56.145105 [DEBUG] raft-test: close: done +--- PASS: TestControl_DeposeAfterCommandAppendedThenElect (0.37s) + --- PASS: TestControl_DeposeAfterCommandAppendedThenElect/0 (0.14s) + logger.go:44: 17:52:13.070375 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 17:52:13.070446 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 17:52:13.070512 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 17:52:13.070546 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 17:52:13.070571 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 17:52:13.070633 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:13.070650 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 17:52:13.070690 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:13.070704 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 17:52:13.070737 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:13.070749 [DEBUG] raft-test: setup: done + logger.go:44: 17:52:13.070763 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 17:52:13.070783 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 17:52:13.070801 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 17:52:13.070827 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.070851 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.070873 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.089816 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 17:52:13.089896 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 2 + logger.go:44: 17:52:13.089961 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.089978 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 1 + logger.go:44: 17:52:13.090021 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 17:52:13.093835 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.093956 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 17:52:13.093978 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 2 + logger.go:44: 17:52:13.094018 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.094031 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 1 + logger.go:44: 17:52:13.094056 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 17:52:13.094074 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 17:52:13.094087 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 17:52:13.094114 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.094146 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 17:52:13.094181 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 17:52:13.094212 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.094240 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 17:52:13.105567 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.105623 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 3 + logger.go:44: 17:52:13.105655 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.105680 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 1 + logger.go:44: 17:52:13.105723 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 17:52:13.105769 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.117807 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.117869 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 + logger.go:44: 17:52:13.117940 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.117957 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 + logger.go:44: 17:52:13.118002 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 17:52:13.117818 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.118035 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 3 + logger.go:44: 17:52:13.118053 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.118070 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 1 + logger.go:44: 17:52:13.118112 [INFO] raft: Duplicate RequestVote for same term: 3 + logger.go:44: 17:52:13.118147 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.118186 [INFO] raft: Duplicate RequestVote for same term: 3 + logger.go:44: 17:52:13.133825 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.133892 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 4 + logger.go:44: 17:52:13.133926 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.133950 [DEBUG] raft: Vote granted from 1 in term 4. Tally: 1 + logger.go:44: 17:52:13.133994 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 17:52:13.133827 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.134024 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 4 + logger.go:44: 17:52:13.134040 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.134053 [DEBUG] raft: Vote granted from 2 in term 4. Tally: 1 + logger.go:44: 17:52:13.134081 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 17:52:13.134109 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.134134 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.137283 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.137327 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 4 + logger.go:44: 17:52:13.137363 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.137379 [DEBUG] raft: Vote granted from 0 in term 4. Tally: 1 + logger.go:44: 17:52:13.137427 [INFO] raft: Duplicate RequestVote for same term: 4 + logger.go:44: 17:52:13.137448 [INFO] raft: Duplicate RequestVote for same term: 4 + logger.go:44: 17:52:13.153834 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.153895 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 5 + logger.go:44: 17:52:13.153933 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.153948 [DEBUG] raft: Vote granted from 1 in term 5. Tally: 1 + logger.go:44: 17:52:13.157255 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.157298 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 17:52:13.161800 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.161853 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 5 + logger.go:44: 17:52:13.161906 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.161920 [DEBUG] raft: Vote granted from 0 in term 5. Tally: 1 + logger.go:44: 17:52:13.161957 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.162014 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 5 + logger.go:44: 17:52:13.162045 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.162067 [INFO] raft: Duplicate RequestVote for same term: 5 + logger.go:44: 17:52:13.162084 [DEBUG] raft: Vote granted from 2 in term 5. Tally: 1 + logger.go:44: 17:52:13.161988 [INFO] raft: Duplicate RequestVote for same term: 5 + logger.go:44: 17:52:13.162149 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 17:52:13.162171 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.181810 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.181866 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 6 + logger.go:44: 17:52:13.181897 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.181910 [DEBUG] raft: Vote granted from 1 in term 6. Tally: 1 + logger.go:44: 17:52:13.182020 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.182082 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 17:52:13.182105 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.182121 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 6 + logger.go:44: 17:52:13.182149 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.182179 [DEBUG] raft: Vote granted from 0 in term 6. Tally: 1 + logger.go:44: 17:52:13.182224 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.182241 [DEBUG] raft: Vote granted from 2 in term 6. Tally: 2 + logger.go:44: 17:52:13.182266 [INFO] raft: Election won. Tally: 2 + logger.go:44: 17:52:13.182277 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 17:52:13.182297 [INFO] raft: Added peer 1, starting replication + logger.go:44: 17:52:13.182312 [INFO] raft: Added peer 2, starting replication + logger.go:44: 17:52:13.182379 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=6,index=2] + logger.go:44: 17:52:13.182440 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 17:52:13.182474 [INFO] raft: Duplicate RequestVote for same term: 6 + logger.go:44: 17:52:13.182498 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 17:52:13.182515 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 17:52:13.182545 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=6,index=2] + logger.go:44: 17:52:13.182582 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "0") + logger.go:44: 17:52:13.182605 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 17:52:13.182637 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 17:52:13.182648 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 17:52:13.182656 [DEBUG] raft-test: elect: done + logger.go:44: 17:52:13.182727 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=6,index=3] + logger.go:44: 17:52:13.182787 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 17:52:13.182818 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=6,index=4] + logger.go:44: 17:52:13.182829 [DEBUG] raft-test: server 0: pipeline: append to 2: append fault: command 2 + logger.go:44: 17:52:13.182882 [DEBUG] raft-test: server 0: pipeline: append to 1: 2 entries [Command:term=6,index=3 Command:term=6,index=4] + logger.go:44: 17:52:13.182895 [DEBUG] raft-test: server 0: pipeline: append to 1: append fault: command 2 + logger.go:44: 17:52:13.182911 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 17:52:13.182963 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 17:52:13.182974 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 17:52:13.182996 [DEBUG] raft-test: fsm 2: applied 1 + logger.go:44: 17:52:13.183016 [DEBUG] raft-test: fsm 1: applied 1 + logger.go:44: 17:52:13.184296 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 17:52:13.184311 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected + logger.go:44: 17:52:13.184324 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 17:52:13.184338 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 17:52:13.184405 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:13.184416 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:13.184425 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 17:52:13.184839 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 17:52:13.184851 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 17:52:13.184862 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 17:52:13.184874 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 17:52:13.185335 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:13.186111 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:13.186126 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 17:52:13.189892 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=6,index=4] + logger.go:44: 17:52:13.189943 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:13.189963 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 17:52:13.189992 [DEBUG] raft-test: server 0: transport: append to 1: 2 entries [Command:term=6,index=3 Command:term=6,index=4] + logger.go:44: 17:52:13.190015 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:13.190026 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 17:52:13.193822 [WARN] raft: Failed to contact 1 in 11.204823ms + logger.go:44: 17:52:13.193869 [WARN] raft: Failed to contact 2 in 11.039027ms + logger.go:44: 17:52:13.193881 [WARN] raft: Failed to contact quorum of nodes, stepping down + logger.go:44: 17:52:13.193954 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.193976 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 17:52:13.194075 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 17:52:13.194039 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 17:52:13.194114 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 17:52:13.194140 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 17:52:13.197219 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:13.200625 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 17:52:13.200696 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 7 + logger.go:44: 17:52:13.200722 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.200737 [DEBUG] raft: Vote granted from 1 in term 7. Tally: 1 + logger.go:44: 17:52:13.200778 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 17:52:13.200804 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.201853 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:13.201919 [DEBUG] raft-test: server 0: transport: append to 1: newer term + logger.go:44: 17:52:13.201945 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:13.206662 [DEBUG] raft-test: server 0: transport: append to 1: 2 entries [Command:term=6,index=3 Command:term=6,index=4] + logger.go:44: 17:52:13.206741 [DEBUG] raft-test: server 0: transport: append to 1: newer term + logger.go:44: 17:52:13.206788 [ERR] raft: peer {Voter 1 1} has newer term, stopping replication + logger.go:44: 17:52:13.206812 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=6,index=4] + logger.go:44: 17:52:13.206849 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:13.206869 [DEBUG] raft-test: server 0: transport: append to 1: newer term + logger.go:44: 17:52:13.206892 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:13.209842 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 17:52:13.209895 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 7 + logger.go:44: 17:52:13.209932 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.209956 [DEBUG] raft: Vote granted from 0 in term 7. Tally: 1 + logger.go:44: 17:52:13.210019 [DEBUG] raft: Vote granted from 2 in term 7. Tally: 2 + logger.go:44: 17:52:13.210030 [INFO] raft: Election won. Tally: 2 + logger.go:44: 17:52:13.210043 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 17:52:13.210062 [INFO] raft: Added peer 1, starting replication + logger.go:44: 17:52:13.210080 [INFO] raft: Added peer 2, starting replication + logger.go:44: 17:52:13.210142 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=7,index=5] + logger.go:44: 17:52:13.210206 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 17:52:13.210251 [INFO] raft: Duplicate RequestVote for same term: 7 + logger.go:44: 17:52:13.210272 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 17:52:13.210308 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 17:52:13.210344 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=7,index=5] + logger.go:44: 17:52:13.210384 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "0") + logger.go:44: 17:52:13.210407 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 17:52:13.210446 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 17:52:13.210457 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 17:52:13.210467 [DEBUG] raft-test: elect: done + logger.go:44: 17:52:13.210496 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=7,index=6] + logger.go:44: 17:52:13.210531 [DEBUG] raft-test: fsm 2: applied 2 + logger.go:44: 17:52:13.210564 [DEBUG] raft-test: fsm 0: applied 2 + logger.go:44: 17:52:13.210582 [DEBUG] raft-test: fsm 0: applied 3 + logger.go:44: 17:52:13.210609 [DEBUG] raft-test: close: start + logger.go:44: 17:52:13.210637 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=7,index=6] + logger.go:44: 17:52:13.210691 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 17:52:13.210710 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 17:52:13.210726 [DEBUG] raft-test: fsm 1: applied 2 + logger.go:44: 17:52:13.210739 [DEBUG] raft-test: fsm 1: applied 3 + logger.go:44: 17:52:13.210758 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 17:52:13.210784 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 17:52:13.210819 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 17:52:13.210834 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 17:52:13.210856 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 17:52:13.210875 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 17:52:13.210885 [DEBUG] raft-test: close: done + --- PASS: TestControl_DeposeAfterCommandAppendedThenElect/1 (0.18s) + logger.go:44: 17:52:13.211048 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 17:52:13.211128 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 17:52:13.211189 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 17:52:13.211211 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 17:52:13.211232 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 17:52:13.211286 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:13.211301 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 17:52:13.211339 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:13.211353 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 17:52:13.211387 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:13.211401 [DEBUG] raft-test: setup: done + logger.go:44: 17:52:13.211414 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 17:52:13.211434 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 17:52:13.211454 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 17:52:13.211477 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.211502 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.211525 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.233837 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 17:52:13.233911 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 2 + logger.go:44: 17:52:13.233975 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.233991 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 1 + logger.go:44: 17:52:13.234033 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 17:52:13.234065 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.234091 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 17:52:13.234105 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 2 + logger.go:44: 17:52:13.234136 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.234153 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 1 + logger.go:44: 17:52:13.234192 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 17:52:13.234221 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.243920 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 17:52:13.243992 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 17:52:13.244048 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.244065 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 17:52:13.244117 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 17:52:13.244164 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 17:52:13.253649 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.253722 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 3 + logger.go:44: 17:52:13.253777 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.253785 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.253842 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.253843 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 + logger.go:44: 17:52:13.253877 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.253892 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 + logger.go:44: 17:52:13.253916 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 17:52:13.253827 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 17:52:13.253793 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 1 + logger.go:44: 17:52:13.253990 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.273839 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.273909 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 4 + logger.go:44: 17:52:13.273950 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.273965 [DEBUG] raft: Vote granted from 2 in term 4. Tally: 1 + logger.go:44: 17:52:13.274007 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 17:52:13.274036 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.277824 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.277919 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 3 + logger.go:44: 17:52:13.277955 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.277971 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 1 + logger.go:44: 17:52:13.278032 [DEBUG] raft: Newer term discovered, fallback to follower + logger.go:44: 17:52:13.278050 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.278090 [INFO] raft: Duplicate RequestVote for same term: 3 + logger.go:44: 17:52:13.281402 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.281459 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 4 + logger.go:44: 17:52:13.281499 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.281514 [DEBUG] raft: Vote granted from 1 in term 4. Tally: 1 + logger.go:44: 17:52:13.281556 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 17:52:13.281585 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.301846 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.301909 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 17:52:13.301975 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 5 + logger.go:44: 17:52:13.302021 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.302036 [DEBUG] raft: Vote granted from 0 in term 5. Tally: 1 + logger.go:44: 17:52:13.301871 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.302087 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 5 + logger.go:44: 17:52:13.302104 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.302116 [DEBUG] raft: Vote granted from 2 in term 5. Tally: 1 + logger.go:44: 17:52:13.302132 [INFO] raft: Duplicate RequestVote for same term: 5 + logger.go:44: 17:52:13.301923 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 5 + logger.go:44: 17:52:13.302192 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.302206 [INFO] raft: Duplicate RequestVote for same term: 5 + logger.go:44: 17:52:13.302220 [DEBUG] raft: Vote granted from 1 in term 5. Tally: 1 + logger.go:44: 17:52:13.302258 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.302283 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 17:52:13.302306 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.302330 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 17:52:13.321865 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.321938 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 6 + logger.go:44: 17:52:13.321981 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.321996 [DEBUG] raft: Vote granted from 2 in term 6. Tally: 1 + logger.go:44: 17:52:13.322051 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 17:52:13.322093 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.329455 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.329547 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 6 + logger.go:44: 17:52:13.329587 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.329605 [DEBUG] raft: Vote granted from 0 in term 6. Tally: 1 + logger.go:44: 17:52:13.329664 [INFO] raft: Duplicate RequestVote for same term: 6 + logger.go:44: 17:52:13.329710 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.329725 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 6 + logger.go:44: 17:52:13.329785 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.329800 [DEBUG] raft: Vote granted from 1 in term 6. Tally: 1 + logger.go:44: 17:52:13.329816 [INFO] raft: Duplicate RequestVote for same term: 6 + logger.go:44: 17:52:13.329867 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.329895 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 17:52:13.349864 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.349934 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 7 + logger.go:44: 17:52:13.349968 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.349982 [DEBUG] raft: Vote granted from 0 in term 7. Tally: 1 + logger.go:44: 17:52:13.350047 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.350064 [DEBUG] raft: Vote granted from 2 in term 7. Tally: 2 + logger.go:44: 17:52:13.350072 [INFO] raft: Election won. Tally: 2 + logger.go:44: 17:52:13.350083 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 17:52:13.350098 [INFO] raft: Added peer 1, starting replication + logger.go:44: 17:52:13.350111 [INFO] raft: Added peer 2, starting replication + logger.go:44: 17:52:13.350157 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=7,index=2] + logger.go:44: 17:52:13.350216 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 17:52:13.350266 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.350292 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 17:52:13.350309 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 17:52:13.350341 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=7,index=2] + logger.go:44: 17:52:13.350395 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 17:52:13.350444 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 17:52:13.350456 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 17:52:13.350465 [DEBUG] raft-test: elect: done + logger.go:44: 17:52:13.350511 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=7,index=3] + logger.go:44: 17:52:13.350583 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 17:52:13.350620 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=7,index=4] + logger.go:44: 17:52:13.350631 [DEBUG] raft-test: server 0: pipeline: append to 2: append fault: command 2 + logger.go:44: 17:52:13.350673 [DEBUG] raft-test: server 0: pipeline: append to 1: 2 entries [Command:term=7,index=3 Command:term=7,index=4] + logger.go:44: 17:52:13.350706 [DEBUG] raft-test: server 0: pipeline: append to 1: append fault: command 2 + logger.go:44: 17:52:13.350727 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 17:52:13.350782 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 17:52:13.350795 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 17:52:13.350816 [DEBUG] raft-test: fsm 2: applied 1 + logger.go:44: 17:52:13.350839 [DEBUG] raft-test: fsm 1: applied 1 + logger.go:44: 17:52:13.353463 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:13.353517 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:13.353535 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 17:52:13.353565 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:13.353579 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:13.353592 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 17:52:13.353614 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 17:52:13.353629 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 17:52:13.353647 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 17:52:13.353666 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 17:52:13.353691 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 17:52:13.353706 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected + logger.go:44: 17:52:13.353732 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 17:52:13.353772 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 17:52:13.356527 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=7,index=4] + logger.go:44: 17:52:13.356569 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:13.356592 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 17:52:13.356617 [DEBUG] raft-test: server 0: transport: append to 1: 2 entries [Command:term=7,index=3 Command:term=7,index=4] + logger.go:44: 17:52:13.356633 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:13.356646 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 17:52:13.367098 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 17:52:13.367147 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 8 + logger.go:44: 17:52:13.367192 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.367208 [DEBUG] raft: Vote granted from 1 in term 8. Tally: 1 + logger.go:44: 17:52:13.367252 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 17:52:13.367283 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.373731 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 17:52:13.373817 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 8 + logger.go:44: 17:52:13.373841 [WARN] raft: Failed to contact 1 in 23.451302ms + logger.go:44: 17:52:13.373879 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 17:52:13.373888 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:13.373905 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:13.373906 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.373846 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.373962 [DEBUG] raft: Vote granted from 2 in term 8. Tally: 1 + logger.go:44: 17:52:13.373882 [WARN] raft: Failed to contact 2 in 23.274795ms + logger.go:44: 17:52:13.373990 [WARN] raft: Failed to contact quorum of nodes, stepping down + logger.go:44: 17:52:13.374016 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.374034 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 17:52:13.374047 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 17:52:13.373819 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:13.374074 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:13.374084 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 17:52:13.373917 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 17:52:13.373930 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=7,index=4] + logger.go:44: 17:52:13.374116 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:13.374139 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 17:52:13.374186 [DEBUG] raft-test: elect: start (server 1) + logger.go:44: 17:52:13.374204 [DEBUG] raft-test: server 1: establish outbound connection to all other nodes + logger.go:44: 17:52:13.374225 [DEBUG] raft-test: elect: server 1: wait to become leader within 1.5s + logger.go:44: 17:52:13.373944 [DEBUG] raft-test: server 0: transport: append to 1: 2 entries [Command:term=7,index=3 Command:term=7,index=4] + logger.go:44: 17:52:13.374253 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:13.374267 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 17:52:13.383846 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.383935 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 9 + logger.go:44: 17:52:13.383973 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.383988 [DEBUG] raft: Vote granted from 1 in term 9. Tally: 1 + logger.go:44: 17:52:13.384074 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.384092 [DEBUG] raft: Vote granted from 2 in term 9. Tally: 2 + logger.go:44: 17:52:13.384101 [INFO] raft: Election won. Tally: 2 + logger.go:44: 17:52:13.384112 [INFO] raft: Node at 1 [Leader] entering Leader state + logger.go:44: 17:52:13.384130 [INFO] raft: Added peer 0, starting replication + logger.go:44: 17:52:13.384145 [INFO] raft: Added peer 2, starting replication + logger.go:44: 17:52:13.384193 [DEBUG] raft-test: server 1: transport: append to 2: 1 entry [Noop:term=9,index=5] + logger.go:44: 17:52:13.384253 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 17:52:13.384307 [DEBUG] raft-test: server 1: leadership: acquired + logger.go:44: 17:52:13.384328 [DEBUG] raft-test: elect: server 1: wait for other servers to become followers within 40ms + logger.go:44: 17:52:13.384360 [DEBUG] raft-test: server 1: transport: append to 0: 1 entry [Noop:term=9,index=5] + logger.go:44: 17:52:13.384403 [INFO] raft: pipelining replication to peer {Voter 0 0} + logger.go:44: 17:52:13.384456 [DEBUG] raft-test: elect: server 0: became follower + logger.go:44: 17:52:13.384467 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 17:52:13.384475 [DEBUG] raft-test: elect: done + logger.go:44: 17:52:13.390494 [DEBUG] raft-test: server 1: pipeline: append to 2: 0 entries + logger.go:44: 17:52:13.390667 [DEBUG] raft-test: server 1: pipeline: append to 0: 0 entries + logger.go:44: 17:52:13.384550 [DEBUG] raft-test: fsm 1: applied 2 + logger.go:44: 17:52:13.391386 [DEBUG] raft-test: server 1: pipeline: append to 2: 1 entry [Command:term=9,index=6] + logger.go:44: 17:52:13.391394 [DEBUG] raft-test: server 1: pipeline: append to 0: 1 entry [Command:term=9,index=6] + logger.go:44: 17:52:13.391398 [DEBUG] raft-test: fsm 2: applied 2 + logger.go:44: 17:52:13.391402 [DEBUG] raft-test: server 1: transport: append to 2: 0 entries + logger.go:44: 17:52:13.391559 [DEBUG] raft-test: fsm 1: applied 3 + logger.go:44: 17:52:13.391592 [DEBUG] raft-test: close: start + logger.go:44: 17:52:13.391642 [INFO] raft: aborting pipeline replication to peer {Voter 0 0} + logger.go:44: 17:52:13.391662 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 17:52:13.391690 [DEBUG] raft-test: server 1: transport: append to 0: 0 entries + logger.go:44: 17:52:13.391734 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 17:52:13.391755 [DEBUG] raft-test: fsm 0: applied 2 + logger.go:44: 17:52:13.391776 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 17:52:13.391801 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 17:52:13.391845 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 17:52:13.391860 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 17:52:13.391871 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 17:52:13.391881 [DEBUG] raft-test: close: done + --- PASS: TestControl_DeposeAfterCommandAppendedThenElect/2 (0.05s) + logger.go:44: 17:52:13.392059 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 17:52:13.392130 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 17:52:13.392194 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 17:52:13.392218 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 17:52:13.392242 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 17:52:13.392309 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:13.392327 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 17:52:13.392371 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:13.392386 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 17:52:13.392423 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:13.392438 [DEBUG] raft-test: setup: done + logger.go:44: 17:52:13.392453 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 17:52:13.392475 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 17:52:13.392498 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 17:52:13.392551 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.392578 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.392605 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.408398 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 17:52:13.408461 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 2 + logger.go:44: 17:52:13.408512 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.408528 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 1 + logger.go:44: 17:52:13.408568 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 17:52:13.408597 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.410558 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 17:52:13.410585 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 17:52:13.410623 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.410638 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 17:52:13.410679 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 17:52:13.410733 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 2 + logger.go:44: 17:52:13.410743 [INFO] raft: Election won. Tally: 2 + logger.go:44: 17:52:13.410758 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 17:52:13.410777 [INFO] raft: Added peer 1, starting replication + logger.go:44: 17:52:13.410796 [INFO] raft: Added peer 2, starting replication + logger.go:44: 17:52:13.410844 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] + logger.go:44: 17:52:13.410884 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "0") + logger.go:44: 17:52:13.410909 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 17:52:13.410935 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 17:52:13.410951 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 17:52:13.410978 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] + logger.go:44: 17:52:13.411015 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 17:52:13.411044 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 17:52:13.411066 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 17:52:13.411077 [DEBUG] raft-test: elect: done + logger.go:44: 17:52:13.411116 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] + logger.go:44: 17:52:13.411163 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 17:52:13.411193 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=4] + logger.go:44: 17:52:13.411203 [DEBUG] raft-test: server 0: pipeline: append to 2: append fault: command 2 + logger.go:44: 17:52:13.411229 [DEBUG] raft-test: server 0: pipeline: append to 1: 2 entries [Command:term=2,index=3 Command:term=2,index=4] + logger.go:44: 17:52:13.411239 [DEBUG] raft-test: server 0: pipeline: append to 1: append fault: command 2 + logger.go:44: 17:52:13.411255 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 17:52:13.411303 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 17:52:13.411313 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 17:52:13.411331 [DEBUG] raft-test: fsm 2: applied 1 + logger.go:44: 17:52:13.411350 [DEBUG] raft-test: fsm 1: applied 1 + logger.go:44: 17:52:13.412563 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 17:52:13.412647 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected + logger.go:44: 17:52:13.412672 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 17:52:13.412697 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 17:52:13.412741 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 17:52:13.412756 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 17:52:13.412787 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 17:52:13.412804 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 17:52:13.413309 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:13.413332 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:13.413347 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 17:52:13.413720 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:13.413736 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:13.413763 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 17:52:13.418144 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=4] + logger.go:44: 17:52:13.418199 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:13.418217 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 17:52:13.418251 [DEBUG] raft-test: server 0: transport: append to 1: 2 entries [Command:term=2,index=3 Command:term=2,index=4] + logger.go:44: 17:52:13.418269 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:13.418281 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 17:52:13.421875 [WARN] raft: Failed to contact 2 in 10.712752ms + logger.go:44: 17:52:13.421911 [WARN] raft: Failed to contact 1 in 10.848302ms + logger.go:44: 17:52:13.421921 [WARN] raft: Failed to contact quorum of nodes, stepping down + logger.go:44: 17:52:13.421950 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.421964 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 17:52:13.421974 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 17:52:13.422003 [DEBUG] raft-test: elect: start (server 2) + logger.go:44: 17:52:13.422017 [DEBUG] raft-test: server 2: establish outbound connection to all other nodes + logger.go:44: 17:52:13.422033 [DEBUG] raft-test: elect: server 2: wait to become leader within 1.5s + logger.go:44: 17:52:13.434456 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:13.434525 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:13.434546 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 17:52:13.434599 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=4] + logger.go:44: 17:52:13.434615 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:13.434635 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 17:52:13.434682 [DEBUG] raft-test: server 0: transport: append to 1: 2 entries [Command:term=2,index=3 Command:term=2,index=4] + logger.go:44: 17:52:13.434704 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:13.434719 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 17:52:13.434742 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:13.434759 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:13.434771 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 17:52:13.439984 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 17:52:13.440035 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 + logger.go:44: 17:52:13.440060 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.440091 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 + logger.go:44: 17:52:13.440143 [WARN] raft: Rejecting vote request from 2 since we have a leader: 0 + logger.go:44: 17:52:13.440221 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 2 + logger.go:44: 17:52:13.440232 [INFO] raft: Election won. Tally: 2 + logger.go:44: 17:52:13.440249 [INFO] raft: Node at 2 [Leader] entering Leader state + logger.go:44: 17:52:13.440270 [INFO] raft: Added peer 0, starting replication + logger.go:44: 17:52:13.440286 [INFO] raft: Added peer 1, starting replication + logger.go:44: 17:52:13.440339 [DEBUG] raft-test: server 2: transport: append to 1: 1 entry [Noop:term=3,index=5] + logger.go:44: 17:52:13.440409 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 17:52:13.440447 [DEBUG] raft-test: server 2: leadership: acquired + logger.go:44: 17:52:13.440467 [DEBUG] raft-test: elect: server 2: wait for other servers to become followers within 40ms + logger.go:44: 17:52:13.440503 [DEBUG] raft-test: server 2: transport: append to 0: 1 entry [Noop:term=3,index=5] + logger.go:44: 17:52:13.440554 [INFO] raft: pipelining replication to peer {Voter 0 0} + logger.go:44: 17:52:13.440597 [DEBUG] raft-test: elect: server 0: became follower + logger.go:44: 17:52:13.440610 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 17:52:13.440621 [DEBUG] raft-test: elect: done + logger.go:44: 17:52:13.440654 [DEBUG] raft-test: server 2: pipeline: append to 1: 1 entry [Command:term=3,index=6] + logger.go:44: 17:52:13.440711 [DEBUG] raft-test: fsm 1: applied 2 + logger.go:44: 17:52:13.440777 [DEBUG] raft-test: fsm 2: applied 2 + logger.go:44: 17:52:13.440792 [DEBUG] raft-test: fsm 2: applied 3 + logger.go:44: 17:52:13.440821 [DEBUG] raft-test: close: start + logger.go:44: 17:52:13.440854 [DEBUG] raft-test: server 2: pipeline: append to 0: 1 entry [Command:term=3,index=6] + logger.go:44: 17:52:13.440910 [INFO] raft: aborting pipeline replication to peer {Voter 0 0} + logger.go:44: 17:52:13.440935 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 17:52:13.440953 [DEBUG] raft-test: fsm 0: applied 2 + logger.go:44: 17:52:13.440967 [DEBUG] raft-test: fsm 0: applied 3 + logger.go:44: 17:52:13.440989 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 17:52:13.441019 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 17:52:13.441045 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 17:52:13.441071 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 17:52:13.441092 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 17:52:13.441105 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 17:52:13.441115 [DEBUG] raft-test: close: done === RUN TestControl_DeposeAfterCommandCommitted ---- PASS: TestControl_DeposeAfterCommandCommitted (0.03s) - logger.go:44: 11:26:56.145289 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 11:26:56.145347 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 11:26:56.145411 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 11:26:56.145460 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 11:26:56.145488 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 11:26:56.145543 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:56.145561 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 11:26:56.145606 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:56.145620 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 11:26:56.145653 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:56.145666 [DEBUG] raft-test: setup: done - logger.go:44: 11:26:56.145678 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 11:26:56.145699 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 11:26:56.145724 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 11:26:56.145747 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.145771 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.145794 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.163977 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 11:26:56.164042 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 11:26:56.164106 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:56.164121 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 11:26:56.164192 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 2 - logger.go:44: 11:26:56.164204 [INFO] raft: Election won. Tally: 2 - logger.go:44: 11:26:56.164215 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 11:26:56.164233 [INFO] raft: Added peer 1, starting replication - logger.go:44: 11:26:56.164248 [INFO] raft: Added peer 2, starting replication - logger.go:44: 11:26:56.164318 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] - logger.go:44: 11:26:56.164391 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 11:26:56.164466 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 11:26:56.164491 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 11:26:56.164532 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] - logger.go:44: 11:26:56.164594 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 11:26:56.164645 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 11:26:56.164658 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 11:26:56.164667 [DEBUG] raft-test: elect: done - logger.go:44: 11:26:56.164721 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] - logger.go:44: 11:26:56.164785 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 11:26:56.164811 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 11:26:56.164830 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 11:26:56.164849 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] - logger.go:44: 11:26:56.164912 [DEBUG] raft-test: fsm 1: applied 1 - logger.go:44: 11:26:56.166375 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 11:26:56.166415 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected - logger.go:44: 11:26:56.166441 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 11:26:56.166462 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 11:26:56.166493 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 11:26:56.166508 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 11:26:56.166524 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 11:26:56.166545 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 11:26:56.167567 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:56.167594 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:56.167625 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 11:26:56.167655 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:56.167667 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:56.167677 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 11:26:56.167715 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:56.167727 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:56.167742 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 11:26:56.168364 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:56.168385 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:56.168406 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 11:26:56.175442 [WARN] raft: Failed to contact 1 in 10.489134ms - logger.go:44: 11:26:56.175503 [WARN] raft: Failed to contact 2 in 10.6212ms - logger.go:44: 11:26:56.175515 [WARN] raft: Failed to contact quorum of nodes, stepping down - logger.go:44: 11:26:56.175567 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.175591 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 11:26:56.175607 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 11:26:56.175677 [DEBUG] raft-test: close: start - logger.go:44: 11:26:56.175732 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:56.175751 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:56.175768 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 11:26:56.175841 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:56.175857 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:56.175870 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 11:26:56.175904 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 11:26:56.175935 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 11:26:56.175962 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 11:26:56.175975 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 11:26:56.176004 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 11:26:56.176023 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 11:26:56.176033 [DEBUG] raft-test: close: done +--- PASS: TestControl_DeposeAfterCommandCommitted (0.04s) + logger.go:44: 17:52:13.441262 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 17:52:13.441328 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 17:52:13.441394 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 17:52:13.441427 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 17:52:13.441454 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 17:52:13.441536 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:13.441554 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 17:52:13.441601 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:13.441619 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 17:52:13.441659 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:13.441672 [DEBUG] raft-test: setup: done + logger.go:44: 17:52:13.441687 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 17:52:13.441707 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 17:52:13.441732 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 17:52:13.441798 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.441823 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.441849 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.468212 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 17:52:13.468293 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 2 + logger.go:44: 17:52:13.468347 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.468364 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 1 + logger.go:44: 17:52:13.468401 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 17:52:13.468433 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.468461 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 17:52:13.468478 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 17:52:13.468509 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.468520 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 17:52:13.468567 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 2 + logger.go:44: 17:52:13.468577 [INFO] raft: Election won. Tally: 2 + logger.go:44: 17:52:13.468589 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 17:52:13.468607 [INFO] raft: Added peer 1, starting replication + logger.go:44: 17:52:13.468624 [INFO] raft: Added peer 2, starting replication + logger.go:44: 17:52:13.468682 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] + logger.go:44: 17:52:13.468735 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 17:52:13.468768 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 17:52:13.468792 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 17:52:13.468808 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 17:52:13.468842 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] + logger.go:44: 17:52:13.468880 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "0") + logger.go:44: 17:52:13.468904 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 17:52:13.468939 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 17:52:13.468951 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 17:52:13.468963 [DEBUG] raft-test: elect: done + logger.go:44: 17:52:13.469015 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] + logger.go:44: 17:52:13.469073 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 17:52:13.469096 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 17:52:13.469109 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 17:52:13.469127 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] + logger.go:44: 17:52:13.469175 [DEBUG] raft-test: fsm 1: applied 1 + logger.go:44: 17:52:13.472595 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 17:52:13.472638 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 17:52:13.472660 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 17:52:13.472681 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 17:52:13.472728 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:13.472741 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:13.472757 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 17:52:13.472780 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 17:52:13.472793 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected + logger.go:44: 17:52:13.472806 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 17:52:13.472831 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 17:52:13.472857 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:13.472871 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:13.472882 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 17:52:13.473881 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:13.473917 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:13.473945 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 17:52:13.476186 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:13.476221 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:13.476242 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 17:52:13.482272 [WARN] raft: Failed to contact 1 in 13.089509ms + logger.go:44: 17:52:13.482336 [WARN] raft: Failed to contact 2 in 13.201602ms + logger.go:44: 17:52:13.482349 [WARN] raft: Failed to contact quorum of nodes, stepping down + logger.go:44: 17:52:13.482386 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.482409 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 17:52:13.482426 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 17:52:13.482499 [DEBUG] raft-test: close: start + logger.go:44: 17:52:13.482548 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:13.482566 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:13.482596 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 17:52:13.482634 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:13.482653 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:13.482668 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 17:52:13.482695 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 17:52:13.482729 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 17:52:13.482758 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 17:52:13.482772 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 17:52:13.482805 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 17:52:13.482825 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 17:52:13.482836 [DEBUG] raft-test: close: done === RUN TestControl_DeposeAfterCommandCommittedThenElect === RUN TestControl_DeposeAfterCommandCommittedThenElect/0 === RUN TestControl_DeposeAfterCommandCommittedThenElect/1 === RUN TestControl_DeposeAfterCommandCommittedThenElect/2 ---- PASS: TestControl_DeposeAfterCommandCommittedThenElect (0.25s) - --- PASS: TestControl_DeposeAfterCommandCommittedThenElect/0 (0.10s) - logger.go:44: 11:26:56.176162 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 11:26:56.176221 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 11:26:56.176288 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 11:26:56.176322 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 11:26:56.176345 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 11:26:56.176412 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:56.176426 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 11:26:56.176465 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:56.176478 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 11:26:56.176511 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:56.176527 [DEBUG] raft-test: setup: done - logger.go:44: 11:26:56.176540 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 11:26:56.176571 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 11:26:56.176601 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 11:26:56.176625 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.176651 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.176677 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.197710 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 11:26:56.197796 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 11:26:56.197850 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:56.197865 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 11:26:56.197931 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 2 - logger.go:44: 11:26:56.197942 [INFO] raft: Election won. Tally: 2 - logger.go:44: 11:26:56.197958 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 11:26:56.198002 [INFO] raft: Added peer 1, starting replication - logger.go:44: 11:26:56.198027 [INFO] raft: Added peer 2, starting replication - logger.go:44: 11:26:56.198082 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] - logger.go:44: 11:26:56.198145 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 11:26:56.198209 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 11:26:56.198232 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 11:26:56.198268 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] - logger.go:44: 11:26:56.198320 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 11:26:56.198368 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 11:26:56.198381 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 11:26:56.198389 [DEBUG] raft-test: elect: done - logger.go:44: 11:26:56.198443 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] - logger.go:44: 11:26:56.198516 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 11:26:56.198548 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 11:26:56.198563 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 11:26:56.198581 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] - logger.go:44: 11:26:56.198648 [DEBUG] raft-test: fsm 1: applied 1 - logger.go:44: 11:26:56.200854 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:56.200894 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:56.200929 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 11:26:56.200959 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 11:26:56.200975 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 11:26:56.200997 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 11:26:56.201018 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 11:26:56.201061 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 11:26:56.201078 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected - logger.go:44: 11:26:56.201093 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 11:26:56.201109 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 11:26:56.201139 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:56.201153 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:56.201166 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 11:26:56.203450 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:56.203662 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:56.203687 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 11:26:56.206854 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:56.206922 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:56.206955 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 11:26:56.213618 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:56.213676 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:56.213693 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 11:26:56.217655 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:56.217709 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:56.217735 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 11:26:56.219675 [WARN] raft: Failed to contact 1 in 20.971333ms - logger.go:44: 11:26:56.219731 [WARN] raft: Failed to contact 2 in 21.111501ms - logger.go:44: 11:26:56.219742 [WARN] raft: Failed to contact quorum of nodes, stepping down - logger.go:44: 11:26:56.220007 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.220032 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 11:26:56.219846 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 11:26:56.220164 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 - logger.go:44: 11:26:56.220228 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:56.220246 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 - logger.go:44: 11:26:56.220304 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 11:26:56.220366 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 11:26:56.220105 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 11:26:56.220406 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 11:26:56.220426 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 11:26:56.220453 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 11:26:56.220479 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:56.220508 [DEBUG] raft-test: server 0: transport: append to 2: newer term - logger.go:44: 11:26:56.224177 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:56.224245 [DEBUG] raft-test: server 0: transport: append to 2: newer term - logger.go:44: 11:26:56.224269 [ERR] raft: peer {Voter 2 2} has newer term, stopping replication - logger.go:44: 11:26:56.224291 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:56.224311 [DEBUG] raft-test: server 0: transport: append to 2: newer term - logger.go:44: 11:26:56.227442 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 11:26:56.227501 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 3 - logger.go:44: 11:26:56.227584 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:56.227598 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 1 - logger.go:44: 11:26:56.227636 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 11:26:56.227691 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 11:26:56.245279 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 11:26:56.245359 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 4 - logger.go:44: 11:26:56.245443 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:56.245461 [DEBUG] raft: Vote granted from 2 in term 4. Tally: 1 - logger.go:44: 11:26:56.245536 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 11:26:56.245571 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 11:26:56.249954 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 11:26:56.250026 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 3 - logger.go:44: 11:26:56.250054 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:56.250082 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 1 - logger.go:44: 11:26:56.250146 [DEBUG] raft: Newer term discovered, fallback to follower - logger.go:44: 11:26:56.250161 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.250194 [INFO] raft: Duplicate RequestVote for same term: 3 - logger.go:44: 11:26:56.257315 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 11:26:56.257378 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 4 - logger.go:44: 11:26:56.257417 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:56.257432 [DEBUG] raft: Vote granted from 1 in term 4. Tally: 1 - logger.go:44: 11:26:56.257475 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 11:26:56.257504 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 11:26:56.266330 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 11:26:56.266402 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 5 - logger.go:44: 11:26:56.266443 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:56.266486 [DEBUG] raft: Vote granted from 2 in term 5. Tally: 1 - logger.go:44: 11:26:56.266543 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 11:26:56.266579 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 11:26:56.275470 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 11:26:56.275796 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 5 - logger.go:44: 11:26:56.275838 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:56.275857 [DEBUG] raft: Vote granted from 0 in term 5. Tally: 1 - logger.go:44: 11:26:56.275993 [INFO] raft: Duplicate RequestVote for same term: 5 - logger.go:44: 11:26:56.276145 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.276223 [DEBUG] raft: Vote granted from 1 in term 5. Tally: 2 - logger.go:44: 11:26:56.276242 [INFO] raft: Election won. Tally: 2 - logger.go:44: 11:26:56.276261 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 11:26:56.276290 [INFO] raft: Added peer 1, starting replication - logger.go:44: 11:26:56.276308 [INFO] raft: Added peer 2, starting replication - logger.go:44: 11:26:56.276365 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=5,index=4] - logger.go:44: 11:26:56.276424 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "0") - logger.go:44: 11:26:56.276461 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 11:26:56.276486 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 11:26:56.276506 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 11:26:56.276549 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=5,index=4] - logger.go:44: 11:26:56.276612 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 11:26:56.276653 [DEBUG] raft-test: fsm 2: applied 1 - logger.go:44: 11:26:56.276682 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 11:26:56.276693 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 11:26:56.276703 [DEBUG] raft-test: elect: done - logger.go:44: 11:26:56.276735 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=5,index=5] - logger.go:44: 11:26:56.276784 [DEBUG] raft-test: fsm 0: applied 2 - logger.go:44: 11:26:56.276814 [DEBUG] raft-test: close: start - logger.go:44: 11:26:56.276843 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=5,index=5] - logger.go:44: 11:26:56.276901 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 11:26:56.276921 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 11:26:56.276938 [DEBUG] raft-test: fsm 2: applied 2 - logger.go:44: 11:26:56.276964 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 11:26:56.277002 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 11:26:56.277036 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 11:26:56.277062 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 11:26:56.277082 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 11:26:56.277094 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 11:26:56.277103 [DEBUG] raft-test: close: done - --- PASS: TestControl_DeposeAfterCommandCommittedThenElect/1 (0.06s) - logger.go:44: 11:26:56.277341 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 11:26:56.277405 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 11:26:56.277473 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 11:26:56.277499 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 11:26:56.277531 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 11:26:56.277601 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:56.277619 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 11:26:56.277677 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:56.277692 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 11:26:56.277730 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:56.277745 [DEBUG] raft-test: setup: done - logger.go:44: 11:26:56.277760 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 11:26:56.277783 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 11:26:56.277802 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 11:26:56.277827 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.277855 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.277881 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.295483 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 11:26:56.295566 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 2 - logger.go:44: 11:26:56.295623 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:56.295638 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 1 - logger.go:44: 11:26:56.295678 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 11:26:56.295706 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 11:26:56.299762 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 11:26:56.300113 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 11:26:56.300216 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:56.300274 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 11:26:56.300383 [INFO] raft: Duplicate RequestVote for same term: 2 - logger.go:44: 11:26:56.300479 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 2 - logger.go:44: 11:26:56.300489 [INFO] raft: Election won. Tally: 2 - logger.go:44: 11:26:56.300501 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 11:26:56.300567 [INFO] raft: Added peer 1, starting replication - logger.go:44: 11:26:56.300605 [INFO] raft: Added peer 2, starting replication - logger.go:44: 11:26:56.300661 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] - logger.go:44: 11:26:56.300746 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "0") - logger.go:44: 11:26:56.300779 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 11:26:56.300855 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 11:26:56.300873 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 11:26:56.300914 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] - logger.go:44: 11:26:56.300959 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 11:26:56.300995 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 11:26:56.301051 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 11:26:56.301078 [DEBUG] raft-test: elect: done - logger.go:44: 11:26:56.301171 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] - logger.go:44: 11:26:56.301227 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 11:26:56.301250 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 11:26:56.301262 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 11:26:56.301278 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] - logger.go:44: 11:26:56.301339 [DEBUG] raft-test: fsm 1: applied 1 - logger.go:44: 11:26:56.303574 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:56.303606 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:56.303619 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 11:26:56.303656 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 11:26:56.303669 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 11:26:56.303684 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 11:26:56.303864 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 11:26:56.303897 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:56.303918 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:56.303930 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 11:26:56.303948 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 11:26:56.303961 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected - logger.go:44: 11:26:56.303974 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 11:26:56.303989 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 11:26:56.305732 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:56.305774 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:56.305792 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 11:26:56.307176 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:56.307219 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:56.307239 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 11:26:56.316620 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:56.316742 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:56.316768 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 11:26:56.318285 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:56.318333 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:56.318354 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 11:26:56.318741 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:56.318760 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:56.318784 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 11:26:56.321086 [WARN] raft: Failed to contact 1 in 19.759366ms - logger.go:44: 11:26:56.321125 [WARN] raft: Failed to contact 2 in 19.866432ms - logger.go:44: 11:26:56.321159 [WARN] raft: Failed to contact quorum of nodes, stepping down - logger.go:44: 11:26:56.321189 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.321210 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 11:26:56.321225 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 11:26:56.321285 [DEBUG] raft-test: elect: start (server 1) - logger.go:44: 11:26:56.321303 [DEBUG] raft-test: server 1: establish outbound connection to all other nodes - logger.go:44: 11:26:56.321322 [DEBUG] raft-test: elect: server 1: wait to become leader within 1.5s - logger.go:44: 11:26:56.321607 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:56.321659 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:56.321681 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 11:26:56.328110 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 11:26:56.328191 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 - logger.go:44: 11:26:56.328221 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:56.328249 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 - logger.go:44: 11:26:56.328295 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 11:26:56.328326 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 11:26:56.333125 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 11:26:56.333427 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 3 - logger.go:44: 11:26:56.333477 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:56.333503 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 1 - logger.go:44: 11:26:56.333643 [INFO] raft: Duplicate RequestVote for same term: 3 - logger.go:44: 11:26:56.333695 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 2 - logger.go:44: 11:26:56.333704 [INFO] raft: Election won. Tally: 2 - logger.go:44: 11:26:56.333716 [INFO] raft: Node at 1 [Leader] entering Leader state - logger.go:44: 11:26:56.333778 [INFO] raft: Added peer 0, starting replication - logger.go:44: 11:26:56.333801 [INFO] raft: Added peer 2, starting replication - logger.go:44: 11:26:56.333873 [DEBUG] raft-test: server 1: transport: append to 2: 1 entry [Noop:term=3,index=4] - logger.go:44: 11:26:56.333936 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "1") - logger.go:44: 11:26:56.333971 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 11:26:56.333999 [DEBUG] raft-test: server 1: leadership: acquired - logger.go:44: 11:26:56.334016 [DEBUG] raft-test: elect: server 1: wait for other servers to become followers within 40ms - logger.go:44: 11:26:56.334075 [DEBUG] raft-test: server 1: transport: append to 0: 1 entry [Noop:term=3,index=4] - logger.go:44: 11:26:56.334182 [INFO] raft: pipelining replication to peer {Voter 0 0} - logger.go:44: 11:26:56.334227 [DEBUG] raft-test: fsm 2: applied 1 - logger.go:44: 11:26:56.334266 [DEBUG] raft-test: elect: server 0: became follower - logger.go:44: 11:26:56.334276 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 11:26:56.334292 [DEBUG] raft-test: elect: done - logger.go:44: 11:26:56.334320 [DEBUG] raft-test: server 1: pipeline: append to 2: 1 entry [Command:term=3,index=5] - logger.go:44: 11:26:56.334366 [DEBUG] raft-test: fsm 1: applied 2 - logger.go:44: 11:26:56.334393 [DEBUG] raft-test: close: start - logger.go:44: 11:26:56.334484 [DEBUG] raft-test: server 1: pipeline: append to 0: 1 entry [Command:term=3,index=5] - logger.go:44: 11:26:56.334591 [INFO] raft: aborting pipeline replication to peer {Voter 0 0} - logger.go:44: 11:26:56.334610 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 11:26:56.334626 [DEBUG] raft-test: fsm 0: applied 2 - logger.go:44: 11:26:56.334645 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 11:26:56.334676 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:56.334688 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:56.334701 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 11:26:56.334728 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 11:26:56.334751 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 11:26:56.334863 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 11:26:56.334879 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 11:26:56.334890 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 11:26:56.334901 [DEBUG] raft-test: close: done - --- PASS: TestControl_DeposeAfterCommandCommittedThenElect/2 (0.09s) - logger.go:44: 11:26:56.335123 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 11:26:56.335194 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 11:26:56.335256 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 11:26:56.335282 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 11:26:56.335303 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 11:26:56.335439 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:56.335547 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 11:26:56.335609 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:56.335627 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 11:26:56.335764 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:56.335779 [DEBUG] raft-test: setup: done - logger.go:44: 11:26:56.335854 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 11:26:56.335956 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 11:26:56.335985 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 11:26:56.336014 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.336200 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.336226 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.366155 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 11:26:56.366214 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 2 - logger.go:44: 11:26:56.366274 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:56.366290 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 1 - logger.go:44: 11:26:56.366341 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 11:26:56.366374 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 11:26:56.366397 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 11:26:56.366410 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 11:26:56.366432 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:56.366444 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 11:26:56.366495 [INFO] raft: Duplicate RequestVote for same term: 2 - logger.go:44: 11:26:56.366525 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 11:26:56.366539 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 2 - logger.go:44: 11:26:56.366568 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:56.366581 [INFO] raft: Duplicate RequestVote for same term: 2 - logger.go:44: 11:26:56.366597 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 1 - logger.go:44: 11:26:56.366626 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 11:26:56.366652 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 11:26:56.381802 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 11:26:56.381870 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 3 - logger.go:44: 11:26:56.381931 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:56.381950 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 1 - logger.go:44: 11:26:56.381999 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 11:26:56.382047 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 11:26:56.391787 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 11:26:56.392046 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 3 - logger.go:44: 11:26:56.392087 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:56.392102 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 1 - logger.go:44: 11:26:56.392174 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.392193 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 2 - logger.go:44: 11:26:56.392201 [INFO] raft: Election won. Tally: 2 - logger.go:44: 11:26:56.392213 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 11:26:56.392252 [INFO] raft: Added peer 1, starting replication - logger.go:44: 11:26:56.392268 [INFO] raft: Added peer 2, starting replication - logger.go:44: 11:26:56.392371 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=3,index=2] - logger.go:44: 11:26:56.392466 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 11:26:56.392532 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 11:26:56.392541 [INFO] raft: Duplicate RequestVote for same term: 3 - logger.go:44: 11:26:56.392468 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 11:26:56.392502 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=3,index=2] - logger.go:44: 11:26:56.392755 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "0") - logger.go:44: 11:26:56.392800 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 11:26:56.393790 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 11:26:56.393817 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 11:26:56.393837 [DEBUG] raft-test: elect: done - logger.go:44: 11:26:56.393924 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=3,index=3] - logger.go:44: 11:26:56.393966 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=3,index=3] - logger.go:44: 11:26:56.394040 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 11:26:56.394063 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 11:26:56.394080 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 11:26:56.398156 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 11:26:56.398201 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 11:26:56.398225 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 11:26:56.398302 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 11:26:56.398350 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:56.398365 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:56.398377 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 11:26:56.398402 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:56.398415 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:56.398426 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 11:26:56.398445 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 11:26:56.398458 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected - logger.go:44: 11:26:56.398471 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 11:26:56.398486 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 11:26:56.400033 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:56.400154 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:56.400177 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 11:26:56.401414 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:56.401438 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:56.401454 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 11:26:56.411450 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 11:26:56.411514 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 4 - logger.go:44: 11:26:56.411548 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:56.411564 [DEBUG] raft: Vote granted from 1 in term 4. Tally: 1 - logger.go:44: 11:26:56.411606 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 11:26:56.411647 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 11:26:56.415375 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:56.415411 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:56.415427 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 11:26:56.415457 [WARN] raft: Failed to contact 2 in 21.4425ms - logger.go:44: 11:26:56.415503 [WARN] raft: Failed to contact 1 in 21.339986ms - logger.go:44: 11:26:56.415516 [WARN] raft: Failed to contact quorum of nodes, stepping down - logger.go:44: 11:26:56.415545 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.415565 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 11:26:56.415583 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 11:26:56.415639 [DEBUG] raft-test: elect: start (server 2) - logger.go:44: 11:26:56.415659 [DEBUG] raft-test: server 2: establish outbound connection to all other nodes - logger.go:44: 11:26:56.415679 [DEBUG] raft-test: elect: server 2: wait to become leader within 1.5s - logger.go:44: 11:26:56.415704 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:56.415718 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:56.415731 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 11:26:56.415766 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:56.415780 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 11:26:56.415805 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 11:26:56.417625 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:56.417700 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:56.417729 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 11:26:56.420096 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 11:26:56.420203 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 4 - logger.go:44: 11:26:56.420228 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:56.420241 [DEBUG] raft: Vote granted from 2 in term 4. Tally: 1 - logger.go:44: 11:26:56.420349 [INFO] raft: Duplicate RequestVote for same term: 4 - logger.go:44: 11:26:56.420455 [DEBUG] raft: Vote granted from 0 in term 4. Tally: 2 - logger.go:44: 11:26:56.420465 [INFO] raft: Election won. Tally: 2 - logger.go:44: 11:26:56.420477 [INFO] raft: Node at 2 [Leader] entering Leader state - logger.go:44: 11:26:56.420503 [INFO] raft: Added peer 0, starting replication - logger.go:44: 11:26:56.420516 [INFO] raft: Added peer 1, starting replication - logger.go:44: 11:26:56.420674 [DEBUG] raft-test: server 2: transport: append to 1: 1 entry [Noop:term=4,index=4] - logger.go:44: 11:26:56.420775 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "2") - logger.go:44: 11:26:56.420809 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 11:26:56.420866 [DEBUG] raft-test: server 2: leadership: acquired - logger.go:44: 11:26:56.420884 [DEBUG] raft-test: elect: server 2: wait for other servers to become followers within 40ms - logger.go:44: 11:26:56.420996 [DEBUG] raft-test: server 2: transport: append to 0: 1 entry [Noop:term=4,index=4] - logger.go:44: 11:26:56.421048 [INFO] raft: pipelining replication to peer {Voter 0 0} - logger.go:44: 11:26:56.421101 [DEBUG] raft-test: elect: server 0: became follower - logger.go:44: 11:26:56.421113 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 11:26:56.421132 [DEBUG] raft-test: elect: done - logger.go:44: 11:26:56.421164 [DEBUG] raft-test: server 2: pipeline: append to 1: 1 entry [Command:term=4,index=5] - logger.go:44: 11:26:56.421210 [DEBUG] raft-test: fsm 1: applied 1 - logger.go:44: 11:26:56.421281 [DEBUG] raft-test: fsm 2: applied 1 - logger.go:44: 11:26:56.421294 [DEBUG] raft-test: fsm 2: applied 2 - logger.go:44: 11:26:56.421322 [DEBUG] raft-test: close: start - logger.go:44: 11:26:56.421367 [DEBUG] raft-test: server 2: pipeline: append to 0: 1 entry [Command:term=4,index=5] - logger.go:44: 11:26:56.421418 [INFO] raft: aborting pipeline replication to peer {Voter 0 0} - logger.go:44: 11:26:56.421436 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 11:26:56.421460 [DEBUG] raft-test: fsm 0: applied 2 - logger.go:44: 11:26:56.421480 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 11:26:56.421544 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 11:26:56.421571 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 11:26:56.421588 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 11:26:56.421609 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 11:26:56.421621 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 11:26:56.421632 [DEBUG] raft-test: close: done +--- PASS: TestControl_DeposeAfterCommandCommittedThenElect (0.24s) + --- PASS: TestControl_DeposeAfterCommandCommittedThenElect/0 (0.14s) + logger.go:44: 17:52:13.482960 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 17:52:13.483022 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 17:52:13.483090 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 17:52:13.483117 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 17:52:13.483144 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 17:52:13.483201 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:13.483222 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 17:52:13.483263 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:13.483278 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 17:52:13.483317 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:13.483334 [DEBUG] raft-test: setup: done + logger.go:44: 17:52:13.483348 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 17:52:13.483369 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 17:52:13.483388 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 17:52:13.483411 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.483437 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.483465 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.501841 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 17:52:13.501926 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 17:52:13.501979 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.501995 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 17:52:13.502054 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 2 + logger.go:44: 17:52:13.502069 [INFO] raft: Election won. Tally: 2 + logger.go:44: 17:52:13.502082 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 17:52:13.502109 [INFO] raft: Added peer 1, starting replication + logger.go:44: 17:52:13.502122 [INFO] raft: Added peer 2, starting replication + logger.go:44: 17:52:13.502175 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] + logger.go:44: 17:52:13.502232 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 17:52:13.502290 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 17:52:13.502310 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 17:52:13.502342 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] + logger.go:44: 17:52:13.502387 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 17:52:13.502427 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 17:52:13.502438 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 17:52:13.502447 [DEBUG] raft-test: elect: done + logger.go:44: 17:52:13.502496 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] + logger.go:44: 17:52:13.502553 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 17:52:13.502576 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 17:52:13.502589 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 17:52:13.502605 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] + logger.go:44: 17:52:13.502656 [DEBUG] raft-test: fsm 1: applied 1 + logger.go:44: 17:52:13.508384 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 17:52:13.508423 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected + logger.go:44: 17:52:13.508457 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 17:52:13.508479 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 17:52:13.508522 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:13.508535 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:13.508546 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 17:52:13.508568 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 17:52:13.508579 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 17:52:13.508592 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 17:52:13.508605 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 17:52:13.508639 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:13.508651 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:13.508662 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 17:52:13.512617 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:13.512678 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:13.512701 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 17:52:13.513831 [WARN] raft: Failed to contact 1 in 11.181288ms + logger.go:44: 17:52:13.513871 [WARN] raft: Failed to contact 2 in 11.292804ms + logger.go:44: 17:52:13.513882 [WARN] raft: Failed to contact quorum of nodes, stepping down + logger.go:44: 17:52:13.513918 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.513935 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 17:52:13.513948 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 17:52:13.514000 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 17:52:13.514018 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 17:52:13.514040 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 17:52:13.513848 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:13.514126 [DEBUG] raft-test: fsm 2: applied 1 + logger.go:44: 17:52:13.533319 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 17:52:13.533392 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 + logger.go:44: 17:52:13.533450 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.533467 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 + logger.go:44: 17:52:13.533510 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 17:52:13.533545 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.533572 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 17:52:13.533589 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 3 + logger.go:44: 17:52:13.533630 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.533649 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 1 + logger.go:44: 17:52:13.533676 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 17:52:13.533705 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.541840 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 17:52:13.541895 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 3 + logger.go:44: 17:52:13.541934 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.541948 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 1 + logger.go:44: 17:52:13.542000 [INFO] raft: Duplicate RequestVote for same term: 3 + logger.go:44: 17:52:13.542039 [INFO] raft: Duplicate RequestVote for same term: 3 + logger.go:44: 17:52:13.557408 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.557470 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 4 + logger.go:44: 17:52:13.557510 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.557527 [DEBUG] raft: Vote granted from 1 in term 4. Tally: 1 + logger.go:44: 17:52:13.557574 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 17:52:13.557627 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.557648 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.557661 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 4 + logger.go:44: 17:52:13.557679 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.557691 [DEBUG] raft: Vote granted from 2 in term 4. Tally: 1 + logger.go:44: 17:52:13.557715 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 17:52:13.557784 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.569795 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.569853 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 4 + logger.go:44: 17:52:13.569895 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.569910 [DEBUG] raft: Vote granted from 0 in term 4. Tally: 1 + logger.go:44: 17:52:13.569968 [INFO] raft: Duplicate RequestVote for same term: 4 + logger.go:44: 17:52:13.570008 [INFO] raft: Duplicate RequestVote for same term: 4 + logger.go:44: 17:52:13.581062 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.581143 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 5 + logger.go:44: 17:52:13.581179 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.581195 [DEBUG] raft: Vote granted from 1 in term 5. Tally: 1 + logger.go:44: 17:52:13.581246 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 17:52:13.581266 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.581280 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 5 + logger.go:44: 17:52:13.581298 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.581309 [DEBUG] raft: Vote granted from 2 in term 5. Tally: 1 + logger.go:44: 17:52:13.581332 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 17:52:13.581358 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.581384 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.600171 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.600252 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 6 + logger.go:44: 17:52:13.600288 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.600306 [DEBUG] raft: Vote granted from 2 in term 6. Tally: 1 + logger.go:44: 17:52:13.600354 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 17:52:13.600387 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.600451 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.600468 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 5 + logger.go:44: 17:52:13.600503 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.600518 [DEBUG] raft: Vote granted from 0 in term 5. Tally: 1 + logger.go:44: 17:52:13.600567 [DEBUG] raft: Newer term discovered, fallback to follower + logger.go:44: 17:52:13.600606 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.600644 [INFO] raft: Duplicate RequestVote for same term: 5 + logger.go:44: 17:52:13.614331 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.614391 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 6 + logger.go:44: 17:52:13.614424 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.614438 [DEBUG] raft: Vote granted from 1 in term 6. Tally: 1 + logger.go:44: 17:52:13.614480 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 17:52:13.614522 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.621911 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 17:52:13.621974 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 7 + logger.go:44: 17:52:13.622027 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.622043 [DEBUG] raft: Vote granted from 2 in term 7. Tally: 1 + logger.go:44: 17:52:13.622091 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 17:52:13.622109 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.623438 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 17:52:13.623469 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 7 + logger.go:44: 17:52:13.623495 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.623510 [DEBUG] raft: Vote granted from 0 in term 7. Tally: 1 + logger.go:44: 17:52:13.623554 [INFO] raft: Duplicate RequestVote for same term: 7 + logger.go:44: 17:52:13.623605 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.623626 [DEBUG] raft: Vote granted from 1 in term 7. Tally: 2 + logger.go:44: 17:52:13.623637 [INFO] raft: Election won. Tally: 2 + logger.go:44: 17:52:13.623651 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 17:52:13.623673 [INFO] raft: Added peer 1, starting replication + logger.go:44: 17:52:13.623690 [INFO] raft: Added peer 2, starting replication + logger.go:44: 17:52:13.623757 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=7,index=4] + logger.go:44: 17:52:13.623809 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "0") + logger.go:44: 17:52:13.623839 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 17:52:13.623862 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 17:52:13.623882 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 17:52:13.623922 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=7,index=4] + logger.go:44: 17:52:13.623973 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 17:52:13.624009 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 17:52:13.624022 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 17:52:13.624033 [DEBUG] raft-test: elect: done + logger.go:44: 17:52:13.624065 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=7,index=5] + logger.go:44: 17:52:13.624114 [DEBUG] raft-test: fsm 0: applied 2 + logger.go:44: 17:52:13.624149 [DEBUG] raft-test: close: start + logger.go:44: 17:52:13.624182 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=7,index=5] + logger.go:44: 17:52:13.624236 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 17:52:13.624256 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 17:52:13.624275 [DEBUG] raft-test: fsm 1: applied 2 + logger.go:44: 17:52:13.624307 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 17:52:13.624342 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 17:52:13.624372 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 17:52:13.624384 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 17:52:13.624403 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 17:52:13.624422 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 17:52:13.624431 [DEBUG] raft-test: close: done + --- PASS: TestControl_DeposeAfterCommandCommittedThenElect/1 (0.05s) + logger.go:44: 17:52:13.624590 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 17:52:13.624649 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 17:52:13.624707 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 17:52:13.624732 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 17:52:13.624755 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 17:52:13.624808 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:13.624822 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 17:52:13.624859 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:13.624872 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 17:52:13.624924 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:13.624938 [DEBUG] raft-test: setup: done + logger.go:44: 17:52:13.624951 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 17:52:13.624969 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 17:52:13.624988 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 17:52:13.625009 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.625034 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.625056 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.645861 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 17:52:13.645920 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 17:52:13.645971 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.645987 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 17:52:13.646033 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 17:52:13.646047 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 2 + logger.go:44: 17:52:13.646079 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.646097 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 17:52:13.646128 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 1 + logger.go:44: 17:52:13.646180 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 2 + logger.go:44: 17:52:13.646191 [INFO] raft: Election won. Tally: 2 + logger.go:44: 17:52:13.646205 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 17:52:13.646225 [INFO] raft: Added peer 1, starting replication + logger.go:44: 17:52:13.646241 [INFO] raft: Added peer 2, starting replication + logger.go:44: 17:52:13.646295 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] + logger.go:44: 17:52:13.646356 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "0") + logger.go:44: 17:52:13.646391 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 17:52:13.646452 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.646478 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 17:52:13.646500 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 17:52:13.646519 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 17:52:13.646556 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] + logger.go:44: 17:52:13.646606 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 17:52:13.646643 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 17:52:13.646663 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 17:52:13.646674 [DEBUG] raft-test: elect: done + logger.go:44: 17:52:13.646723 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] + logger.go:44: 17:52:13.646781 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 17:52:13.646806 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 17:52:13.646820 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 17:52:13.646836 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] + logger.go:44: 17:52:13.646884 [DEBUG] raft-test: fsm 1: applied 1 + logger.go:44: 17:52:13.648668 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 17:52:13.648718 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected + logger.go:44: 17:52:13.648750 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 17:52:13.648767 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 17:52:13.648884 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 17:52:13.648896 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 17:52:13.648910 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 17:52:13.648923 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 17:52:13.648978 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:13.648989 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:13.648998 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 17:52:13.649052 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:13.649062 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:13.649070 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 17:52:13.650441 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:13.650484 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:13.650506 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 17:52:13.650759 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:13.650775 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:13.650791 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 17:52:13.657829 [WARN] raft: Failed to contact 1 in 10.940409ms + logger.go:44: 17:52:13.657879 [WARN] raft: Failed to contact 2 in 11.050401ms + logger.go:44: 17:52:13.657892 [WARN] raft: Failed to contact quorum of nodes, stepping down + logger.go:44: 17:52:13.657928 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.657953 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 17:52:13.657968 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 17:52:13.658024 [DEBUG] raft-test: elect: start (server 1) + logger.go:44: 17:52:13.658045 [DEBUG] raft-test: server 1: establish outbound connection to all other nodes + logger.go:44: 17:52:13.658066 [DEBUG] raft-test: elect: server 1: wait to become leader within 1.5s + logger.go:44: 17:52:13.661518 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:13.661561 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:13.661587 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 17:52:13.662367 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:13.662386 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:13.662403 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 17:52:13.662444 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:13.662454 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:13.662463 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 17:52:13.662505 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:13.662516 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:13.662527 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 17:52:13.669863 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 17:52:13.669976 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 3 + logger.go:44: 17:52:13.670005 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.670027 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 1 + logger.go:44: 17:52:13.669867 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 17:52:13.670083 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 + logger.go:44: 17:52:13.670101 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.670115 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 + logger.go:44: 17:52:13.670142 [INFO] raft: Duplicate RequestVote for same term: 3 + logger.go:44: 17:52:13.670197 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 2 + logger.go:44: 17:52:13.670209 [INFO] raft: Election won. Tally: 2 + logger.go:44: 17:52:13.670223 [INFO] raft: Node at 1 [Leader] entering Leader state + logger.go:44: 17:52:13.670243 [INFO] raft: Added peer 0, starting replication + logger.go:44: 17:52:13.670262 [INFO] raft: Added peer 2, starting replication + logger.go:44: 17:52:13.670326 [DEBUG] raft-test: server 1: transport: append to 2: 1 entry [Noop:term=3,index=4] + logger.go:44: 17:52:13.670385 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "1") + logger.go:44: 17:52:13.670421 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 17:52:13.670455 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.670478 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 17:52:13.670499 [DEBUG] raft-test: server 1: leadership: acquired + logger.go:44: 17:52:13.670521 [DEBUG] raft-test: elect: server 1: wait for other servers to become followers within 40ms + logger.go:44: 17:52:13.670556 [DEBUG] raft-test: server 1: transport: append to 0: 1 entry [Noop:term=3,index=4] + logger.go:44: 17:52:13.670605 [INFO] raft: pipelining replication to peer {Voter 0 0} + logger.go:44: 17:52:13.670642 [DEBUG] raft-test: fsm 2: applied 1 + logger.go:44: 17:52:13.670668 [DEBUG] raft-test: elect: server 0: became follower + logger.go:44: 17:52:13.670680 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 17:52:13.670690 [DEBUG] raft-test: elect: done + logger.go:44: 17:52:13.670723 [DEBUG] raft-test: server 1: pipeline: append to 2: 1 entry [Command:term=3,index=5] + logger.go:44: 17:52:13.670770 [DEBUG] raft-test: fsm 1: applied 2 + logger.go:44: 17:52:13.670803 [DEBUG] raft-test: close: start + logger.go:44: 17:52:13.670856 [DEBUG] raft-test: server 1: pipeline: append to 0: 1 entry [Command:term=3,index=5] + logger.go:44: 17:52:13.670901 [INFO] raft: aborting pipeline replication to peer {Voter 0 0} + logger.go:44: 17:52:13.670918 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 17:52:13.670932 [DEBUG] raft-test: fsm 0: applied 2 + logger.go:44: 17:52:13.670950 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 17:52:13.670975 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 17:52:13.670997 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 17:52:13.671015 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 17:52:13.671027 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 17:52:13.671050 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 17:52:13.671059 [DEBUG] raft-test: close: done + --- PASS: TestControl_DeposeAfterCommandCommittedThenElect/2 (0.05s) + logger.go:44: 17:52:13.671182 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 17:52:13.671239 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 17:52:13.671306 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 17:52:13.671334 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 17:52:13.671361 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 17:52:13.671420 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:13.671435 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 17:52:13.671474 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:13.671488 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 17:52:13.671521 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:13.671535 [DEBUG] raft-test: setup: done + logger.go:44: 17:52:13.671548 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 17:52:13.671560 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 17:52:13.671582 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 17:52:13.671605 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.671628 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.671701 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.693045 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 17:52:13.693104 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 2 + logger.go:44: 17:52:13.693162 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.693179 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 1 + logger.go:44: 17:52:13.693220 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 17:52:13.693252 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.693279 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 17:52:13.693303 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 17:52:13.693336 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.693350 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 17:52:13.693404 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 2 + logger.go:44: 17:52:13.693419 [INFO] raft: Election won. Tally: 2 + logger.go:44: 17:52:13.693439 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 17:52:13.693457 [INFO] raft: Added peer 1, starting replication + logger.go:44: 17:52:13.693472 [INFO] raft: Added peer 2, starting replication + logger.go:44: 17:52:13.693533 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] + logger.go:44: 17:52:13.693587 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 17:52:13.693621 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 17:52:13.693646 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 17:52:13.693664 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 17:52:13.693696 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] + logger.go:44: 17:52:13.693734 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "0") + logger.go:44: 17:52:13.693806 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 17:52:13.693848 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 17:52:13.693893 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 17:52:13.693906 [DEBUG] raft-test: elect: done + logger.go:44: 17:52:13.693986 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] + logger.go:44: 17:52:13.694038 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] + logger.go:44: 17:52:13.694119 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 17:52:13.694151 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 17:52:13.694167 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 17:52:13.699245 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 17:52:13.699295 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected + logger.go:44: 17:52:13.699335 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 17:52:13.699355 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 17:52:13.699394 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:13.699407 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:13.699419 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 17:52:13.699459 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:13.699477 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:13.699488 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 17:52:13.699508 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 17:52:13.699521 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 17:52:13.699535 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 17:52:13.699550 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 17:52:13.701888 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:13.701931 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:13.701952 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 17:52:13.701999 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:13.702012 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:13.702025 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 17:52:13.709194 [WARN] raft: Failed to contact 1 in 14.984555ms + logger.go:44: 17:52:13.709268 [WARN] raft: Failed to contact 2 in 15.07935ms + logger.go:44: 17:52:13.709282 [WARN] raft: Failed to contact quorum of nodes, stepping down + logger.go:44: 17:52:13.709328 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.709349 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 17:52:13.709362 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 17:52:13.709414 [DEBUG] raft-test: elect: start (server 2) + logger.go:44: 17:52:13.709433 [DEBUG] raft-test: server 2: establish outbound connection to all other nodes + logger.go:44: 17:52:13.709452 [DEBUG] raft-test: elect: server 2: wait to become leader within 1.5s + logger.go:44: 17:52:13.717867 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:13.717915 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:13.717937 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 17:52:13.717981 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 17:52:13.718002 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 3 + logger.go:44: 17:52:13.718027 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.718041 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 1 + logger.go:44: 17:52:13.718075 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 17:52:13.718098 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:13.718111 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:13.718122 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 17:52:13.718152 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:13.718167 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:13.718180 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 17:52:13.718208 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.718232 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 17:52:13.718245 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 + logger.go:44: 17:52:13.718280 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.718292 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 + logger.go:44: 17:52:13.718328 [INFO] raft: Duplicate RequestVote for same term: 3 + logger.go:44: 17:52:13.718385 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 2 + logger.go:44: 17:52:13.718396 [INFO] raft: Election won. Tally: 2 + logger.go:44: 17:52:13.718408 [INFO] raft: Node at 2 [Leader] entering Leader state + logger.go:44: 17:52:13.718425 [INFO] raft: Added peer 0, starting replication + logger.go:44: 17:52:13.718439 [INFO] raft: Added peer 1, starting replication + logger.go:44: 17:52:13.718486 [DEBUG] raft-test: server 2: transport: append to 1: 1 entry [Noop:term=3,index=4] + logger.go:44: 17:52:13.718526 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "2") + logger.go:44: 17:52:13.718554 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 17:52:13.718582 [DEBUG] raft-test: server 2: leadership: acquired + logger.go:44: 17:52:13.718601 [DEBUG] raft-test: elect: server 2: wait for other servers to become followers within 40ms + logger.go:44: 17:52:13.718631 [DEBUG] raft-test: server 2: transport: append to 0: 1 entry [Noop:term=3,index=4] + logger.go:44: 17:52:13.718682 [INFO] raft: pipelining replication to peer {Voter 0 0} + logger.go:44: 17:52:13.718731 [DEBUG] raft-test: elect: server 0: became follower + logger.go:44: 17:52:13.718743 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 17:52:13.718752 [DEBUG] raft-test: elect: done + logger.go:44: 17:52:13.718781 [DEBUG] raft-test: server 2: pipeline: append to 1: 1 entry [Command:term=3,index=5] + logger.go:44: 17:52:13.718827 [DEBUG] raft-test: fsm 1: applied 1 + logger.go:44: 17:52:13.718888 [DEBUG] raft-test: fsm 2: applied 1 + logger.go:44: 17:52:13.718900 [DEBUG] raft-test: fsm 2: applied 2 + logger.go:44: 17:52:13.718928 [DEBUG] raft-test: close: start + logger.go:44: 17:52:13.718956 [DEBUG] raft-test: server 2: pipeline: append to 0: 1 entry [Command:term=3,index=5] + logger.go:44: 17:52:13.719004 [INFO] raft: aborting pipeline replication to peer {Voter 0 0} + logger.go:44: 17:52:13.719021 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 17:52:13.719039 [DEBUG] raft-test: fsm 0: applied 2 + logger.go:44: 17:52:13.719058 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 17:52:13.719086 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:13.719098 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:13.719110 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 17:52:13.719125 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 17:52:13.719149 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 17:52:13.719168 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 17:52:13.719186 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 17:52:13.719206 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 17:52:13.719216 [DEBUG] raft-test: close: done === RUN TestControl_SnapshotAfterCommandCommitted ---- PASS: TestControl_SnapshotAfterCommandCommitted (0.02s) - logger.go:44: 11:26:56.421904 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 11:26:56.422013 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 11:26:56.422139 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 11:26:56.422177 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 11:26:56.422202 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 11:26:56.422274 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:56.422290 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 11:26:56.422336 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:56.422355 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 11:26:56.422388 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:56.422405 [DEBUG] raft-test: setup: done - logger.go:44: 11:26:56.422419 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 11:26:56.422439 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 11:26:56.422465 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 11:26:56.422488 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.422513 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.422537 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.439957 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 11:26:56.440056 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 11:26:56.440112 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:56.440144 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 11:26:56.440205 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 2 - logger.go:44: 11:26:56.440215 [INFO] raft: Election won. Tally: 2 - logger.go:44: 11:26:56.440228 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 11:26:56.440250 [INFO] raft: Added peer 1, starting replication - logger.go:44: 11:26:56.440267 [INFO] raft: Added peer 2, starting replication - logger.go:44: 11:26:56.440321 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] - logger.go:44: 11:26:56.440403 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 11:26:56.440473 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 11:26:56.440492 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 11:26:56.440526 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] - logger.go:44: 11:26:56.440572 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 11:26:56.440610 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 11:26:56.440622 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 11:26:56.440632 [DEBUG] raft-test: elect: done - logger.go:44: 11:26:56.440699 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] - logger.go:44: 11:26:56.440760 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 11:26:56.440804 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 11:26:56.440841 [DEBUG] raft-test: server 0: pipeline: append to 1: 2 entries [Command:term=2,index=3 Command:term=2,index=4] - logger.go:44: 11:26:56.440870 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 11:26:56.440918 [DEBUG] raft-test: fsm 0: applied 2 - logger.go:44: 11:26:56.440955 [DEBUG] raft-test: server 0: control: take snapshot - logger.go:44: 11:26:56.441015 [DEBUG] raft-test: fsm 2: applied 1 - logger.go:44: 11:26:56.441051 [INFO] raft: Starting snapshot up to 4 - logger.go:44: 11:26:56.441122 [INFO] raft: Compacting logs from 1 to 3 - logger.go:44: 11:26:56.441135 [INFO] raft: Snapshot to 4 complete - logger.go:44: 11:26:56.441171 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Other:term=2,index=5] - logger.go:44: 11:26:56.441202 [DEBUG] raft-test: fsm 1: applied 1 - logger.go:44: 11:26:56.441219 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Other:term=2,index=5] - logger.go:44: 11:26:56.441305 [DEBUG] raft-test: fsm 2: applied 2 - logger.go:44: 11:26:56.441317 [DEBUG] raft-test: fsm 1: applied 2 - logger.go:44: 11:26:56.441340 [DEBUG] raft-test: close: start - logger.go:44: 11:26:56.441378 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 11:26:56.441395 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 11:26:56.441425 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 11:26:56.441458 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 11:26:56.441481 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 11:26:56.441501 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 11:26:56.441534 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 11:26:56.441552 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 11:26:56.441563 [DEBUG] raft-test: close: done +--- PASS: TestControl_SnapshotAfterCommandCommitted (0.03s) + logger.go:44: 17:52:13.719361 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 17:52:13.719419 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 17:52:13.719484 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 17:52:13.719512 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 17:52:13.719534 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 17:52:13.719595 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:13.719610 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 17:52:13.719662 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:13.719677 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 17:52:13.719709 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:13.719722 [DEBUG] raft-test: setup: done + logger.go:44: 17:52:13.719734 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 17:52:13.719752 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 17:52:13.719785 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 17:52:13.719812 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.719839 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.719863 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.737113 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 17:52:13.737185 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 2 + logger.go:44: 17:52:13.737232 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.737247 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 1 + logger.go:44: 17:52:13.737282 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 17:52:13.737307 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 17:52:13.744236 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 17:52:13.744314 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 17:52:13.744366 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.744382 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 17:52:13.744428 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 17:52:13.744479 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 2 + logger.go:44: 17:52:13.744492 [INFO] raft: Election won. Tally: 2 + logger.go:44: 17:52:13.744506 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 17:52:13.744524 [INFO] raft: Added peer 1, starting replication + logger.go:44: 17:52:13.744544 [INFO] raft: Added peer 2, starting replication + logger.go:44: 17:52:13.744601 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] + logger.go:44: 17:52:13.744645 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "0") + logger.go:44: 17:52:13.744677 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 17:52:13.744712 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 17:52:13.744728 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 17:52:13.744761 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] + logger.go:44: 17:52:13.744807 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 17:52:13.744852 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 17:52:13.744867 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 17:52:13.744894 [DEBUG] raft-test: elect: done + logger.go:44: 17:52:13.744944 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] + logger.go:44: 17:52:13.745009 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 17:52:13.745045 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=4] + logger.go:44: 17:52:13.745103 [DEBUG] raft-test: server 0: pipeline: append to 1: 2 entries [Command:term=2,index=3 Command:term=2,index=4] + logger.go:44: 17:52:13.745126 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 17:52:13.745171 [DEBUG] raft-test: fsm 0: applied 2 + logger.go:44: 17:52:13.745214 [DEBUG] raft-test: server 0: control: take snapshot + logger.go:44: 17:52:13.745276 [DEBUG] raft-test: fsm 2: applied 1 + logger.go:44: 17:52:13.745314 [INFO] raft: Starting snapshot up to 4 + logger.go:44: 17:52:13.745399 [INFO] raft: Compacting logs from 1 to 3 + logger.go:44: 17:52:13.745414 [INFO] raft: Snapshot to 4 complete + logger.go:44: 17:52:13.745481 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Other:term=2,index=5] + logger.go:44: 17:52:13.745520 [DEBUG] raft-test: fsm 1: applied 1 + logger.go:44: 17:52:13.745538 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Other:term=2,index=5] + logger.go:44: 17:52:13.745609 [DEBUG] raft-test: fsm 2: applied 2 + logger.go:44: 17:52:13.745621 [DEBUG] raft-test: fsm 1: applied 2 + logger.go:44: 17:52:13.745645 [DEBUG] raft-test: close: start + logger.go:44: 17:52:13.745680 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 17:52:13.745720 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 17:52:13.745762 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 17:52:13.745825 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 17:52:13.745906 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 17:52:13.745921 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 17:52:13.745946 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 17:52:13.745962 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 17:52:13.745970 [DEBUG] raft-test: close: done === RUN TestControl_RestoreAfterDisconnection ---- PASS: TestControl_RestoreAfterDisconnection (0.09s) - logger.go:44: 11:26:56.441657 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 11:26:56.441737 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 11:26:56.441785 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 11:26:56.441824 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 11:26:56.441855 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 11:26:56.441926 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:56.441947 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 11:26:56.441997 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:56.442010 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 11:26:56.442044 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 11:26:56.442090 [DEBUG] raft-test: setup: done - logger.go:44: 11:26:56.442112 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 11:26:56.442137 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 11:26:56.442156 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 11:26:56.442178 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.442212 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.442234 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.461793 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 11:26:56.461881 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 11:26:56.461955 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:56.461973 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 11:26:56.462039 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 2 - logger.go:44: 11:26:56.462052 [INFO] raft: Election won. Tally: 2 - logger.go:44: 11:26:56.462065 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 11:26:56.472302 [INFO] raft: Added peer 1, starting replication - logger.go:44: 11:26:56.472351 [INFO] raft: Added peer 2, starting replication - logger.go:44: 11:26:56.473489 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] - logger.go:44: 11:26:56.473625 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 11:26:56.473673 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 11:26:56.473699 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 11:26:56.491705 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:56.491712 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 11:26:56.491861 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 2 - logger.go:44: 11:26:56.491898 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:56.491915 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 1 - logger.go:44: 11:26:56.491935 [INFO] raft: Duplicate RequestVote for same term: 2 - logger.go:44: 11:26:56.491987 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 11:26:56.492034 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 11:26:56.495552 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] - logger.go:44: 11:26:56.495720 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "0") - logger.go:44: 11:26:56.495773 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 11:26:56.495852 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 11:26:56.496005 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 - logger.go:44: 11:26:56.496072 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:56.496094 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 - logger.go:44: 11:26:56.496166 [WARN] raft: Failed to contact 2 in 22.488573ms - logger.go:44: 11:26:56.496277 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 11:26:56.496184 [DEBUG] raft-test: server 0: transport: append to 2: newer term - logger.go:44: 11:26:56.496133 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 11:26:56.496195 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 11:26:56.496119 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 11:26:56.496475 [ERR] raft: peer {Voter 2 2} has newer term, stopping replication - logger.go:44: 11:26:56.496494 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 11:26:56.496534 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.496556 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 11:26:56.496582 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 11:26:56.502013 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 11:26:56.502064 [DEBUG] raft-test: elect: server 0: lost leadership - logger.go:44: 11:26:56.502085 [DEBUG] raft-test: elect: server 0: retry 1 - logger.go:44: 11:26:56.502117 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 11:26:56.502140 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 11:26:56.517611 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 11:26:56.517681 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 3 - logger.go:44: 11:26:56.517720 [DEBUG] raft: Votes needed: 2 - logger.go:44: 11:26:56.517741 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 1 - logger.go:44: 11:26:56.517795 [INFO] raft: Duplicate RequestVote for same term: 3 - logger.go:44: 11:26:56.517845 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 2 - logger.go:44: 11:26:56.517855 [INFO] raft: Election won. Tally: 2 - logger.go:44: 11:26:56.517867 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 11:26:56.517894 [INFO] raft: Added peer 1, starting replication - logger.go:44: 11:26:56.517908 [INFO] raft: Added peer 2, starting replication - logger.go:44: 11:26:56.517969 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=3,index=3] - logger.go:44: 11:26:56.518018 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "0") - logger.go:44: 11:26:56.518050 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 11:26:56.518071 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 11:26:56.518089 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 11:26:56.518123 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=3,index=3] - logger.go:44: 11:26:56.518185 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 11:26:56.518223 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 11:26:56.518235 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 11:26:56.518243 [DEBUG] raft-test: elect: done - logger.go:44: 11:26:56.518290 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=3,index=4] - logger.go:44: 11:26:56.518347 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 11:26:56.518372 [DEBUG] raft-test: term: disconnect 1 - logger.go:44: 11:26:56.518382 [DEBUG] raft-test: server 0: disconnecting follower 1 - logger.go:44: 11:26:56.518408 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=3,index=5] - logger.go:44: 11:26:56.518438 [DEBUG] raft-test: server 0: pipeline: append to 1: 2 entries [Command:term=3,index=4 Command:term=3,index=5] - logger.go:44: 11:26:56.518451 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 11:26:56.518465 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 11:26:56.518479 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 11:26:56.518502 [DEBUG] raft-test: server 0: transport: append to 1: 2 entries [Command:term=3,index=4 Command:term=3,index=5] - logger.go:44: 11:26:56.518516 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:56.518528 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 11:26:56.518574 [DEBUG] raft-test: fsm 0: applied 2 - logger.go:44: 11:26:56.518609 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=3,index=6] - logger.go:44: 11:26:56.518644 [DEBUG] raft-test: fsm 2: applied 1 - logger.go:44: 11:26:56.518681 [DEBUG] raft-test: fsm 0: applied 3 - logger.go:44: 11:26:56.518716 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=3,index=7] - logger.go:44: 11:26:56.518749 [DEBUG] raft-test: fsm 2: applied 2 - logger.go:44: 11:26:56.518784 [DEBUG] raft-test: fsm 0: applied 4 - logger.go:44: 11:26:56.518809 [DEBUG] raft-test: server 0: control: take snapshot - logger.go:44: 11:26:56.518846 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=3,index=8] - logger.go:44: 11:26:56.518867 [DEBUG] raft-test: fsm 2: applied 3 - logger.go:44: 11:26:56.518901 [INFO] raft: Starting snapshot up to 7 - logger.go:44: 11:26:56.518946 [INFO] raft: Compacting logs from 1 to 7 - logger.go:44: 11:26:56.518978 [INFO] raft: Snapshot to 7 complete - logger.go:44: 11:26:56.519014 [DEBUG] raft-test: fsm 0: applied 5 - logger.go:44: 11:26:56.519034 [DEBUG] raft-test: server 0: reconnecting follower 1 - logger.go:44: 11:26:56.519046 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 11:26:56.519065 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 11:26:56.519075 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 11:26:56.519107 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=3,index=9] - logger.go:44: 11:26:56.519134 [DEBUG] raft-test: fsm 2: applied 4 - logger.go:44: 11:26:56.519172 [DEBUG] raft-test: fsm 0: applied 6 - logger.go:44: 11:26:56.519210 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Other:term=3,index=10] - logger.go:44: 11:26:56.519263 [DEBUG] raft-test: fsm 2: applied 5 - logger.go:44: 11:26:56.519307 [DEBUG] raft-test: fsm 2: applied 6 - logger.go:44: 11:26:56.523468 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:56.523542 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:56.523494 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 11:26:56.525197 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 11:26:56.529509 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 11:26:56.529627 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:56.529850 [ERR] raft: Failed to get log at index 3: log not found - logger.go:44: 11:26:56.530031 [INFO] raft: Copied 8 bytes to local snapshot - logger.go:44: 11:26:56.530079 [INFO] raft: Compacting logs from 1 to 2 - logger.go:44: 11:26:56.530091 [INFO] raft: Installed remote snapshot - logger.go:44: 11:26:56.530162 [DEBUG] raft-test: server 0: transport: append to 1: 3 entries [Command:term=3,index=8 Command:term=3,index=9 Other:term=3,index=10] - logger.go:44: 11:26:56.530229 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 11:26:56.530258 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 11:26:56.530317 [DEBUG] raft-test: fsm 1: applied 5 - logger.go:44: 11:26:56.530329 [DEBUG] raft-test: fsm 1: applied 6 - logger.go:44: 11:26:56.530364 [DEBUG] raft-test: close: start - logger.go:44: 11:26:56.530424 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 11:26:56.530443 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 11:26:56.530465 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 11:26:56.530492 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 11:26:56.530515 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 11:26:56.530527 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 11:26:56.530552 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 11:26:56.530568 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 11:26:56.530576 [DEBUG] raft-test: close: done +--- PASS: TestControl_RestoreAfterDisconnection (0.05s) + logger.go:44: 17:52:13.746054 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 17:52:13.746109 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 17:52:13.746169 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 17:52:13.746190 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 17:52:13.746211 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 17:52:13.746264 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:13.746278 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 17:52:13.746318 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:13.746330 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 17:52:13.746369 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 17:52:13.746383 [DEBUG] raft-test: setup: done + logger.go:44: 17:52:13.746397 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 17:52:13.746417 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 17:52:13.746438 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 17:52:13.746464 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.753997 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.754070 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.781991 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 17:52:13.782100 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 17:52:13.782136 [DEBUG] raft: Votes needed: 2 + logger.go:44: 17:52:13.782155 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 17:52:13.782225 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 2 + logger.go:44: 17:52:13.782235 [INFO] raft: Election won. Tally: 2 + logger.go:44: 17:52:13.782249 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 17:52:13.782300 [INFO] raft: Added peer 1, starting replication + logger.go:44: 17:52:13.782315 [INFO] raft: Added peer 2, starting replication + logger.go:44: 17:52:13.782368 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] + logger.go:44: 17:52:13.782438 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 17:52:13.782494 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 17:52:13.782512 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 17:52:13.782545 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] + logger.go:44: 17:52:13.782591 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 17:52:13.782631 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 17:52:13.782643 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 17:52:13.782652 [DEBUG] raft-test: elect: done + logger.go:44: 17:52:13.782704 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] + logger.go:44: 17:52:13.782767 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 17:52:13.782794 [DEBUG] raft-test: term: disconnect 1 + logger.go:44: 17:52:13.782808 [DEBUG] raft-test: server 0: disconnecting follower 1 + logger.go:44: 17:52:13.782839 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=4] + logger.go:44: 17:52:13.782873 [DEBUG] raft-test: server 0: pipeline: append to 1: 2 entries [Command:term=2,index=3 Command:term=2,index=4] + logger.go:44: 17:52:13.782889 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 17:52:13.782904 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 17:52:13.782923 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 17:52:13.782944 [DEBUG] raft-test: server 0: transport: append to 1: 2 entries [Command:term=2,index=3 Command:term=2,index=4] + logger.go:44: 17:52:13.782961 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:13.782976 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 17:52:13.783013 [DEBUG] raft-test: fsm 0: applied 2 + logger.go:44: 17:52:13.783045 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=5] + logger.go:44: 17:52:13.783078 [DEBUG] raft-test: fsm 2: applied 1 + logger.go:44: 17:52:13.783114 [DEBUG] raft-test: fsm 0: applied 3 + logger.go:44: 17:52:13.783149 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=6] + logger.go:44: 17:52:13.783171 [DEBUG] raft-test: fsm 2: applied 2 + logger.go:44: 17:52:13.783204 [DEBUG] raft-test: fsm 0: applied 4 + logger.go:44: 17:52:13.783230 [DEBUG] raft-test: server 0: control: take snapshot + logger.go:44: 17:52:13.783269 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=7] + logger.go:44: 17:52:13.783309 [DEBUG] raft-test: fsm 2: applied 3 + logger.go:44: 17:52:13.783341 [INFO] raft: Starting snapshot up to 6 + logger.go:44: 17:52:13.783382 [INFO] raft: Compacting logs from 1 to 6 + logger.go:44: 17:52:13.783411 [INFO] raft: Snapshot to 6 complete + logger.go:44: 17:52:13.783448 [DEBUG] raft-test: fsm 0: applied 5 + logger.go:44: 17:52:13.783467 [DEBUG] raft-test: server 0: reconnecting follower 1 + logger.go:44: 17:52:13.783479 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 17:52:13.783500 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 17:52:13.783511 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 17:52:13.783539 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=8] + logger.go:44: 17:52:13.783564 [DEBUG] raft-test: fsm 2: applied 4 + logger.go:44: 17:52:13.783596 [DEBUG] raft-test: fsm 0: applied 6 + logger.go:44: 17:52:13.783634 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Other:term=2,index=9] + logger.go:44: 17:52:13.783655 [DEBUG] raft-test: fsm 2: applied 5 + logger.go:44: 17:52:13.783699 [DEBUG] raft-test: fsm 2: applied 6 + logger.go:44: 17:52:13.785116 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 17:52:13.785272 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:13.785374 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:13.786486 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 17:52:13.787515 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:13.787593 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:13.793916 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 17:52:13.793970 [ERR] raft: Failed to get log at index 2: log not found + logger.go:44: 17:52:13.794041 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:13.793922 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 17:52:13.794177 [INFO] raft: Copied 8 bytes to local snapshot + logger.go:44: 17:52:13.794227 [INFO] raft: Compacting logs from 1 to 1 + logger.go:44: 17:52:13.794240 [INFO] raft: Installed remote snapshot + logger.go:44: 17:52:13.794304 [DEBUG] raft-test: server 0: transport: append to 1: 3 entries [Command:term=2,index=7 Command:term=2,index=8 Other:term=2,index=9] + logger.go:44: 17:52:13.794359 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 17:52:13.794383 [DEBUG] raft-test: fsm 1: applied 5 + logger.go:44: 17:52:13.794397 [DEBUG] raft-test: fsm 1: applied 6 + logger.go:44: 17:52:13.794435 [DEBUG] raft-test: close: start + logger.go:44: 17:52:13.794480 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 17:52:13.794501 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 17:52:13.794526 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 17:52:13.794556 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 17:52:13.794583 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 17:52:13.794595 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 17:52:13.794617 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 17:52:13.794635 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 17:52:13.794645 [DEBUG] raft-test: close: done === RUN TestFSM_Restore --- PASS: TestFSM_Restore (0.00s) === RUN TestServer_StartAndShutdown --- PASS: TestServer_StartAndShutdown (0.03s) - logger.go:44: 11:26:56.530692 [DEBUG] raft-test: setup: start (1 servers) - logger.go:44: 11:26:56.530731 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 11:26:56.530795 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 11:26:56.530838 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0}] - logger.go:44: 11:26:56.530858 [DEBUG] raft-test: setup: done - logger.go:44: 11:26:56.530872 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 11:26:56.530890 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 11:26:56.530909 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 11:26:56.530937 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 11:26:56.559674 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 11:26:56.559760 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 11:26:56.559817 [DEBUG] raft: Votes needed: 1 - logger.go:44: 11:26:56.559833 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 11:26:56.559842 [INFO] raft: Election won. Tally: 1 - logger.go:44: 11:26:56.559855 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 11:26:56.559955 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 11:26:56.559974 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 11:26:56.560005 [DEBUG] raft-test: elect: done - logger.go:44: 11:26:56.560065 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 11:26:56.560082 [DEBUG] raft-test: close: start - logger.go:44: 11:26:56.560119 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 11:26:56.560140 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 11:26:56.560149 [DEBUG] raft-test: close: done + logger.go:44: 17:52:13.794757 [DEBUG] raft-test: setup: start (1 servers) + logger.go:44: 17:52:13.794794 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 17:52:13.794841 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 17:52:13.794881 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0}] + logger.go:44: 17:52:13.794899 [DEBUG] raft-test: setup: done + logger.go:44: 17:52:13.794913 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 17:52:13.794931 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 17:52:13.794949 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 17:52:13.795002 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 17:52:13.821860 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 17:52:13.821984 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 17:52:13.822010 [DEBUG] raft: Votes needed: 1 + logger.go:44: 17:52:13.822029 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 17:52:13.822038 [INFO] raft: Election won. Tally: 1 + logger.go:44: 17:52:13.822050 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 17:52:13.822114 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 17:52:13.822134 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 17:52:13.822165 [DEBUG] raft-test: elect: done + logger.go:44: 17:52:13.822225 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 17:52:13.822244 [DEBUG] raft-test: close: start + logger.go:44: 17:52:13.822292 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 17:52:13.822317 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 17:52:13.822329 [DEBUG] raft-test: close: done === RUN ExampleControl_Commands ---- PASS: ExampleControl_Commands (0.04s) +--- PASS: ExampleControl_Commands (0.05s) === RUN ExampleControl_Snapshots ---- PASS: ExampleControl_Snapshots (0.08s) +--- PASS: ExampleControl_Snapshots (0.04s) PASS -ok github.com/CanonicalLtd/raft-test 1.356s +ok github.com/CanonicalLtd/raft-test 1.230s === RUN TestNotifier_Acquired --- PASS: TestNotifier_Acquired (0.00s) - logger.go:44: 11:26:42.334302 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 11:26:42.334464 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 17:52:12.476157 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 17:52:12.476323 [DEBUG] raft-test: server 0: leadership: stop watching === RUN TestNotifier_AcquiredTimeout --- PASS: TestNotifier_AcquiredTimeout (0.00s) - logger.go:44: 11:26:42.334655 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 17:52:12.476499 [DEBUG] raft-test: server 0: leadership: stop watching === RUN TestNotifier_LeaderhsipLost --- PASS: TestNotifier_LeaderhsipLost (0.00s) - logger.go:44: 11:26:42.335015 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 11:26:42.335085 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 11:26:42.335109 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 17:52:12.476596 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 17:52:12.476687 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 17:52:12.476702 [DEBUG] raft-test: server 0: leadership: stop watching === RUN TestNotifier_LostTwice --- PASS: TestNotifier_LostTwice (0.00s) - logger.go:44: 11:26:42.335222 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 11:26:42.335251 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 17:52:12.476751 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 17:52:12.476782 [DEBUG] raft-test: server 0: leadership: stop watching === RUN TestNotifier_UnexpectedLeadershipChange --- PASS: TestNotifier_UnexpectedLeadershipChange (0.00s) === RUN TestNotifier_InconsistentNotifications --- PASS: TestNotifier_InconsistentNotifications (0.01s) - logger.go:44: 11:26:42.345577 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 17:52:12.489991 [DEBUG] raft-test: server 0: leadership: acquired === RUN TestNotifier_DoubleAcquiredRequest --- PASS: TestNotifier_DoubleAcquiredRequest (0.00s) === RUN TestNotifier_DoubleNotification --- PASS: TestNotifier_DoubleNotification (0.01s) - logger.go:44: 11:26:42.356004 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 17:52:12.501844 [DEBUG] raft-test: server 0: leadership: acquired === RUN TestNotifier_OneLeadershipAfterTheOther --- PASS: TestNotifier_OneLeadershipAfterTheOther (0.00s) - logger.go:44: 11:26:42.356279 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 11:26:42.356325 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 11:26:42.356350 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 11:26:42.356369 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 17:52:12.502024 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 17:52:12.502051 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 17:52:12.502064 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 17:52:12.502093 [DEBUG] raft-test: server 0: leadership: stop watching === RUN TestTracker_Acquired --- PASS: TestTracker_Acquired (0.00s) - logger.go:44: 11:26:42.356494 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 11:26:42.356513 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 11:26:42.356533 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 11:26:42.356551 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 17:52:12.502157 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 17:52:12.502186 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 17:52:12.502203 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 17:52:12.502215 [DEBUG] raft-test: server 0: leadership: stop watching === RUN TestTracker_AcquiredTimeout --- PASS: TestTracker_AcquiredTimeout (0.00s) - logger.go:44: 11:26:42.356715 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 17:52:12.506320 [DEBUG] raft-test: server 0: leadership: stop watching === RUN TestTracker_AcquiredTwice --- PASS: TestTracker_AcquiredTwice (0.00s) - logger.go:44: 11:26:42.356801 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 11:26:42.356830 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 11:26:42.356850 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 11:26:42.356867 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 17:52:12.507485 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 17:52:12.507522 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 17:52:12.507534 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 17:52:12.507552 [DEBUG] raft-test: server 1: leadership: stop watching === RUN TestTracker_AcquiredAfterLost --- PASS: TestTracker_AcquiredAfterLost (0.00s) - logger.go:44: 11:26:42.356930 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 11:26:42.356964 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 11:26:42.357129 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 11:26:42.357163 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 11:26:42.357182 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 17:52:12.507597 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 17:52:12.507624 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 17:52:12.507643 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 17:52:12.507653 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 17:52:12.507667 [DEBUG] raft-test: server 2: leadership: stop watching === RUN TestTracker_AddSameServerID --- PASS: TestTracker_AddSameServerID (0.00s) - logger.go:44: 11:26:42.357263 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 17:52:12.507721 [DEBUG] raft-test: server 0: leadership: stop watching === RUN TestTracker_AddAfterObserving --- PASS: TestTracker_AddAfterObserving (0.00s) - logger.go:44: 11:26:42.357368 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 17:52:12.507777 [DEBUG] raft-test: server 0: leadership: stop watching PASS -ok github.com/CanonicalLtd/raft-test/internal/election 0.034s +ok github.com/CanonicalLtd/raft-test/internal/election 0.061s === RUN TestEvent --- PASS: TestEvent (0.00s) PASS -ok github.com/CanonicalLtd/raft-test/internal/event 0.028s +ok github.com/CanonicalLtd/raft-test/internal/event 0.036s ? github.com/CanonicalLtd/raft-test/internal/fsms [no test files] === RUN TestNew --- PASS: TestNew (0.00s) - logger.go:44: 11:26:21.931415 [TRACE] raft-test: hello + logger.go:44: 17:52:09.660142 [TRACE] raft-test: hello PASS -ok github.com/CanonicalLtd/raft-test/internal/logging 0.040s +ok github.com/CanonicalLtd/raft-test/internal/logging 0.009s === RUN TestStringifyLogs === RUN TestStringifyLogs/0_entries === RUN TestStringifyLogs/1_entry_[Noop:term=1,index=1] @@ -2503,43 +2687,43 @@ --- PASS: TestPeer_CommandLogsCount (0.00s) === RUN TestFaultyTransport_AppendEntries_Default --- PASS: TestFaultyTransport_AppendEntries_Default (0.00s) - logger.go:44: 11:26:53.614735 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 11:26:53.614843 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:12.582761 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 17:52:12.582859 [DEBUG] raft-test: server 0: transport: append to 1: not connected === RUN TestFaultyTransport_AppendEntries_Connected --- PASS: TestFaultyTransport_AppendEntries_Connected (0.00s) - logger.go:44: 11:26:53.614960 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=1,index=1] + logger.go:44: 17:52:12.582929 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=1,index=1] === RUN TestFaultyTransport_PipelineAppendEntries_Disconnected --- PASS: TestFaultyTransport_PipelineAppendEntries_Disconnected (0.00s) - logger.go:44: 11:26:53.615274 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 11:26:53.615347 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 17:52:12.587697 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 17:52:12.587721 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected === RUN TestFaultyTransport_PipelineAppendEntries_Connected --- PASS: TestFaultyTransport_PipelineAppendEntries_Connected (0.00s) - logger.go:44: 11:26:53.615459 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 17:52:12.587799 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries === RUN TestFaultyTransport_Deposing --- PASS: TestFaultyTransport_Deposing (0.00s) - logger.go:44: 11:26:53.615590 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=1,index=1] - logger.go:44: 11:26:53.615623 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=1,index=1] - logger.go:44: 11:26:53.615654 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=1,index=3] - logger.go:44: 11:26:53.615697 [DEBUG] raft-test: server 0: transport: append to 2: syncing logs - logger.go:44: 11:26:53.615718 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=1,index=3] - logger.go:44: 11:26:53.615760 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=1,index=3] - logger.go:44: 11:26:53.615773 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 11:26:53.615793 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=1,index=3] - logger.go:44: 11:26:53.615805 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected - logger.go:44: 11:26:53.615819 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=1,index=3] - logger.go:44: 11:26:53.615834 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 11:26:53.615878 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=1,index=3] - logger.go:44: 11:26:53.616031 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 17:52:12.587916 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=1,index=1] + logger.go:44: 17:52:12.587945 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=1,index=1] + logger.go:44: 17:52:12.587974 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=1,index=3] + logger.go:44: 17:52:12.588011 [DEBUG] raft-test: server 0: transport: append to 2: syncing logs + logger.go:44: 17:52:12.588031 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=1,index=3] + logger.go:44: 17:52:12.588073 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=1,index=3] + logger.go:44: 17:52:12.588085 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 17:52:12.588105 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=1,index=3] + logger.go:44: 17:52:12.588116 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected + logger.go:44: 17:52:12.588129 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=1,index=3] + logger.go:44: 17:52:12.588142 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 17:52:12.588161 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=1,index=3] + logger.go:44: 17:52:12.588172 [DEBUG] raft-test: server 0: transport: append to 2: not connected === RUN TestNetwork_FaultyEnqueue --- PASS: TestNetwork_FaultyEnqueue (0.00s) - logger.go:44: 11:26:53.616112 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 11:26:53.616131 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=1,index=1] - logger.go:44: 11:26:53.616151 [DEBUG] raft-test: server 0: transport: append to 1: newer term - logger.go:44: 11:26:53.616191 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=0,index=0] - logger.go:44: 11:26:53.616201 [DEBUG] raft-test: server 0: pipeline: append to: 1: enqueue fault: command 1 - logger.go:44: 11:26:53.616250 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 17:52:12.588233 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 17:52:12.588251 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=1,index=1] + logger.go:44: 17:52:12.588270 [DEBUG] raft-test: server 0: transport: append to 1: newer term + logger.go:44: 17:52:12.588306 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=0,index=0] + logger.go:44: 17:52:12.588318 [DEBUG] raft-test: server 0: pipeline: append to: 1: enqueue fault: command 1 + logger.go:44: 17:52:12.588370 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes PASS -ok github.com/CanonicalLtd/raft-test/internal/network 0.012s +ok github.com/CanonicalLtd/raft-test/internal/network 0.036s create-stamp debian/debhelper-build-stamp fakeroot debian/rules binary dh binary --buildsystem=golang --with=golang @@ -2568,12 +2752,14 @@ dpkg-buildpackage: info: binary-only upload (no source included) dpkg-genchanges: info: including full source code in upload I: copying local configuration +I: user script /srv/workspace/pbuilder/274583/tmp/hooks/B01_cleanup starting +I: user script /srv/workspace/pbuilder/274583/tmp/hooks/B01_cleanup finished I: unmounting dev/ptmx filesystem I: unmounting dev/pts filesystem I: unmounting dev/shm filesystem I: unmounting proc filesystem I: unmounting sys filesystem I: cleaning the build env -I: removing directory /srv/workspace/pbuilder/1789904 and its subdirectories -I: Current time: Thu Mar 17 23:27:42 -12 2022 -I: pbuilder-time-stamp: 1647602862 +I: removing directory /srv/workspace/pbuilder/274583 and its subdirectories +I: Current time: Fri Apr 21 07:52:22 +14 2023 +I: pbuilder-time-stamp: 1682013142