Diff of the two buildlogs: -- --- b1/build.log 2020-06-16 10:37:35.671531307 +0000 +++ b2/build.log 2020-06-16 10:45:06.582742016 +0000 @@ -1,6 +1,6 @@ I: pbuilder: network access will be disabled during build -I: Current time: Mon Jul 19 04:59:56 -12 2021 -I: pbuilder-time-stamp: 1626713996 +I: Current time: Wed Jun 17 00:37:37 +14 2020 +I: pbuilder-time-stamp: 1592303857 I: Building the build Environment I: extracting base tarball [/var/cache/pbuilder/buster-reproducible-base.tgz] I: copying local configuration @@ -18,7 +18,7 @@ I: Extracting source gpgv: unknown type of key resource 'trustedkeys.kbx' gpgv: keyblock resource '/root/.gnupg/trustedkeys.kbx': General error -gpgv: Signature made Thu Aug 2 20:35:10 2018 -12 +gpgv: Signature made Fri Aug 3 22:35:10 2018 +14 gpgv: using RSA key 0740778A2DFC4A39C0C8BC8C8F2B113C6535C5A7 gpgv: Can't check signature: No public key dpkg-source: warning: failed to verify signature on ./golang-github-canonicalltd-raft-test_0.0~git20180628.c3345b5-1.dsc @@ -27,138 +27,172 @@ dpkg-source: info: unpacking golang-github-canonicalltd-raft-test_0.0~git20180628.c3345b5-1.debian.tar.xz I: using fakeroot in build. I: Installing the build-deps -I: user script /srv/workspace/pbuilder/13551/tmp/hooks/D02_print_environment starting +I: user script /srv/workspace/pbuilder/5110/tmp/hooks/D01_modify_environment starting +debug: Running on profitbricks-build2-i386. +I: Changing host+domainname to test build reproducibility +I: Adding a custom variable just for the fun of it... +I: Changing /bin/sh to bash +Removing 'diversion of /bin/sh to /bin/sh.distrib by dash' +Adding 'diversion of /bin/sh to /bin/sh.distrib by bash' +Removing 'diversion of /usr/share/man/man1/sh.1.gz to /usr/share/man/man1/sh.distrib.1.gz by dash' +Adding 'diversion of /usr/share/man/man1/sh.1.gz to /usr/share/man/man1/sh.distrib.1.gz by bash' +I: Setting pbuilder2's login shell to /bin/bash +I: Setting pbuilder2's GECOS to second user,second room,second work-phone,second home-phone,second other +I: user script /srv/workspace/pbuilder/5110/tmp/hooks/D01_modify_environment finished +I: user script /srv/workspace/pbuilder/5110/tmp/hooks/D02_print_environment starting I: set - BUILDDIR='/build' - BUILDUSERGECOS='first user,first room,first work-phone,first home-phone,first other' - BUILDUSERNAME='pbuilder1' - BUILD_ARCH='i386' - DEBIAN_FRONTEND='noninteractive' - DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all parallel=18' - DISTRIBUTION='' - HOME='/root' - HOST_ARCH='i386' + BASH=/bin/sh + BASHOPTS=checkwinsize:cmdhist:complete_fullquote:extquote:force_fignore:globasciiranges:hostcomplete:interactive_comments:progcomp:promptvars:sourcepath + BASH_ALIASES=() + BASH_ARGC=() + BASH_ARGV=() + BASH_CMDS=() + BASH_LINENO=([0]="12" [1]="0") + BASH_SOURCE=([0]="/tmp/hooks/D02_print_environment" [1]="/tmp/hooks/D02_print_environment") + BASH_VERSINFO=([0]="5" [1]="0" [2]="3" [3]="1" [4]="release" [5]="i686-pc-linux-gnu") + BASH_VERSION='5.0.3(1)-release' + BUILDDIR=/build + BUILDUSERGECOS='second user,second room,second work-phone,second home-phone,second other' + BUILDUSERNAME=pbuilder2 + BUILD_ARCH=i386 + DEBIAN_FRONTEND=noninteractive + DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all parallel=10' + DIRSTACK=() + DISTRIBUTION= + EUID=0 + FUNCNAME=([0]="Echo" [1]="main") + GROUPS=() + HOME=/root + HOSTNAME=i-capture-the-hostname + HOSTTYPE=i686 + HOST_ARCH=i386 IFS=' ' - INVOCATION_ID='6bc30f8cc5e041258cc70506df49c663' - LANG='C' - LANGUAGE='en_US:en' - LC_ALL='C' - LD_LIBRARY_PATH='/usr/lib/libeatmydata' - LD_PRELOAD='libeatmydata.so' - MAIL='/var/mail/root' - OPTIND='1' - PATH='/usr/sbin:/usr/bin:/sbin:/bin:/usr/games' - PBCURRENTCOMMANDLINEOPERATION='build' - PBUILDER_OPERATION='build' - PBUILDER_PKGDATADIR='/usr/share/pbuilder' - PBUILDER_PKGLIBDIR='/usr/lib/pbuilder' - PBUILDER_SYSCONFDIR='/etc' - PPID='13551' - PS1='# ' - PS2='> ' + INVOCATION_ID=543021c2dc7548579b2a9db8ab3d03bd + LANG=C + LANGUAGE=de_CH:de + LC_ALL=C + LD_LIBRARY_PATH=/usr/lib/libeatmydata + LD_PRELOAD=libeatmydata.so + MACHTYPE=i686-pc-linux-gnu + MAIL=/var/mail/root + OPTERR=1 + OPTIND=1 + OSTYPE=linux-gnu + PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path + PBCURRENTCOMMANDLINEOPERATION=build + PBUILDER_OPERATION=build + PBUILDER_PKGDATADIR=/usr/share/pbuilder + PBUILDER_PKGLIBDIR=/usr/lib/pbuilder + PBUILDER_SYSCONFDIR=/etc + PIPESTATUS=([0]="0") + POSIXLY_CORRECT=y + PPID=5110 PS4='+ ' - PWD='/' - SHELL='/bin/bash' - SHLVL='2' - SUDO_COMMAND='/usr/bin/timeout -k 18.1h 18h /usr/bin/ionice -c 3 /usr/bin/nice /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/tmp.sDs01szlXm/pbuilderrc_jSvI --hookdir /etc/pbuilder/first-build-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/buster-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/tmp.sDs01szlXm/b1 --logfile b1/build.log golang-github-canonicalltd-raft-test_0.0~git20180628.c3345b5-1.dsc' - SUDO_GID='112' - SUDO_UID='107' - SUDO_USER='jenkins' - TERM='unknown' - TZ='/usr/share/zoneinfo/Etc/GMT+12' - USER='root' - _='/usr/bin/systemd-run' - http_proxy='http://85.184.249.68:3128' + PWD=/ + SHELL=/bin/bash + SHELLOPTS=braceexpand:errexit:hashall:interactive-comments:posix + SHLVL=3 + SUDO_COMMAND='/usr/bin/timeout -k 24.1h 24h /usr/bin/ionice -c 3 /usr/bin/nice -n 11 /usr/bin/unshare --uts -- /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/tmp.sDs01szlXm/pbuilderrc_Hkef --hookdir /etc/pbuilder/rebuild-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/buster-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/tmp.sDs01szlXm/b2 --logfile b2/build.log golang-github-canonicalltd-raft-test_0.0~git20180628.c3345b5-1.dsc' + SUDO_GID=112 + SUDO_UID=107 + SUDO_USER=jenkins + TERM=unknown + TZ=/usr/share/zoneinfo/Etc/GMT-14 + UID=0 + USER=root + _='I: set' + http_proxy=http://78.137.99.97:3128 I: uname -a - Linux profitbricks-build6-i386 4.19.0-9-amd64 #1 SMP Debian 4.19.118-2+deb10u1 (2020-06-07) x86_64 GNU/Linux + Linux i-capture-the-hostname 4.19.0-9-686-pae #1 SMP Debian 4.19.118-2+deb10u1 (2020-06-07) i686 GNU/Linux I: ls -l /bin total 5476 - -rwxr-xr-x 1 root root 1302248 Apr 17 2019 bash - -rwxr-xr-x 3 root root 38280 Jul 10 2019 bunzip2 - -rwxr-xr-x 3 root root 38280 Jul 10 2019 bzcat - lrwxrwxrwx 1 root root 6 Jul 10 2019 bzcmp -> bzdiff - -rwxr-xr-x 1 root root 2227 Jul 10 2019 bzdiff - lrwxrwxrwx 1 root root 6 Jul 10 2019 bzegrep -> bzgrep - -rwxr-xr-x 1 root root 4877 Jun 24 2019 bzexe - lrwxrwxrwx 1 root root 6 Jul 10 2019 bzfgrep -> bzgrep - -rwxr-xr-x 1 root root 3641 Jul 10 2019 bzgrep - -rwxr-xr-x 3 root root 38280 Jul 10 2019 bzip2 - -rwxr-xr-x 1 root root 17768 Jul 10 2019 bzip2recover - lrwxrwxrwx 1 root root 6 Jul 10 2019 bzless -> bzmore - -rwxr-xr-x 1 root root 1297 Jul 10 2019 bzmore - -rwxr-xr-x 1 root root 38692 Feb 28 2019 cat - -rwxr-xr-x 1 root root 75588 Feb 28 2019 chgrp - -rwxr-xr-x 1 root root 63268 Feb 28 2019 chmod - -rwxr-xr-x 1 root root 75588 Feb 28 2019 chown - -rwxr-xr-x 1 root root 153732 Feb 28 2019 cp - -rwxr-xr-x 1 root root 132820 Jan 17 2019 dash - -rwxr-xr-x 1 root root 120676 Feb 28 2019 date - -rwxr-xr-x 1 root root 92040 Feb 28 2019 dd - -rwxr-xr-x 1 root root 100620 Feb 28 2019 df - -rwxr-xr-x 1 root root 149736 Feb 28 2019 dir - -rwxr-xr-x 1 root root 79412 Jan 9 2019 dmesg - lrwxrwxrwx 1 root root 8 Sep 26 2018 dnsdomainname -> hostname - lrwxrwxrwx 1 root root 8 Sep 26 2018 domainname -> hostname - -rwxr-xr-x 1 root root 34532 Feb 28 2019 echo - -rwxr-xr-x 1 root root 28 Jan 7 2019 egrep - -rwxr-xr-x 1 root root 34532 Feb 28 2019 false - -rwxr-xr-x 1 root root 28 Jan 7 2019 fgrep - -rwxr-xr-x 1 root root 67700 Jan 9 2019 findmnt - -rwsr-xr-x 1 root root 30112 Apr 22 2020 fusermount - -rwxr-xr-x 1 root root 206392 Jan 7 2019 grep - -rwxr-xr-x 2 root root 2345 Jan 5 2019 gunzip - -rwxr-xr-x 1 root root 6375 Jan 5 2019 gzexe - -rwxr-xr-x 1 root root 100952 Jan 5 2019 gzip - -rwxr-xr-x 1 root root 21916 Sep 26 2018 hostname - -rwxr-xr-x 1 root root 79752 Feb 28 2019 ln - -rwxr-xr-x 1 root root 55400 Jul 26 2018 login - -rwxr-xr-x 1 root root 149736 Feb 28 2019 ls - -rwxr-xr-x 1 root root 112032 Jan 9 2019 lsblk - -rwxr-xr-x 1 root root 87972 Feb 28 2019 mkdir - -rwxr-xr-x 1 root root 79748 Feb 28 2019 mknod - -rwxr-xr-x 1 root root 46916 Feb 28 2019 mktemp - -rwxr-xr-x 1 root root 42348 Jan 9 2019 more - -rwsr-xr-x 1 root root 50592 Jan 9 2019 mount - -rwxr-xr-x 1 root root 13724 Jan 9 2019 mountpoint - -rwxr-xr-x 1 root root 157832 Feb 28 2019 mv - lrwxrwxrwx 1 root root 8 Sep 26 2018 nisdomainname -> hostname - lrwxrwxrwx 1 root root 14 Feb 14 2019 pidof -> /sbin/killall5 - -rwxr-xr-x 1 root root 38660 Feb 28 2019 pwd - lrwxrwxrwx 1 root root 4 Apr 17 2019 rbash -> bash - -rwxr-xr-x 1 root root 46852 Feb 28 2019 readlink - -rwxr-xr-x 1 root root 75588 Feb 28 2019 rm - -rwxr-xr-x 1 root root 42756 Feb 28 2019 rmdir - -rwxr-xr-x 1 root root 22276 Jan 21 2019 run-parts - -rwxr-xr-x 1 root root 125036 Dec 22 2018 sed - lrwxrwxrwx 1 root root 4 Jul 19 02:46 sh -> dash - -rwxr-xr-x 1 root root 34532 Feb 28 2019 sleep - -rwxr-xr-x 1 root root 79652 Feb 28 2019 stty - -rwsr-xr-x 1 root root 71072 Jan 9 2019 su - -rwxr-xr-x 1 root root 34564 Feb 28 2019 sync - -rwxr-xr-x 1 root root 504024 Apr 23 2019 tar - -rwxr-xr-x 1 root root 13860 Jan 21 2019 tempfile - -rwxr-xr-x 1 root root 104292 Feb 28 2019 touch - -rwxr-xr-x 1 root root 34532 Feb 28 2019 true - -rwxr-xr-x 1 root root 17768 Apr 22 2020 ulockmgr_server - -rwsr-xr-x 1 root root 30108 Jan 9 2019 umount - -rwxr-xr-x 1 root root 34532 Feb 28 2019 uname - -rwxr-xr-x 2 root root 2345 Jan 5 2019 uncompress - -rwxr-xr-x 1 root root 149736 Feb 28 2019 vdir - -rwxr-xr-x 1 root root 34208 Jan 9 2019 wdctl - -rwxr-xr-x 1 root root 946 Jan 21 2019 which - lrwxrwxrwx 1 root root 8 Sep 26 2018 ypdomainname -> hostname - -rwxr-xr-x 1 root root 1983 Jan 5 2019 zcat - -rwxr-xr-x 1 root root 1677 Jan 5 2019 zcmp - -rwxr-xr-x 1 root root 5879 Jan 5 2019 zdiff - -rwxr-xr-x 1 root root 29 Jan 5 2019 zegrep - -rwxr-xr-x 1 root root 29 Jan 5 2019 zfgrep - -rwxr-xr-x 1 root root 2080 Jan 5 2019 zforce - -rwxr-xr-x 1 root root 7584 Jan 5 2019 zgrep - -rwxr-xr-x 1 root root 2205 Jan 5 2019 zless - -rwxr-xr-x 1 root root 1841 Jan 5 2019 zmore - -rwxr-xr-x 1 root root 4552 Jan 5 2019 znew -I: user script /srv/workspace/pbuilder/13551/tmp/hooks/D02_print_environment finished + -rwxr-xr-x 1 root root 1302248 Apr 18 2019 bash + -rwxr-xr-x 3 root root 38280 Jul 11 2019 bunzip2 + -rwxr-xr-x 3 root root 38280 Jul 11 2019 bzcat + lrwxrwxrwx 1 root root 6 Jul 11 2019 bzcmp -> bzdiff + -rwxr-xr-x 1 root root 2227 Jul 11 2019 bzdiff + lrwxrwxrwx 1 root root 6 Jul 11 2019 bzegrep -> bzgrep + -rwxr-xr-x 1 root root 4877 Jun 25 2019 bzexe + lrwxrwxrwx 1 root root 6 Jul 11 2019 bzfgrep -> bzgrep + -rwxr-xr-x 1 root root 3641 Jul 11 2019 bzgrep + -rwxr-xr-x 3 root root 38280 Jul 11 2019 bzip2 + -rwxr-xr-x 1 root root 17768 Jul 11 2019 bzip2recover + lrwxrwxrwx 1 root root 6 Jul 11 2019 bzless -> bzmore + -rwxr-xr-x 1 root root 1297 Jul 11 2019 bzmore + -rwxr-xr-x 1 root root 38692 Mar 1 2019 cat + -rwxr-xr-x 1 root root 75588 Mar 1 2019 chgrp + -rwxr-xr-x 1 root root 63268 Mar 1 2019 chmod + -rwxr-xr-x 1 root root 75588 Mar 1 2019 chown + -rwxr-xr-x 1 root root 153732 Mar 1 2019 cp + -rwxr-xr-x 1 root root 132820 Jan 18 2019 dash + -rwxr-xr-x 1 root root 120676 Mar 1 2019 date + -rwxr-xr-x 1 root root 92040 Mar 1 2019 dd + -rwxr-xr-x 1 root root 100620 Mar 1 2019 df + -rwxr-xr-x 1 root root 149736 Mar 1 2019 dir + -rwxr-xr-x 1 root root 79412 Jan 10 2019 dmesg + lrwxrwxrwx 1 root root 8 Sep 27 2018 dnsdomainname -> hostname + lrwxrwxrwx 1 root root 8 Sep 27 2018 domainname -> hostname + -rwxr-xr-x 1 root root 34532 Mar 1 2019 echo + -rwxr-xr-x 1 root root 28 Jan 8 2019 egrep + -rwxr-xr-x 1 root root 34532 Mar 1 2019 false + -rwxr-xr-x 1 root root 28 Jan 8 2019 fgrep + -rwxr-xr-x 1 root root 67700 Jan 10 2019 findmnt + -rwsr-xr-x 1 root root 30112 Apr 23 09:38 fusermount + -rwxr-xr-x 1 root root 206392 Jan 8 2019 grep + -rwxr-xr-x 2 root root 2345 Jan 6 2019 gunzip + -rwxr-xr-x 1 root root 6375 Jan 6 2019 gzexe + -rwxr-xr-x 1 root root 100952 Jan 6 2019 gzip + -rwxr-xr-x 1 root root 21916 Sep 27 2018 hostname + -rwxr-xr-x 1 root root 79752 Mar 1 2019 ln + -rwxr-xr-x 1 root root 55400 Jul 27 2018 login + -rwxr-xr-x 1 root root 149736 Mar 1 2019 ls + -rwxr-xr-x 1 root root 112032 Jan 10 2019 lsblk + -rwxr-xr-x 1 root root 87972 Mar 1 2019 mkdir + -rwxr-xr-x 1 root root 79748 Mar 1 2019 mknod + -rwxr-xr-x 1 root root 46916 Mar 1 2019 mktemp + -rwxr-xr-x 1 root root 42348 Jan 10 2019 more + -rwsr-xr-x 1 root root 50592 Jan 10 2019 mount + -rwxr-xr-x 1 root root 13724 Jan 10 2019 mountpoint + -rwxr-xr-x 1 root root 157832 Mar 1 2019 mv + lrwxrwxrwx 1 root root 8 Sep 27 2018 nisdomainname -> hostname + lrwxrwxrwx 1 root root 14 Feb 15 2019 pidof -> /sbin/killall5 + -rwxr-xr-x 1 root root 38660 Mar 1 2019 pwd + lrwxrwxrwx 1 root root 4 Apr 18 2019 rbash -> bash + -rwxr-xr-x 1 root root 46852 Mar 1 2019 readlink + -rwxr-xr-x 1 root root 75588 Mar 1 2019 rm + -rwxr-xr-x 1 root root 42756 Mar 1 2019 rmdir + -rwxr-xr-x 1 root root 22276 Jan 22 2019 run-parts + -rwxr-xr-x 1 root root 125036 Dec 23 2018 sed + lrwxrwxrwx 1 root root 4 Jun 17 00:41 sh -> bash + lrwxrwxrwx 1 root root 4 Jun 15 22:33 sh.distrib -> dash + -rwxr-xr-x 1 root root 34532 Mar 1 2019 sleep + -rwxr-xr-x 1 root root 79652 Mar 1 2019 stty + -rwsr-xr-x 1 root root 71072 Jan 10 2019 su + -rwxr-xr-x 1 root root 34564 Mar 1 2019 sync + -rwxr-xr-x 1 root root 504024 Apr 24 2019 tar + -rwxr-xr-x 1 root root 13860 Jan 22 2019 tempfile + -rwxr-xr-x 1 root root 104292 Mar 1 2019 touch + -rwxr-xr-x 1 root root 34532 Mar 1 2019 true + -rwxr-xr-x 1 root root 17768 Apr 23 09:38 ulockmgr_server + -rwsr-xr-x 1 root root 30108 Jan 10 2019 umount + -rwxr-xr-x 1 root root 34532 Mar 1 2019 uname + -rwxr-xr-x 2 root root 2345 Jan 6 2019 uncompress + -rwxr-xr-x 1 root root 149736 Mar 1 2019 vdir + -rwxr-xr-x 1 root root 34208 Jan 10 2019 wdctl + -rwxr-xr-x 1 root root 946 Jan 22 2019 which + lrwxrwxrwx 1 root root 8 Sep 27 2018 ypdomainname -> hostname + -rwxr-xr-x 1 root root 1983 Jan 6 2019 zcat + -rwxr-xr-x 1 root root 1677 Jan 6 2019 zcmp + -rwxr-xr-x 1 root root 5879 Jan 6 2019 zdiff + -rwxr-xr-x 1 root root 29 Jan 6 2019 zegrep + -rwxr-xr-x 1 root root 29 Jan 6 2019 zfgrep + -rwxr-xr-x 1 root root 2080 Jan 6 2019 zforce + -rwxr-xr-x 1 root root 7584 Jan 6 2019 zgrep + -rwxr-xr-x 1 root root 2205 Jan 6 2019 zless + -rwxr-xr-x 1 root root 1841 Jan 6 2019 zmore + -rwxr-xr-x 1 root root 4552 Jan 6 2019 znew +I: user script /srv/workspace/pbuilder/5110/tmp/hooks/D02_print_environment finished -> Attempting to satisfy build-dependencies -> Creating pbuilder-satisfydepends-dummy package Package: pbuilder-satisfydepends-dummy @@ -315,7 +349,7 @@ Get: 107 http://deb.debian.org/debian buster/main i386 golang-github-hashicorp-go-msgpack-dev all 0.0~git20150518.0.fa3f638-6 [42.7 kB] Get: 108 http://deb.debian.org/debian buster/main i386 golang-github-hashicorp-logutils-dev all 0.0~git20150609.0.0dc08b1-1 [8150 B] Get: 109 http://deb.debian.org/debian buster/main i386 golang-github-hashicorp-raft-dev all 1.0.0+git20180823.82694fb-3 [85.9 kB] -Fetched 90.7 MB in 1s (74.0 MB/s) +Fetched 90.7 MB in 16s (5517 kB/s) debconf: delaying package configuration, since apt-utils is not installed Selecting previously unselected package libbsd0:i386. (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 19234 files and directories currently installed.) @@ -774,7 +808,7 @@ fakeroot is already the newest version (1.23-1). 0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded. I: Building the package -I: Running cd /build/golang-github-canonicalltd-raft-test-0.0~git20180628.c3345b5/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-buildpackage -us -uc -b +I: Running cd /build/golang-github-canonicalltd-raft-test-0.0~git20180628.c3345b5/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-buildpackage -us -uc -b dpkg-buildpackage: info: source package golang-github-canonicalltd-raft-test dpkg-buildpackage: info: source version 0.0~git20180628.c3345b5-1 dpkg-buildpackage: info: source distribution unstable @@ -792,114 +826,114 @@ dh_autoreconf -O--buildsystem=golang dh_auto_configure -O--buildsystem=golang dh_auto_build -O--buildsystem=golang - cd obj-i686-linux-gnu && go install -gcflags=all=\"-trimpath=/build/golang-github-canonicalltd-raft-test-0.0\~git20180628.c3345b5/obj-i686-linux-gnu/src\" -asmflags=all=\"-trimpath=/build/golang-github-canonicalltd-raft-test-0.0\~git20180628.c3345b5/obj-i686-linux-gnu/src\" -v -p 18 github.com/CanonicalLtd/raft-test github.com/CanonicalLtd/raft-test/internal/election github.com/CanonicalLtd/raft-test/internal/event github.com/CanonicalLtd/raft-test/internal/fsms github.com/CanonicalLtd/raft-test/internal/logging github.com/CanonicalLtd/raft-test/internal/network -internal/race + cd obj-i686-linux-gnu && go install -gcflags=all=\"-trimpath=/build/golang-github-canonicalltd-raft-test-0.0\~git20180628.c3345b5/obj-i686-linux-gnu/src\" -asmflags=all=\"-trimpath=/build/golang-github-canonicalltd-raft-test-0.0\~git20180628.c3345b5/obj-i686-linux-gnu/src\" -v -p 10 github.com/CanonicalLtd/raft-test github.com/CanonicalLtd/raft-test/internal/election github.com/CanonicalLtd/raft-test/internal/event github.com/CanonicalLtd/raft-test/internal/fsms github.com/CanonicalLtd/raft-test/internal/logging github.com/CanonicalLtd/raft-test/internal/network +unicode/utf8 errors -encoding -crypto/subtle -internal/nettrace -crypto/internal/subtle -unicode/utf16 -runtime/internal/sys math/bits -runtime/cgo -vendor/golang_org/x/crypto/cryptobyte/asn1 -github.com/CanonicalLtd/raft-test/internal/event +internal/race unicode -unicode/utf8 +internal/cpu runtime/internal/atomic +container/list +crypto/internal/subtle +runtime/internal/sys sync/atomic -internal/cpu +crypto/subtle +encoding +internal/nettrace vendor/golang_org/x/net/dns/dnsmessage -container/list +unicode/utf16 +vendor/golang_org/x/crypto/cryptobyte/asn1 +github.com/CanonicalLtd/raft-test/internal/event math internal/testlog -github.com/hashicorp/golang-lru/simplelru +runtime/cgo internal/bytealg +github.com/hashicorp/golang-lru/simplelru runtime strconv crypto/rc4 sync internal/singleflight -io math/rand -reflect +io syscall -hash -bytes -crypto/internal/randutil +reflect strings crypto/cipher -hash/crc32 -crypto/hmac +bytes +hash +crypto/internal/randutil crypto +crypto/hmac +hash/crc32 hash/crc64 -crypto/sha1 -crypto/md5 crypto/sha512 +crypto/md5 +crypto/sha1 crypto/sha256 -crypto/aes github.com/hashicorp/logutils vendor/golang_org/x/text/transform bufio path html +crypto/aes internal/syscall/unix time internal/poll -encoding/binary -sort os +sort +encoding/binary github.com/hashicorp/go-immutable-radix -encoding/base64 -crypto/des -vendor/golang_org/x/crypto/internal/chacha20 -vendor/golang_org/x/crypto/curve25519 -vendor/golang_org/x/crypto/poly1305 -vendor/golang_org/x/crypto/chacha20poly1305 -encoding/pem -runtime/debug os/signal path/filepath fmt +runtime/debug +encoding/base64 +crypto/des +vendor/golang_org/x/crypto/poly1305 +vendor/golang_org/x/crypto/curve25519 +vendor/golang_org/x/crypto/internal/chacha20 io/ioutil +encoding/pem +vendor/golang_org/x/crypto/chacha20poly1305 context -net/http/internal -encoding/hex log -github.com/pkg/errors +encoding/hex net/url -flag -mime vendor/golang_org/x/net/http2/hpack -text/template/parse -encoding/gob -encoding/json compress/flate -mime/quotedprintable +encoding/json +mime vendor/golang_org/x/text/unicode/norm math/big +mime/quotedprintable +net/http/internal vendor/golang_org/x/text/unicode/bidi -runtime/trace net -testing +text/template/parse +encoding/gob +github.com/pkg/errors +flag +runtime/trace vendor/golang_org/x/text/secure/bidirule compress/gzip +testing text/template -github.com/CanonicalLtd/raft-test/internal/logging vendor/golang_org/x/net/idna -html/template +github.com/CanonicalLtd/raft-test/internal/logging crypto/rand crypto/dsa crypto/elliptic encoding/asn1 crypto/rsa -crypto/ecdsa +html/template crypto/x509/pkix vendor/golang_org/x/crypto/cryptobyte +crypto/ecdsa vendor/golang_org/x/net/http/httpproxy -net/textproto crypto/x509 +net/textproto vendor/golang_org/x/net/http/httpguts mime/multipart crypto/tls @@ -909,1638 +943,1759 @@ github.com/armon/go-metrics github.com/hashicorp/go-msgpack/codec github.com/hashicorp/raft -github.com/CanonicalLtd/raft-test/internal/fsms github.com/CanonicalLtd/raft-test/internal/election +github.com/CanonicalLtd/raft-test/internal/fsms github.com/CanonicalLtd/raft-test/internal/network github.com/CanonicalLtd/raft-test dh_auto_test -O--buildsystem=golang - cd obj-i686-linux-gnu && go test -vet=off -v -p 18 github.com/CanonicalLtd/raft-test github.com/CanonicalLtd/raft-test/internal/election github.com/CanonicalLtd/raft-test/internal/event github.com/CanonicalLtd/raft-test/internal/fsms github.com/CanonicalLtd/raft-test/internal/logging github.com/CanonicalLtd/raft-test/internal/network + cd obj-i686-linux-gnu && go test -vet=off -v -p 10 github.com/CanonicalLtd/raft-test github.com/CanonicalLtd/raft-test/internal/election github.com/CanonicalLtd/raft-test/internal/event github.com/CanonicalLtd/raft-test/internal/fsms github.com/CanonicalLtd/raft-test/internal/logging github.com/CanonicalLtd/raft-test/internal/network === RUN TestCluster_Default --- PASS: TestCluster_Default (0.00s) - logger.go:44: 17:00:29.428933 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 17:00:29.429015 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 17:00:29.429058 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 17:00:29.429071 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 17:00:29.429081 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 17:00:29.429133 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:29.429149 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 17:00:29.429167 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:29.429178 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 17:00:29.429193 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:29.429201 [DEBUG] raft-test: setup: done - logger.go:44: 17:00:29.429222 [DEBUG] raft-test: close: start - logger.go:44: 17:00:29.429242 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:29.429223 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:29.429534 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:29.429570 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 17:00:29.429608 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 17:00:29.429622 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 17:00:29.429629 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 17:00:29.429654 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 17:00:29.429668 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 17:00:29.429679 [DEBUG] raft-test: close: done + logger.go:44: 10:44:56.067005 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 10:44:56.067214 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 10:44:56.067317 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 10:44:56.067355 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 10:44:56.067388 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 10:44:56.067570 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:56.067620 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 10:44:56.067680 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:56.067704 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 10:44:56.067761 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:56.067787 [DEBUG] raft-test: setup: done + logger.go:44: 10:44:56.067846 [DEBUG] raft-test: close: start + logger.go:44: 10:44:56.067864 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.067915 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.068266 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 10:44:56.068548 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 10:44:56.068716 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 10:44:56.068796 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 10:44:56.068860 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 10:44:56.068923 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 10:44:56.068972 [DEBUG] raft-test: close: done === RUN TestControl_ElectAndDepose ---- PASS: TestControl_ElectAndDepose (0.05s) - logger.go:44: 17:00:29.429885 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 17:00:29.429925 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 17:00:29.429945 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 17:00:29.429960 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 17:00:29.429970 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 17:00:29.430003 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 17:00:29.430022 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 17:00:29.430039 [DEBUG] raft-test: setup: done - logger.go:44: 17:00:29.430045 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 17:00:29.430053 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 17:00:29.430077 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 17:00:29.456443 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] - logger.go:44: 17:00:29.456454 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 17:00:29.456485 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 17:00:29.456495 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] - logger.go:44: 17:00:29.457730 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 17:00:29.457744 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 17:00:29.457749 [DEBUG] raft-test: elect: done - logger.go:44: 17:00:29.457761 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 17:00:29.457770 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 17:00:29.457841 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 17:00:29.458535 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 17:00:29.458549 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected - logger.go:44: 17:00:29.458720 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:29.458734 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:29.459409 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:29.459426 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:29.459993 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 17:00:29.460007 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 17:00:29.460199 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:29.460212 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:29.462096 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:29.462110 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:29.471324 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:29.471338 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:29.472077 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:29.472091 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:29.472412 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:29.472426 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:29.473660 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:29.473674 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:29.476889 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 17:00:29.476901 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 17:00:29.476910 [DEBUG] raft-test: close: start - logger.go:44: 17:00:29.476933 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:29.476948 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:29.476976 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:29.477000 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:29.477040 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 17:00:29.477073 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 17:00:29.477110 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 17:00:29.477135 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 17:00:29.477161 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 17:00:29.477177 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 17:00:29.477182 [DEBUG] raft-test: close: done +--- PASS: TestControl_ElectAndDepose (0.08s) + logger.go:44: 10:44:56.069287 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 10:44:56.069419 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 10:44:56.070254 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 10:44:56.070300 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 10:44:56.070335 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 10:44:56.070450 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 10:44:56.070511 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 10:44:56.070568 [DEBUG] raft-test: setup: done + logger.go:44: 10:44:56.070590 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 10:44:56.070625 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 10:44:56.070661 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 10:44:56.126758 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 10:44:56.126806 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=3,index=2] + logger.go:44: 10:44:56.126950 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 10:44:56.126971 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=3,index=2] + logger.go:44: 10:44:56.128280 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 10:44:56.128324 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 10:44:56.128345 [DEBUG] raft-test: elect: done + logger.go:44: 10:44:56.128414 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 10:44:56.128472 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 10:44:56.128833 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 10:44:56.128893 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 10:44:56.129204 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 10:44:56.129234 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected + logger.go:44: 10:44:56.129318 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.129360 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.129692 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.129724 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.130513 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.130540 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.131154 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.131186 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.142169 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.142272 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.142448 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.142489 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.142498 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.142538 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.143183 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.143212 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.147414 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 10:44:56.147444 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 10:44:56.147471 [DEBUG] raft-test: close: start + logger.go:44: 10:44:56.147538 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.147569 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.147631 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.147577 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.147776 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 10:44:56.147897 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 10:44:56.148195 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 10:44:56.148229 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 10:44:56.148255 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 10:44:56.148333 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 10:44:56.148404 [DEBUG] raft-test: close: done === RUN TestControl_DeposeAfterCommandEnqueued --- PASS: TestControl_DeposeAfterCommandEnqueued (0.04s) - logger.go:44: 17:00:29.477225 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 17:00:29.477266 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 17:00:29.477289 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 17:00:29.477301 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 17:00:29.477311 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 17:00:29.477336 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:29.477349 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 17:00:29.477366 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:29.477373 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 17:00:29.477378 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:29.477388 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:29.477390 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:29.477396 [DEBUG] raft-test: setup: done - logger.go:44: 17:00:29.477416 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 17:00:29.477423 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 17:00:29.477432 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 17:00:29.477409 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:29.497578 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 17:00:29.497599 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 17:00:29.497621 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:29.497628 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 17:00:29.497647 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 2 - logger.go:44: 17:00:29.497652 [INFO] raft: Election won. Tally: 2 - logger.go:44: 17:00:29.497658 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 17:00:29.497673 [INFO] raft: Added peer 1, starting replication - logger.go:44: 17:00:29.497680 [INFO] raft: Added peer 2, starting replication - logger.go:44: 17:00:29.497714 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] - logger.go:44: 17:00:29.497737 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 17:00:29.497753 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 17:00:29.497767 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 17:00:29.497778 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] - logger.go:44: 17:00:29.497810 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 17:00:29.498906 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 17:00:29.498915 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 17:00:29.498920 [DEBUG] raft-test: elect: done - logger.go:44: 17:00:29.498956 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] - logger.go:44: 17:00:29.498967 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] - logger.go:44: 17:00:29.498987 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 17:00:29.499017 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 17:00:29.499023 [DEBUG] raft-test: server 0: pipeline: append to: 2: enqueue fault: command 2 - logger.go:44: 17:00:29.499035 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=4] - logger.go:44: 17:00:29.499041 [DEBUG] raft-test: server 0: pipeline: append to: 1: enqueue fault: command 2 - logger.go:44: 17:00:29.499060 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 17:00:29.499075 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 17:00:29.499081 [DEBUG] raft-test: fsm 1: applied 1 - logger.go:44: 17:00:29.499083 [DEBUG] raft-test: fsm 2: applied 1 - logger.go:44: 17:00:29.500415 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:29.500425 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:29.500431 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 17:00:29.500540 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:29.500553 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:29.500560 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 17:00:29.512430 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:29.512444 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:29.512451 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 17:00:29.512509 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:29.512524 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:29.512531 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 17:00:29.518002 [WARN] raft: Failed to contact 1 in 18.91203ms - logger.go:44: 17:00:29.518019 [WARN] raft: Failed to contact 2 in 18.9287ms - logger.go:44: 17:00:29.518025 [WARN] raft: Failed to contact quorum of nodes, stepping down - logger.go:44: 17:00:29.518043 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:29.518052 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 17:00:29.518078 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 17:00:29.518088 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 17:00:29.518096 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 17:00:29.518065 [DEBUG] raft-test: close: start - logger.go:44: 17:00:29.518125 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 17:00:29.518131 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 17:00:29.518147 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 17:00:29.518160 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 17:00:29.518188 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 17:00:29.518198 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 17:00:29.518206 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 17:00:29.518221 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 17:00:29.518233 [DEBUG] raft-test: close: done + logger.go:44: 10:44:56.148580 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 10:44:56.148711 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 10:44:56.148786 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 10:44:56.148852 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 10:44:56.148894 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 10:44:56.148987 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:56.149046 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 10:44:56.149158 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:56.149194 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 10:44:56.149257 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:56.149295 [DEBUG] raft-test: setup: done + logger.go:44: 10:44:56.149321 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 10:44:56.149348 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 10:44:56.149390 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 10:44:56.149157 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.149502 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.149577 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.167785 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 10:44:56.167865 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 10:44:56.167927 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.167953 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 10:44:56.168044 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 2 + logger.go:44: 10:44:56.168065 [INFO] raft: Election won. Tally: 2 + logger.go:44: 10:44:56.168091 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 10:44:56.168151 [INFO] raft: Added peer 1, starting replication + logger.go:44: 10:44:56.168186 [INFO] raft: Added peer 2, starting replication + logger.go:44: 10:44:56.168280 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 10:44:56.168345 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 10:44:56.168310 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] + logger.go:44: 10:44:56.168420 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] + logger.go:44: 10:44:56.168613 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 10:44:56.168617 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 10:44:56.169637 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 10:44:56.169666 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 10:44:56.169682 [DEBUG] raft-test: elect: done + logger.go:44: 10:44:56.169804 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] + logger.go:44: 10:44:56.169810 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] + logger.go:44: 10:44:56.169947 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 10:44:56.170050 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=4] + logger.go:44: 10:44:56.170074 [DEBUG] raft-test: server 0: pipeline: append to: 2: enqueue fault: command 2 + logger.go:44: 10:44:56.170132 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=4] + logger.go:44: 10:44:56.170154 [DEBUG] raft-test: fsm 2: applied 1 + logger.go:44: 10:44:56.170163 [DEBUG] raft-test: server 0: pipeline: append to: 1: enqueue fault: command 2 + logger.go:44: 10:44:56.170302 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 10:44:56.170348 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 10:44:56.170352 [DEBUG] raft-test: fsm 1: applied 1 + logger.go:44: 10:44:56.170969 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.170997 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.171019 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 10:44:56.171389 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.171416 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.171437 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 10:44:56.184226 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.184284 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.184305 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 10:44:56.184557 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.184585 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.184605 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 10:44:56.187020 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 10:44:56.187059 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 3 + logger.go:44: 10:44:56.187096 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.187118 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 1 + logger.go:44: 10:44:56.187151 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 10:44:56.187214 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 10:44:56.188741 [WARN] raft: Failed to contact 1 in 18.353206ms + logger.go:44: 10:44:56.188770 [WARN] raft: Failed to contact 2 in 18.580726ms + logger.go:44: 10:44:56.188790 [WARN] raft: Failed to contact quorum of nodes, stepping down + logger.go:44: 10:44:56.188884 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.188914 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 10:44:56.188934 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 10:44:56.188968 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 10:44:56.189197 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 10:44:56.189002 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 10:44:56.189283 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 10:44:56.189081 [DEBUG] raft-test: close: start + logger.go:44: 10:44:56.189398 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 10:44:56.189469 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 10:44:56.189549 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 10:44:56.189570 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 10:44:56.189590 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 10:44:56.189634 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 10:44:56.189682 [DEBUG] raft-test: close: done === RUN TestControl_DeposeAfterCommandEnqueuedThenElect --- PASS: TestControl_DeposeAfterCommandEnqueuedThenElect (0.00s) === RUN TestControl_DeposeAfterCommandEnqueuedThenElectSame ---- PASS: TestControl_DeposeAfterCommandEnqueuedThenElectSame (0.19s) - logger.go:44: 17:00:29.518311 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 17:00:29.518353 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 17:00:29.518370 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 17:00:29.518382 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 17:00:29.518392 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 17:00:29.518418 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 17:00:29.518436 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 17:00:29.518453 [DEBUG] raft-test: setup: done - logger.go:44: 17:00:29.518460 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 17:00:29.518466 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 17:00:29.518475 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 17:00:29.639070 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 17:00:29.639101 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=7,index=2] - logger.go:44: 17:00:29.639105 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 17:00:29.639110 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=7,index=2] - logger.go:44: 17:00:29.640245 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 17:00:29.640264 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 17:00:29.640269 [DEBUG] raft-test: elect: done - logger.go:44: 17:00:29.640314 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=7,index=3] - logger.go:44: 17:00:29.640338 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=7,index=3] - logger.go:44: 17:00:29.640383 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 17:00:29.640440 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=7,index=4] - logger.go:44: 17:00:29.640449 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=7,index=4] - logger.go:44: 17:00:29.640468 [DEBUG] raft-test: server 0: pipeline: append to: 1: enqueue fault: command 2 - logger.go:44: 17:00:29.640452 [DEBUG] raft-test: server 0: pipeline: append to: 2: enqueue fault: command 2 - logger.go:44: 17:00:29.640502 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 17:00:29.640520 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 17:00:29.640542 [DEBUG] raft-test: fsm 2: applied 1 - logger.go:44: 17:00:29.640565 [DEBUG] raft-test: fsm 1: applied 1 - logger.go:44: 17:00:29.641306 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:29.641315 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:29.641582 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:29.641595 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:29.653219 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:29.653237 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:29.654810 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:29.654825 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:29.659343 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 17:00:29.659355 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 17:00:29.659385 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 17:00:29.659399 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 17:00:29.659415 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 17:00:29.708988 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=10,index=5] - logger.go:44: 17:00:29.708994 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 17:00:29.709036 [DEBUG] raft-test: server 0: transport: append to 2: 2 entries [Command:term=7,index=4 Noop:term=10,index=5] - logger.go:44: 17:00:29.709021 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=10,index=5] - logger.go:44: 17:00:29.709082 [DEBUG] raft-test: fsm 0: applied 2 - logger.go:44: 17:00:29.709043 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 17:00:29.709103 [DEBUG] raft-test: server 0: transport: append to 1: 2 entries [Command:term=7,index=4 Noop:term=10,index=5] - logger.go:44: 17:00:29.709104 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 17:00:29.709138 [DEBUG] raft-test: fsm 1: applied 2 - logger.go:44: 17:00:29.709141 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 17:00:29.709155 [DEBUG] raft-test: elect: done - logger.go:44: 17:00:29.709185 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=10,index=6] - logger.go:44: 17:00:29.709210 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=10,index=6] - logger.go:44: 17:00:29.709227 [DEBUG] raft-test: fsm 2: applied 2 - logger.go:44: 17:00:29.709239 [DEBUG] raft-test: fsm 0: applied 3 - logger.go:44: 17:00:29.709263 [DEBUG] raft-test: close: start - logger.go:44: 17:00:29.709299 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 17:00:29.709320 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 17:00:29.709354 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 17:00:29.709367 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 17:00:29.709375 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 17:00:29.709391 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 17:00:29.709403 [DEBUG] raft-test: close: done +--- PASS: TestControl_DeposeAfterCommandEnqueuedThenElectSame (0.28s) + logger.go:44: 10:44:56.189882 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 10:44:56.189996 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 10:44:56.190043 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 10:44:56.190079 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 10:44:56.190112 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 10:44:56.190185 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 10:44:56.190242 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 10:44:56.190315 [DEBUG] raft-test: setup: done + logger.go:44: 10:44:56.190336 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 10:44:56.190359 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 10:44:56.190383 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 10:44:56.257576 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 10:44:56.257656 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 10:44:56.257708 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=4,index=2] + logger.go:44: 10:44:56.257584 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=4,index=2] + logger.go:44: 10:44:56.258875 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 10:44:56.258904 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 10:44:56.258920 [DEBUG] raft-test: elect: done + logger.go:44: 10:44:56.259018 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=4,index=3] + logger.go:44: 10:44:56.259093 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=4,index=3] + logger.go:44: 10:44:56.259148 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 10:44:56.259237 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=4,index=4] + logger.go:44: 10:44:56.259260 [DEBUG] raft-test: server 0: pipeline: append to: 2: enqueue fault: command 2 + logger.go:44: 10:44:56.259276 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=4,index=4] + logger.go:44: 10:44:56.259316 [DEBUG] raft-test: server 0: pipeline: append to: 1: enqueue fault: command 2 + logger.go:44: 10:44:56.259328 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.259364 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 10:44:56.259420 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 10:44:56.259350 [DEBUG] raft-test: fsm 2: applied 1 + logger.go:44: 10:44:56.259394 [DEBUG] raft-test: fsm 1: applied 1 + logger.go:44: 10:44:56.259858 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.259888 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.261563 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.261634 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.272766 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.272808 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.274024 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.274062 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.278110 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 10:44:56.278153 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 10:44:56.278229 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 10:44:56.278269 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 10:44:56.278312 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 10:44:56.464894 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 10:44:56.464937 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 10:44:56.465002 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=12,index=5] + logger.go:44: 10:44:56.465127 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=12,index=5] + logger.go:44: 10:44:56.465203 [DEBUG] raft-test: server 0: transport: append to 2: 2 entries [Command:term=4,index=4 Noop:term=12,index=5] + logger.go:44: 10:44:56.465347 [DEBUG] raft-test: server 0: transport: append to 1: 2 entries [Command:term=4,index=4 Noop:term=12,index=5] + logger.go:44: 10:44:56.465351 [DEBUG] raft-test: fsm 0: applied 2 + logger.go:44: 10:44:56.466199 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 10:44:56.466223 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 10:44:56.466259 [DEBUG] raft-test: elect: done + logger.go:44: 10:44:56.466411 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=12,index=6] + logger.go:44: 10:44:56.466515 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=12,index=6] + logger.go:44: 10:44:56.466521 [DEBUG] raft-test: fsm 0: applied 3 + logger.go:44: 10:44:56.466670 [DEBUG] raft-test: close: start + logger.go:44: 10:44:56.466535 [DEBUG] raft-test: fsm 1: applied 2 + logger.go:44: 10:44:56.466784 [DEBUG] raft-test: fsm 2: applied 2 + logger.go:44: 10:44:56.466803 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 10:44:56.466883 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 10:44:56.467003 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 10:44:56.467034 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 10:44:56.467066 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 10:44:56.467138 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 10:44:56.467200 [DEBUG] raft-test: close: done === RUN TestControl_DeposeAfterCommandAppended ---- PASS: TestControl_DeposeAfterCommandAppended (0.11s) - logger.go:44: 17:00:29.709458 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 17:00:29.709508 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 17:00:29.709533 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 17:00:29.709546 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 17:00:29.709558 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 17:00:29.709596 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 17:00:29.709623 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 17:00:29.709657 [DEBUG] raft-test: setup: done - logger.go:44: 17:00:29.709668 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 17:00:29.709686 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 17:00:29.709700 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 17:00:29.809986 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=5,index=2] - logger.go:44: 17:00:29.810020 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=5,index=2] - logger.go:44: 17:00:29.809991 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 17:00:29.810051 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 17:00:29.810066 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 17:00:29.810072 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 17:00:29.810076 [DEBUG] raft-test: elect: done - logger.go:44: 17:00:29.810101 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=5,index=3] - logger.go:44: 17:00:29.810108 [DEBUG] raft-test: server 0: pipeline: append to 2: append fault: command 1 - logger.go:44: 17:00:29.810123 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=5,index=3] - logger.go:44: 17:00:29.810129 [DEBUG] raft-test: server 0: pipeline: append to 1: append fault: command 1 - logger.go:44: 17:00:29.810159 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 17:00:29.810179 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 17:00:29.811393 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 17:00:29.811424 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 17:00:29.811717 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 17:00:29.811732 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected - logger.go:44: 17:00:29.812728 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:29.812788 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:29.812933 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=5,index=3] - logger.go:44: 17:00:29.812944 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:29.813182 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:29.813197 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:29.813397 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=5,index=3] - logger.go:44: 17:00:29.813407 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:29.820176 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 17:00:29.820189 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 17:00:29.820206 [DEBUG] raft-test: close: start - logger.go:44: 17:00:29.820225 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=5,index=3] - logger.go:44: 17:00:29.820233 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:29.820234 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=5,index=3] - logger.go:44: 17:00:29.820263 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:29.820278 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 17:00:29.820292 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 17:00:29.820304 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 17:00:29.820310 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 17:00:29.820321 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 17:00:29.820334 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 17:00:29.820343 [DEBUG] raft-test: close: done +--- PASS: TestControl_DeposeAfterCommandAppended (0.04s) + logger.go:44: 10:44:56.467387 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 10:44:56.467512 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 10:44:56.467594 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 10:44:56.467638 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 10:44:56.467681 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 10:44:56.467798 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 10:44:56.467895 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 10:44:56.467977 [DEBUG] raft-test: setup: done + logger.go:44: 10:44:56.468004 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 10:44:56.468031 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 10:44:56.468063 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 10:44:56.492318 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 10:44:56.492356 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] + logger.go:44: 10:44:56.492409 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] + logger.go:44: 10:44:56.492386 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 10:44:56.492562 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 10:44:56.492581 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 10:44:56.492596 [DEBUG] raft-test: elect: done + logger.go:44: 10:44:56.492670 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] + logger.go:44: 10:44:56.492690 [DEBUG] raft-test: server 0: pipeline: append to 2: append fault: command 1 + logger.go:44: 10:44:56.492737 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] + logger.go:44: 10:44:56.492768 [DEBUG] raft-test: server 0: pipeline: append to 1: append fault: command 1 + logger.go:44: 10:44:56.492847 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 10:44:56.492889 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 10:44:56.494283 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.494316 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.494706 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 10:44:56.494731 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected + logger.go:44: 10:44:56.494777 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.494834 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.494955 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 10:44:56.494998 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 10:44:56.496554 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=2,index=3] + logger.go:44: 10:44:56.496589 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.496623 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=3] + logger.go:44: 10:44:56.496654 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.502728 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 10:44:56.502783 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 10:44:56.502828 [DEBUG] raft-test: close: start + logger.go:44: 10:44:56.503011 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=2,index=3] + logger.go:44: 10:44:56.503018 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=3] + logger.go:44: 10:44:56.503074 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.503038 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.503259 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 10:44:56.503358 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 10:44:56.503479 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 10:44:56.503536 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 10:44:56.503605 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 10:44:56.503668 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 10:44:56.503710 [DEBUG] raft-test: close: done === RUN TestControl_DeposeAfterCommandAppendedThenElect === RUN TestControl_DeposeAfterCommandAppendedThenElect/0 === RUN TestControl_DeposeAfterCommandAppendedThenElect/1 === RUN TestControl_DeposeAfterCommandAppendedThenElect/2 ---- PASS: TestControl_DeposeAfterCommandAppendedThenElect (0.23s) - --- PASS: TestControl_DeposeAfterCommandAppendedThenElect/0 (0.11s) - logger.go:44: 17:00:29.820417 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 17:00:29.820469 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 17:00:29.820489 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 17:00:29.820512 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 17:00:29.820531 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 17:00:29.820566 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:29.820580 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 17:00:29.820599 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:29.820607 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 17:00:29.820624 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:29.820632 [DEBUG] raft-test: setup: done - logger.go:44: 17:00:29.820638 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 17:00:29.820645 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 17:00:29.820601 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:29.820655 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 17:00:29.820671 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:29.820671 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:29.839037 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 17:00:29.839061 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 2 - logger.go:44: 17:00:29.839086 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:29.839099 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 1 - logger.go:44: 17:00:29.839116 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 17:00:29.839116 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 17:00:29.846663 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 17:00:29.846683 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 2 - logger.go:44: 17:00:29.846700 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:29.846708 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 1 - logger.go:44: 17:00:29.846719 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 17:00:29.846723 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 17:00:29.849009 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 17:00:29.849025 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 17:00:29.849038 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:29.849045 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 17:00:29.849058 [INFO] raft: Duplicate RequestVote for same term: 2 - logger.go:44: 17:00:29.849063 [INFO] raft: Duplicate RequestVote for same term: 2 - logger.go:44: 17:00:29.855591 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 17:00:29.855620 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 - logger.go:44: 17:00:29.855640 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:29.855648 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 - logger.go:44: 17:00:29.855687 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 17:00:29.855705 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 17:00:29.864661 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 17:00:29.864681 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 3 - logger.go:44: 17:00:29.864709 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:29.864721 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 1 - logger.go:44: 17:00:29.864733 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:29.864743 [INFO] raft: Duplicate RequestVote for same term: 3 - logger.go:44: 17:00:29.864751 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 2 - logger.go:44: 17:00:29.864766 [INFO] raft: Election won. Tally: 2 - logger.go:44: 17:00:29.864773 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 17:00:29.864790 [INFO] raft: Added peer 1, starting replication - logger.go:44: 17:00:29.864798 [INFO] raft: Added peer 2, starting replication - logger.go:44: 17:00:29.864811 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 17:00:29.864835 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 17:00:29.864845 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=3,index=2] - logger.go:44: 17:00:29.864846 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=3,index=2] - logger.go:44: 17:00:29.864876 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 17:00:29.864912 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "0") - logger.go:44: 17:00:29.864941 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 17:00:29.865962 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 17:00:29.865973 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 17:00:29.865977 [DEBUG] raft-test: elect: done - logger.go:44: 17:00:29.866012 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=3,index=3] - logger.go:44: 17:00:29.866017 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=3,index=3] - logger.go:44: 17:00:29.866050 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 17:00:29.866086 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=3,index=4] - logger.go:44: 17:00:29.866094 [DEBUG] raft-test: server 0: pipeline: append to 2: append fault: command 2 - logger.go:44: 17:00:29.866101 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=3,index=4] - logger.go:44: 17:00:29.866120 [DEBUG] raft-test: server 0: pipeline: append to 1: append fault: command 2 - logger.go:44: 17:00:29.866125 [DEBUG] raft-test: fsm 2: applied 1 - logger.go:44: 17:00:29.866191 [DEBUG] raft-test: fsm 1: applied 1 - logger.go:44: 17:00:29.866191 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 17:00:29.866229 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 17:00:29.867417 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 17:00:29.867428 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 17:00:29.867450 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 17:00:29.867465 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 17:00:29.867453 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:29.867499 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:29.867507 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 17:00:29.867483 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:29.867531 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:29.867542 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 17:00:29.867533 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 17:00:29.867567 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected - logger.go:44: 17:00:29.867575 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 17:00:29.867587 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 17:00:29.868992 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=3,index=4] - logger.go:44: 17:00:29.869025 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:29.869032 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=3,index=4] - logger.go:44: 17:00:29.869057 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:29.869068 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 17:00:29.869034 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 17:00:29.880519 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:29.880536 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:29.880544 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 17:00:29.880539 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:29.880564 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:29.880570 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 17:00:29.880835 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=3,index=4] - logger.go:44: 17:00:29.880850 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:29.880859 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 17:00:29.881134 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=3,index=4] - logger.go:44: 17:00:29.881143 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:29.881151 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 17:00:29.883978 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 17:00:29.883996 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 4 - logger.go:44: 17:00:29.884016 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:29.884024 [DEBUG] raft: Vote granted from 2 in term 4. Tally: 1 - logger.go:44: 17:00:29.884034 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 17:00:29.884035 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 17:00:29.885132 [WARN] raft: Failed to contact 1 in 19.080128ms - logger.go:44: 17:00:29.885148 [WARN] raft: Failed to contact 2 in 19.084818ms - logger.go:44: 17:00:29.885154 [WARN] raft: Failed to contact quorum of nodes, stepping down - logger.go:44: 17:00:29.885172 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:29.885182 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 17:00:29.885209 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 17:00:29.885197 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 17:00:29.885227 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 17:00:29.885237 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 17:00:29.888614 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 17:00:29.888642 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 4 - logger.go:44: 17:00:29.888654 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:29.888665 [DEBUG] raft: Vote granted from 1 in term 4. Tally: 1 - logger.go:44: 17:00:29.888674 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 17:00:29.888681 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 17:00:29.892396 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=3,index=4] - logger.go:44: 17:00:29.892430 [DEBUG] raft-test: server 0: transport: append to 2: newer term - logger.go:44: 17:00:29.892440 [ERR] raft: peer {Voter 2 2} has newer term, stopping replication - logger.go:44: 17:00:29.892797 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=3,index=4] - logger.go:44: 17:00:29.892831 [DEBUG] raft-test: server 0: transport: append to 1: newer term - logger.go:44: 17:00:29.892846 [ERR] raft: peer {Voter 1 1} has newer term, stopping replication - logger.go:44: 17:00:29.906739 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 17:00:29.906757 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 5 - logger.go:44: 17:00:29.906776 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:29.906784 [DEBUG] raft: Vote granted from 1 in term 5. Tally: 1 - logger.go:44: 17:00:29.906785 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 17:00:29.906814 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 17:00:29.911245 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 17:00:29.911261 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 4 - logger.go:44: 17:00:29.911279 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:29.911294 [DEBUG] raft: Vote granted from 0 in term 4. Tally: 1 - logger.go:44: 17:00:29.911300 [DEBUG] raft: Newer term discovered, fallback to follower - logger.go:44: 17:00:29.911308 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:29.911321 [INFO] raft: Duplicate RequestVote for same term: 4 - logger.go:44: 17:00:29.914020 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 17:00:29.914040 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 5 - logger.go:44: 17:00:29.914062 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:29.914070 [DEBUG] raft: Vote granted from 2 in term 5. Tally: 1 - logger.go:44: 17:00:29.914081 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 17:00:29.914081 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 17:00:29.922313 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 17:00:29.922351 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 6 - logger.go:44: 17:00:29.922391 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:29.922403 [DEBUG] raft: Vote granted from 1 in term 6. Tally: 1 - logger.go:44: 17:00:29.922407 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 17:00:29.922415 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 17:00:29.933267 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 17:00:29.933302 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 6 - logger.go:44: 17:00:29.933321 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:29.933329 [DEBUG] raft: Vote granted from 0 in term 6. Tally: 1 - logger.go:44: 17:00:29.933358 [INFO] raft: Duplicate RequestVote for same term: 6 - logger.go:44: 17:00:29.933361 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:29.933367 [DEBUG] raft: Vote granted from 2 in term 6. Tally: 2 - logger.go:44: 17:00:29.933392 [INFO] raft: Election won. Tally: 2 - logger.go:44: 17:00:29.933418 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 17:00:29.933427 [INFO] raft: Added peer 1, starting replication - logger.go:44: 17:00:29.933445 [INFO] raft: Added peer 2, starting replication - logger.go:44: 17:00:29.933482 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=6,index=5] - logger.go:44: 17:00:29.933504 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 17:00:29.933519 [DEBUG] raft-test: fsm 0: applied 2 - logger.go:44: 17:00:29.933526 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 17:00:29.933544 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 17:00:29.933535 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=6,index=5] - logger.go:44: 17:00:29.933601 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "0") - logger.go:44: 17:00:29.933622 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 17:00:29.934745 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 17:00:29.934759 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 17:00:29.934764 [DEBUG] raft-test: elect: done - logger.go:44: 17:00:29.934787 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=6,index=6] - logger.go:44: 17:00:29.934808 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=6,index=6] - logger.go:44: 17:00:29.934819 [DEBUG] raft-test: fsm 1: applied 2 - logger.go:44: 17:00:29.934817 [DEBUG] raft-test: fsm 0: applied 3 - logger.go:44: 17:00:29.934849 [DEBUG] raft-test: close: start - logger.go:44: 17:00:29.934854 [DEBUG] raft-test: fsm 2: applied 2 - logger.go:44: 17:00:29.934870 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 17:00:29.934877 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 17:00:29.934907 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 17:00:29.934924 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 17:00:29.934935 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 17:00:29.934946 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 17:00:29.934963 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 17:00:29.934976 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 17:00:29.934984 [DEBUG] raft-test: close: done - --- PASS: TestControl_DeposeAfterCommandAppendedThenElect/1 (0.05s) - logger.go:44: 17:00:29.935618 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 17:00:29.935658 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 17:00:29.935675 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 17:00:29.935687 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 17:00:29.935698 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 17:00:29.935724 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:29.935733 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 17:00:29.935766 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:29.935774 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 17:00:29.935802 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:29.935807 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:29.935833 [DEBUG] raft-test: setup: done - logger.go:44: 17:00:29.935846 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 17:00:29.935858 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 17:00:29.935873 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 17:00:29.935811 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:29.935894 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:29.957891 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 17:00:29.957925 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 2 - logger.go:44: 17:00:29.957960 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:29.957991 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 1 - logger.go:44: 17:00:29.958004 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 17:00:29.958029 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 17:00:29.960369 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 17:00:29.960388 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 17:00:29.960413 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:29.960425 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 17:00:29.960443 [INFO] raft: Duplicate RequestVote for same term: 2 - logger.go:44: 17:00:29.960456 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 2 - logger.go:44: 17:00:29.960464 [INFO] raft: Election won. Tally: 2 - logger.go:44: 17:00:29.960471 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 17:00:29.960479 [INFO] raft: Added peer 1, starting replication - logger.go:44: 17:00:29.960488 [INFO] raft: Added peer 2, starting replication - logger.go:44: 17:00:29.960514 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 17:00:29.960533 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] - logger.go:44: 17:00:29.960548 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] - logger.go:44: 17:00:29.960560 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 17:00:29.960568 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "0") - logger.go:44: 17:00:29.960535 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 17:00:29.960584 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 17:00:29.960620 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 17:00:29.960633 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 17:00:29.960639 [DEBUG] raft-test: elect: done - logger.go:44: 17:00:29.960667 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] - logger.go:44: 17:00:29.960684 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] - logger.go:44: 17:00:29.960727 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 17:00:29.960777 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 17:00:29.960789 [DEBUG] raft-test: server 0: pipeline: append to 2: append fault: command 2 - logger.go:44: 17:00:29.960791 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=4] - logger.go:44: 17:00:29.960850 [DEBUG] raft-test: server 0: pipeline: append to 1: append fault: command 2 - logger.go:44: 17:00:29.960882 [DEBUG] raft-test: fsm 2: applied 1 - logger.go:44: 17:00:29.960890 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 17:00:29.960928 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 17:00:29.960892 [DEBUG] raft-test: fsm 1: applied 1 - logger.go:44: 17:00:29.962503 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 17:00:29.962519 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 17:00:29.962539 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 17:00:29.962551 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 17:00:29.962884 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 17:00:29.962894 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected - logger.go:44: 17:00:29.962903 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 17:00:29.962911 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 17:00:29.962930 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:29.962937 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:29.962947 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 17:00:29.963082 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:29.963096 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:29.963103 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 17:00:29.964156 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=2,index=4] - logger.go:44: 17:00:29.964166 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:29.964174 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 17:00:29.964211 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 17:00:29.964228 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:29.964238 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 17:00:29.975470 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:29.975486 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:29.975493 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 17:00:29.975784 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=2,index=4] - logger.go:44: 17:00:29.975798 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:29.975806 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 17:00:29.975961 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:29.975976 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:29.975983 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 17:00:29.976426 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 17:00:29.976455 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:29.976476 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 17:00:29.980923 [WARN] raft: Failed to contact 1 in 20.203925ms - logger.go:44: 17:00:29.980941 [WARN] raft: Failed to contact 2 in 20.187119ms - logger.go:44: 17:00:29.980947 [WARN] raft: Failed to contact quorum of nodes, stepping down - logger.go:44: 17:00:29.980963 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:29.980973 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 17:00:29.980980 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 17:00:29.980929 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 17:00:29.980992 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 - logger.go:44: 17:00:29.981001 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:29.981008 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 - logger.go:44: 17:00:29.981021 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 17:00:29.981033 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 17:00:29.981070 [DEBUG] raft-test: elect: start (server 1) - logger.go:44: 17:00:29.981082 [DEBUG] raft-test: server 1: establish outbound connection to all other nodes - logger.go:44: 17:00:29.981092 [DEBUG] raft-test: elect: server 1: wait to become leader within 1.5s - logger.go:44: 17:00:29.987377 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:29.987392 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:29.987399 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 17:00:29.987947 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:29.987959 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 17:00:29.987978 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:29.987987 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 17:00:29.987962 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:29.988003 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 17:00:29.987966 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=2,index=4] - logger.go:44: 17:00:29.988016 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:29.988022 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 17:00:29.988796 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 17:00:29.988812 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 3 - logger.go:44: 17:00:29.988827 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:29.988834 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 1 - logger.go:44: 17:00:29.988847 [INFO] raft: Duplicate RequestVote for same term: 3 - logger.go:44: 17:00:29.988853 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 2 - logger.go:44: 17:00:29.988872 [INFO] raft: Election won. Tally: 2 - logger.go:44: 17:00:29.988879 [INFO] raft: Node at 1 [Leader] entering Leader state - logger.go:44: 17:00:29.988887 [INFO] raft: Added peer 0, starting replication - logger.go:44: 17:00:29.988894 [INFO] raft: Added peer 2, starting replication - logger.go:44: 17:00:29.988918 [DEBUG] raft-test: server 1: leadership: acquired - logger.go:44: 17:00:29.988948 [DEBUG] raft-test: elect: server 1: wait for other servers to become followers within 40ms - logger.go:44: 17:00:29.988953 [DEBUG] raft-test: server 1: transport: append to 0: 1 entry [Noop:term=3,index=5] - logger.go:44: 17:00:29.989009 [INFO] raft: pipelining replication to peer {Voter 0 0} - logger.go:44: 17:00:29.988922 [DEBUG] raft-test: server 1: transport: append to 2: 1 entry [Noop:term=3,index=5] - logger.go:44: 17:00:29.989028 [DEBUG] raft-test: fsm 1: applied 2 - logger.go:44: 17:00:29.989035 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "1") - logger.go:44: 17:00:29.989042 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 17:00:29.990126 [DEBUG] raft-test: elect: server 0: became follower - logger.go:44: 17:00:29.990141 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 17:00:29.990146 [DEBUG] raft-test: elect: done - logger.go:44: 17:00:29.990170 [DEBUG] raft-test: server 1: pipeline: append to 2: 1 entry [Command:term=3,index=6] - logger.go:44: 17:00:29.990180 [DEBUG] raft-test: server 1: pipeline: append to 0: 1 entry [Command:term=3,index=6] - logger.go:44: 17:00:29.990203 [DEBUG] raft-test: fsm 2: applied 2 - logger.go:44: 17:00:29.990218 [DEBUG] raft-test: fsm 1: applied 3 - logger.go:44: 17:00:29.990228 [DEBUG] raft-test: fsm 0: applied 2 - logger.go:44: 17:00:29.990249 [DEBUG] raft-test: close: start - logger.go:44: 17:00:29.990277 [INFO] raft: aborting pipeline replication to peer {Voter 0 0} - logger.go:44: 17:00:29.990292 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 17:00:29.990309 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 17:00:29.990325 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 17:00:29.990337 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 17:00:29.990343 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 17:00:29.990350 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 17:00:29.990355 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 17:00:29.990367 [DEBUG] raft-test: close: done - --- PASS: TestControl_DeposeAfterCommandAppendedThenElect/2 (0.06s) - logger.go:44: 17:00:29.990447 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 17:00:29.990506 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 17:00:29.990524 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 17:00:29.990536 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 17:00:29.990547 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 17:00:29.990572 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:29.990587 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 17:00:29.990605 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:29.990618 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 17:00:29.990610 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:29.990649 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:29.990658 [DEBUG] raft-test: setup: done - logger.go:44: 17:00:29.990665 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 17:00:29.990677 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 17:00:29.990684 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:29.990687 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 17:00:29.990681 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:30.011171 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 17:00:30.011190 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 2 - logger.go:44: 17:00:30.011207 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:30.011230 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 1 - logger.go:44: 17:00:30.011231 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 17:00:30.011240 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 17:00:30.015968 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 17:00:30.015983 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 17:00:30.016001 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:30.016008 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 17:00:30.016020 [INFO] raft: Duplicate RequestVote for same term: 2 - logger.go:44: 17:00:30.016041 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 2 - logger.go:44: 17:00:30.016056 [INFO] raft: Election won. Tally: 2 - logger.go:44: 17:00:30.016068 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 17:00:30.016081 [INFO] raft: Added peer 1, starting replication - logger.go:44: 17:00:30.016089 [INFO] raft: Added peer 2, starting replication - logger.go:44: 17:00:30.016130 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] - logger.go:44: 17:00:30.016156 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "0") - logger.go:44: 17:00:30.016174 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 17:00:30.016182 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] - logger.go:44: 17:00:30.016174 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 17:00:30.016230 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 17:00:30.016250 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 17:00:30.016213 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 17:00:30.016259 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 17:00:30.016293 [DEBUG] raft-test: elect: done - logger.go:44: 17:00:30.016346 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] - logger.go:44: 17:00:30.016363 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] - logger.go:44: 17:00:30.016418 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 17:00:30.016465 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 17:00:30.016477 [DEBUG] raft-test: server 0: pipeline: append to 2: append fault: command 2 - logger.go:44: 17:00:30.016471 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=4] - logger.go:44: 17:00:30.016509 [DEBUG] raft-test: server 0: pipeline: append to 1: append fault: command 2 - logger.go:44: 17:00:30.016543 [DEBUG] raft-test: fsm 2: applied 1 - logger.go:44: 17:00:30.016548 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 17:00:30.016601 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 17:00:30.016558 [DEBUG] raft-test: fsm 1: applied 1 - logger.go:44: 17:00:30.017730 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 17:00:30.017748 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected - logger.go:44: 17:00:30.017757 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 17:00:30.017777 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 17:00:30.018324 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 17:00:30.018338 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 17:00:30.018346 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 17:00:30.018359 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 17:00:30.018390 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:30.018408 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:30.018420 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 17:00:30.019233 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:30.019246 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:30.019253 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 17:00:30.019839 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 17:00:30.019865 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:30.019874 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 17:00:30.020290 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=2,index=4] - logger.go:44: 17:00:30.020304 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:30.020313 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 17:00:30.031213 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:30.031228 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:30.031237 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 17:00:30.031682 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=2,index=4] - logger.go:44: 17:00:30.031707 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:30.031720 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 17:00:30.031688 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 17:00:30.031741 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:30.031747 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 17:00:30.031786 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:30.031804 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:30.031812 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 17:00:30.036502 [WARN] raft: Failed to contact 1 in 20.030397ms - logger.go:44: 17:00:30.036540 [WARN] raft: Failed to contact 2 in 20.07806ms - logger.go:44: 17:00:30.036563 [WARN] raft: Failed to contact quorum of nodes, stepping down - logger.go:44: 17:00:30.036594 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 17:00:30.036627 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 17:00:30.036661 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 17:00:30.036644 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 3 - logger.go:44: 17:00:30.036734 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:30.036742 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 1 - logger.go:44: 17:00:30.036619 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:30.036771 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 17:00:30.036693 [DEBUG] raft-test: elect: start (server 2) - logger.go:44: 17:00:30.036806 [DEBUG] raft-test: server 2: establish outbound connection to all other nodes - logger.go:44: 17:00:30.036814 [DEBUG] raft-test: elect: server 2: wait to become leader within 1.5s - logger.go:44: 17:00:30.036759 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 17:00:30.043215 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:30.043232 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:30.043248 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 17:00:30.043845 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=2,index=4] - logger.go:44: 17:00:30.043858 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:30.043867 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 17:00:30.043904 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 17:00:30.043911 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:30.043918 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 17:00:30.044984 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 17:00:30.045000 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 - logger.go:44: 17:00:30.045017 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:30.045024 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 - logger.go:44: 17:00:30.045036 [INFO] raft: Duplicate RequestVote for same term: 3 - logger.go:44: 17:00:30.045052 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 2 - logger.go:44: 17:00:30.045064 [INFO] raft: Election won. Tally: 2 - logger.go:44: 17:00:30.045075 [INFO] raft: Node at 2 [Leader] entering Leader state - logger.go:44: 17:00:30.045085 [INFO] raft: Added peer 0, starting replication - logger.go:44: 17:00:30.045091 [INFO] raft: Added peer 1, starting replication - logger.go:44: 17:00:30.045101 [DEBUG] raft-test: server 2: leadership: acquired - logger.go:44: 17:00:30.045134 [DEBUG] raft-test: elect: server 2: wait for other servers to become followers within 40ms - logger.go:44: 17:00:30.045120 [DEBUG] raft-test: server 2: transport: append to 1: 1 entry [Noop:term=3,index=5] - logger.go:44: 17:00:30.045147 [DEBUG] raft-test: server 2: transport: append to 0: 1 entry [Noop:term=3,index=5] - logger.go:44: 17:00:30.045216 [DEBUG] raft-test: fsm 2: applied 2 - logger.go:44: 17:00:30.045197 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "2") - logger.go:44: 17:00:30.045209 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 17:00:30.045236 [INFO] raft: pipelining replication to peer {Voter 0 0} - logger.go:44: 17:00:30.046297 [DEBUG] raft-test: elect: server 0: became follower - logger.go:44: 17:00:30.046310 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 17:00:30.046316 [DEBUG] raft-test: elect: done - logger.go:44: 17:00:30.046348 [DEBUG] raft-test: server 2: pipeline: append to 0: 1 entry [Command:term=3,index=6] - logger.go:44: 17:00:30.046365 [DEBUG] raft-test: server 2: pipeline: append to 1: 1 entry [Command:term=3,index=6] - logger.go:44: 17:00:30.046377 [DEBUG] raft-test: fsm 0: applied 2 - logger.go:44: 17:00:30.046381 [DEBUG] raft-test: fsm 2: applied 3 - logger.go:44: 17:00:30.046399 [DEBUG] raft-test: fsm 1: applied 2 - logger.go:44: 17:00:30.046400 [DEBUG] raft-test: close: start - logger.go:44: 17:00:30.046438 [INFO] raft: aborting pipeline replication to peer {Voter 0 0} - logger.go:44: 17:00:30.046456 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 17:00:30.046474 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 17:00:30.046502 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 17:00:30.046522 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 17:00:30.046535 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 17:00:30.046541 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 17:00:30.046556 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 17:00:30.046561 [DEBUG] raft-test: close: done +--- PASS: TestControl_DeposeAfterCommandAppendedThenElect (0.20s) + --- PASS: TestControl_DeposeAfterCommandAppendedThenElect/0 (0.06s) + logger.go:44: 10:44:56.503929 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 10:44:56.504079 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 10:44:56.504159 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 10:44:56.504207 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 10:44:56.504249 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 10:44:56.504331 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:56.504388 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 10:44:56.504455 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:56.504478 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.504484 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 10:44:56.504550 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.504635 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:56.504704 [DEBUG] raft-test: setup: done + logger.go:44: 10:44:56.504735 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 10:44:56.504750 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.504772 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 10:44:56.504828 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 10:44:56.521995 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 10:44:56.522034 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 10:44:56.522086 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.522107 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 10:44:56.522160 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 2 + logger.go:44: 10:44:56.522177 [INFO] raft: Election won. Tally: 2 + logger.go:44: 10:44:56.522196 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 10:44:56.522223 [INFO] raft: Added peer 1, starting replication + logger.go:44: 10:44:56.522246 [INFO] raft: Added peer 2, starting replication + logger.go:44: 10:44:56.522349 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] + logger.go:44: 10:44:56.522409 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 10:44:56.522463 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 10:44:56.522500 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 10:44:56.522552 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] + logger.go:44: 10:44:56.522611 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 10:44:56.522668 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 10:44:56.522686 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 10:44:56.522701 [DEBUG] raft-test: elect: done + logger.go:44: 10:44:56.522764 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] + logger.go:44: 10:44:56.522846 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 10:44:56.522917 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=4] + logger.go:44: 10:44:56.522936 [DEBUG] raft-test: server 0: pipeline: append to 1: append fault: command 2 + logger.go:44: 10:44:56.522982 [DEBUG] raft-test: server 0: pipeline: append to 2: 2 entries [Command:term=2,index=3 Command:term=2,index=4] + logger.go:44: 10:44:56.523001 [DEBUG] raft-test: server 0: pipeline: append to 2: append fault: command 2 + logger.go:44: 10:44:56.523044 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 10:44:56.523144 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 10:44:56.523182 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 10:44:56.523204 [DEBUG] raft-test: fsm 1: applied 1 + logger.go:44: 10:44:56.523244 [DEBUG] raft-test: fsm 2: applied 1 + logger.go:44: 10:44:56.524428 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 10:44:56.524455 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected + logger.go:44: 10:44:56.524478 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 10:44:56.524502 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 10:44:56.525011 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 10:44:56.525080 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 10:44:56.525157 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 10:44:56.525204 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 10:44:56.525253 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.525312 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.525331 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 10:44:56.526011 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.526036 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.526055 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 10:44:56.526708 [DEBUG] raft-test: server 0: transport: append to 2: 2 entries [Command:term=2,index=3 Command:term=2,index=4] + logger.go:44: 10:44:56.526739 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.526739 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=2,index=4] + logger.go:44: 10:44:56.526861 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.526900 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 10:44:56.526762 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 10:44:56.532596 [WARN] raft: Failed to contact 2 in 10.162426ms + logger.go:44: 10:44:56.538253 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.538286 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.538332 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 10:44:56.538872 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.538901 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.538920 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 10:44:56.538956 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=2,index=4] + logger.go:44: 10:44:56.539025 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.539063 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 10:44:56.539283 [DEBUG] raft-test: server 0: transport: append to 2: 2 entries [Command:term=2,index=3 Command:term=2,index=4] + logger.go:44: 10:44:56.539313 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.539340 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 10:44:56.542864 [WARN] raft: Failed to contact 2 in 20.455995ms + logger.go:44: 10:44:56.542901 [WARN] raft: Failed to contact 1 in 20.019776ms + logger.go:44: 10:44:56.542924 [WARN] raft: Failed to contact quorum of nodes, stepping down + logger.go:44: 10:44:56.542994 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.543022 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 10:44:56.543044 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 10:44:56.543150 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 10:44:56.543261 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 10:44:56.543301 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 10:44:56.550336 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.550753 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=2,index=4] + logger.go:44: 10:44:56.551159 [DEBUG] raft-test: server 0: transport: append to 2: 2 entries [Command:term=2,index=3 Command:term=2,index=4] + logger.go:44: 10:44:56.560900 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 10:44:56.560943 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 3 + logger.go:44: 10:44:56.561015 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.561041 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 1 + logger.go:44: 10:44:56.561142 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 2 + logger.go:44: 10:44:56.561160 [INFO] raft: Election won. Tally: 2 + logger.go:44: 10:44:56.561180 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 10:44:56.561206 [INFO] raft: Added peer 1, starting replication + logger.go:44: 10:44:56.561286 [INFO] raft: Added peer 2, starting replication + logger.go:44: 10:44:56.561359 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 10:44:56.561410 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 10:44:56.561436 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=3,index=5] + logger.go:44: 10:44:56.561483 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=3,index=5] + logger.go:44: 10:44:56.561547 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 10:44:56.561567 [DEBUG] raft-test: fsm 0: applied 2 + logger.go:44: 10:44:56.561636 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 10:44:56.562657 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 10:44:56.562684 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 10:44:56.562699 [DEBUG] raft-test: elect: done + logger.go:44: 10:44:56.562773 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=3,index=6] + logger.go:44: 10:44:56.562810 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=3,index=6] + logger.go:44: 10:44:56.562888 [DEBUG] raft-test: fsm 2: applied 2 + logger.go:44: 10:44:56.562909 [DEBUG] raft-test: fsm 0: applied 3 + logger.go:44: 10:44:56.562958 [DEBUG] raft-test: close: start + logger.go:44: 10:44:56.563004 [DEBUG] raft-test: fsm 1: applied 2 + logger.go:44: 10:44:56.563026 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 10:44:56.563017 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 10:44:56.563123 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 10:44:56.563267 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 10:44:56.563317 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 10:44:56.563342 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 10:44:56.563388 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 10:44:56.563450 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 10:44:56.563505 [DEBUG] raft-test: close: done + --- PASS: TestControl_DeposeAfterCommandAppendedThenElect/1 (0.10s) + logger.go:44: 10:44:56.563741 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 10:44:56.563887 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 10:44:56.563945 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 10:44:56.563988 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 10:44:56.564028 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 10:44:56.564099 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:56.564157 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 10:44:56.564219 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:56.564236 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.564246 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 10:44:56.564304 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.564360 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:56.564416 [DEBUG] raft-test: setup: done + logger.go:44: 10:44:56.564444 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 10:44:56.564472 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 10:44:56.564504 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 10:44:56.564504 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.581874 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 10:44:56.581960 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 2 + logger.go:44: 10:44:56.582039 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.582074 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 1 + logger.go:44: 10:44:56.582081 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 10:44:56.582123 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 10:44:56.590794 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 10:44:56.590874 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 2 + logger.go:44: 10:44:56.590950 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.590994 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 1 + logger.go:44: 10:44:56.591002 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 10:44:56.591046 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 10:44:56.591060 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 10:44:56.591185 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 10:44:56.591249 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.591275 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 10:44:56.591335 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 10:44:56.591344 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 10:44:56.599208 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 10:44:56.599258 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 + logger.go:44: 10:44:56.599370 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.599398 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 + logger.go:44: 10:44:56.599426 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 10:44:56.599438 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 10:44:56.613041 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 10:44:56.613086 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 3 + logger.go:44: 10:44:56.613223 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.613261 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 1 + logger.go:44: 10:44:56.613325 [INFO] raft: Duplicate RequestVote for same term: 3 + logger.go:44: 10:44:56.613420 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.613559 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 2 + logger.go:44: 10:44:56.613598 [INFO] raft: Election won. Tally: 2 + logger.go:44: 10:44:56.613628 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 10:44:56.613688 [INFO] raft: Added peer 1, starting replication + logger.go:44: 10:44:56.613719 [INFO] raft: Added peer 2, starting replication + logger.go:44: 10:44:56.613815 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 10:44:56.613876 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 10:44:56.613823 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=3,index=2] + logger.go:44: 10:44:56.613942 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=3,index=2] + logger.go:44: 10:44:56.614098 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 10:44:56.614054 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "0") + logger.go:44: 10:44:56.614166 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 10:44:56.615145 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 10:44:56.615179 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 10:44:56.615209 [DEBUG] raft-test: elect: done + logger.go:44: 10:44:56.615332 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=3,index=3] + logger.go:44: 10:44:56.615348 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=3,index=3] + logger.go:44: 10:44:56.615474 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 10:44:56.615612 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=3,index=4] + logger.go:44: 10:44:56.615639 [DEBUG] raft-test: server 0: pipeline: append to 1: append fault: command 2 + logger.go:44: 10:44:56.615653 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=3,index=4] + logger.go:44: 10:44:56.615692 [DEBUG] raft-test: server 0: pipeline: append to 2: append fault: command 2 + logger.go:44: 10:44:56.615796 [DEBUG] raft-test: fsm 1: applied 1 + logger.go:44: 10:44:56.615809 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 10:44:56.615981 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 10:44:56.615835 [DEBUG] raft-test: fsm 2: applied 1 + logger.go:44: 10:44:56.616614 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.616643 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.616664 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 10:44:56.617167 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.617189 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.617208 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 10:44:56.617745 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 10:44:56.617777 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 10:44:56.617801 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 10:44:56.617846 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 10:44:56.617876 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 10:44:56.617904 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected + logger.go:44: 10:44:56.617956 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 10:44:56.618008 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 10:44:56.619348 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=3,index=4] + logger.go:44: 10:44:56.619376 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.619391 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=3,index=4] + logger.go:44: 10:44:56.619459 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.619490 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 10:44:56.619399 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 10:44:56.630075 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.630108 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.630128 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 10:44:56.630157 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.630206 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.630236 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 10:44:56.631039 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=3,index=4] + logger.go:44: 10:44:56.631065 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.631093 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 10:44:56.631097 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=3,index=4] + logger.go:44: 10:44:56.631150 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.631170 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 10:44:56.632144 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 10:44:56.632174 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 4 + logger.go:44: 10:44:56.632227 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.632247 [DEBUG] raft: Vote granted from 1 in term 4. Tally: 1 + logger.go:44: 10:44:56.632277 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 10:44:56.632334 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 10:44:56.634270 [WARN] raft: Failed to contact 1 in 18.733969ms + logger.go:44: 10:44:56.634302 [WARN] raft: Failed to contact 2 in 18.809512ms + logger.go:44: 10:44:56.634319 [WARN] raft: Failed to contact quorum of nodes, stepping down + logger.go:44: 10:44:56.634399 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.634426 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 10:44:56.634445 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 10:44:56.634525 [DEBUG] raft-test: elect: start (server 1) + logger.go:44: 10:44:56.634605 [DEBUG] raft-test: server 1: establish outbound connection to all other nodes + logger.go:44: 10:44:56.634645 [DEBUG] raft-test: elect: server 1: wait to become leader within 1.5s + logger.go:44: 10:44:56.634765 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 10:44:56.634808 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 4 + logger.go:44: 10:44:56.634884 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.634912 [DEBUG] raft: Vote granted from 2 in term 4. Tally: 1 + logger.go:44: 10:44:56.634931 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 10:44:56.634953 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 10:44:56.642519 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.642564 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.642593 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 10:44:56.643001 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.643071 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.643105 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 10:44:56.643129 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=3,index=4] + logger.go:44: 10:44:56.643212 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.643247 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 10:44:56.643357 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=3,index=4] + logger.go:44: 10:44:56.643388 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.643418 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 10:44:56.651634 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 10:44:56.651670 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 4 + logger.go:44: 10:44:56.651721 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.651742 [DEBUG] raft: Vote granted from 0 in term 4. Tally: 1 + logger.go:44: 10:44:56.651778 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 10:44:56.651781 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 10:44:56.658199 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 10:44:56.658242 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 5 + logger.go:44: 10:44:56.658304 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.658332 [DEBUG] raft: Vote granted from 1 in term 5. Tally: 1 + logger.go:44: 10:44:56.658375 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.658402 [DEBUG] raft: Vote granted from 2 in term 5. Tally: 2 + logger.go:44: 10:44:56.658423 [INFO] raft: Election won. Tally: 2 + logger.go:44: 10:44:56.658448 [INFO] raft: Node at 1 [Leader] entering Leader state + logger.go:44: 10:44:56.658475 [INFO] raft: Added peer 0, starting replication + logger.go:44: 10:44:56.658498 [INFO] raft: Added peer 2, starting replication + logger.go:44: 10:44:56.658571 [DEBUG] raft-test: server 1: leadership: acquired + logger.go:44: 10:44:56.658387 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.658611 [DEBUG] raft-test: server 1: transport: append to 2: 1 entry [Noop:term=5,index=5] + logger.go:44: 10:44:56.658630 [DEBUG] raft-test: elect: server 1: wait for other servers to become followers within 40ms + logger.go:44: 10:44:56.658746 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 10:44:56.658753 [DEBUG] raft-test: fsm 1: applied 2 + logger.go:44: 10:44:56.658682 [DEBUG] raft-test: server 1: transport: append to 0: 1 entry [Noop:term=5,index=5] + logger.go:44: 10:44:56.658950 [INFO] raft: pipelining replication to peer {Voter 0 0} + logger.go:44: 10:44:56.659977 [DEBUG] raft-test: elect: server 0: became follower + logger.go:44: 10:44:56.660005 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 10:44:56.660025 [DEBUG] raft-test: elect: done + logger.go:44: 10:44:56.660112 [DEBUG] raft-test: server 1: pipeline: append to 0: 1 entry [Command:term=5,index=6] + logger.go:44: 10:44:56.660133 [DEBUG] raft-test: server 1: pipeline: append to 2: 1 entry [Command:term=5,index=6] + logger.go:44: 10:44:56.661481 [DEBUG] raft-test: server 1: transport: append to 0: 0 entries + logger.go:44: 10:44:56.660281 [DEBUG] raft-test: fsm 0: applied 2 + logger.go:44: 10:44:56.661553 [DEBUG] raft-test: fsm 2: applied 2 + logger.go:44: 10:44:56.661123 [DEBUG] raft-test: fsm 1: applied 3 + logger.go:44: 10:44:56.661650 [DEBUG] raft-test: close: start + logger.go:44: 10:44:56.661271 [DEBUG] raft-test: server 1: transport: append to 2: 0 entries + logger.go:44: 10:44:56.661781 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 10:44:56.661791 [INFO] raft: aborting pipeline replication to peer {Voter 0 0} + logger.go:44: 10:44:56.661905 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 10:44:56.661978 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 10:44:56.662086 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 10:44:56.662126 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 10:44:56.662200 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 10:44:56.662258 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 10:44:56.662277 [DEBUG] raft-test: close: done + --- PASS: TestControl_DeposeAfterCommandAppendedThenElect/2 (0.04s) + logger.go:44: 10:44:56.662582 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 10:44:56.662690 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 10:44:56.662738 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 10:44:56.662771 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 10:44:56.662805 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 10:44:56.662871 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:56.662920 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 10:44:56.662970 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:56.662977 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.662991 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 10:44:56.663040 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.663094 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:56.663151 [DEBUG] raft-test: setup: done + logger.go:44: 10:44:56.663177 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 10:44:56.663203 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 10:44:56.663215 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.663235 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 10:44:56.680206 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 10:44:56.680248 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 2 + logger.go:44: 10:44:56.680301 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.680322 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 1 + logger.go:44: 10:44:56.680351 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 10:44:56.680354 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 10:44:56.681183 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 10:44:56.681217 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 10:44:56.681298 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.681320 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 10:44:56.681357 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 10:44:56.681389 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 2 + logger.go:44: 10:44:56.681425 [INFO] raft: Election won. Tally: 2 + logger.go:44: 10:44:56.681447 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 10:44:56.681492 [INFO] raft: Added peer 1, starting replication + logger.go:44: 10:44:56.681519 [INFO] raft: Added peer 2, starting replication + logger.go:44: 10:44:56.681607 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] + logger.go:44: 10:44:56.681629 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 10:44:56.681673 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 10:44:56.681674 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "0") + logger.go:44: 10:44:56.681707 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 10:44:56.681775 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] + logger.go:44: 10:44:56.681928 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 10:44:56.682937 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 10:44:56.682970 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 10:44:56.682995 [DEBUG] raft-test: elect: done + logger.go:44: 10:44:56.683104 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] + logger.go:44: 10:44:56.683140 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] + logger.go:44: 10:44:56.683212 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 10:44:56.683381 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=4] + logger.go:44: 10:44:56.683401 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=4] + logger.go:44: 10:44:56.683430 [DEBUG] raft-test: server 0: pipeline: append to 1: append fault: command 2 + logger.go:44: 10:44:56.683406 [DEBUG] raft-test: server 0: pipeline: append to 2: append fault: command 2 + logger.go:44: 10:44:56.683596 [DEBUG] raft-test: fsm 1: applied 1 + logger.go:44: 10:44:56.683679 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.683606 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 10:44:56.683746 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 10:44:56.683627 [DEBUG] raft-test: fsm 2: applied 1 + logger.go:44: 10:44:56.683939 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.683968 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.683992 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 10:44:56.685634 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 10:44:56.685641 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 10:44:56.685687 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 10:44:56.685711 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 10:44:56.685735 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 10:44:56.685660 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected + logger.go:44: 10:44:56.685779 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 10:44:56.685827 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 10:44:56.686415 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.686439 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.686459 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 10:44:56.687186 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=2,index=4] + logger.go:44: 10:44:56.687215 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.687237 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 10:44:56.687757 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=4] + logger.go:44: 10:44:56.687795 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.687818 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 10:44:56.696134 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.696166 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.696185 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 10:44:56.698927 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.698963 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.698983 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 10:44:56.699083 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=2,index=4] + logger.go:44: 10:44:56.699122 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.699152 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 10:44:56.699714 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=4] + logger.go:44: 10:44:56.699742 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.699768 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 10:44:56.702010 [WARN] raft: Failed to contact 1 in 18.269748ms + logger.go:44: 10:44:56.702047 [WARN] raft: Failed to contact 2 in 18.809633ms + logger.go:44: 10:44:56.702065 [WARN] raft: Failed to contact quorum of nodes, stepping down + logger.go:44: 10:44:56.702124 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.702179 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 10:44:56.702204 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 10:44:56.702179 [DEBUG] raft-test: elect: start (server 2) + logger.go:44: 10:44:56.702248 [DEBUG] raft-test: server 2: establish outbound connection to all other nodes + logger.go:44: 10:44:56.702279 [DEBUG] raft-test: elect: server 2: wait to become leader within 1.5s + logger.go:44: 10:44:56.702904 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 10:44:56.702936 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 + logger.go:44: 10:44:56.702993 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.703026 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 + logger.go:44: 10:44:56.703068 [WARN] raft: Rejecting vote request from 2 since we have a leader: 0 + logger.go:44: 10:44:56.703077 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 2 + logger.go:44: 10:44:56.703104 [INFO] raft: Election won. Tally: 2 + logger.go:44: 10:44:56.703128 [INFO] raft: Node at 2 [Leader] entering Leader state + logger.go:44: 10:44:56.703151 [INFO] raft: Added peer 0, starting replication + logger.go:44: 10:44:56.703174 [INFO] raft: Added peer 1, starting replication + logger.go:44: 10:44:56.703273 [DEBUG] raft-test: server 2: leadership: acquired + logger.go:44: 10:44:56.703338 [DEBUG] raft-test: elect: server 2: wait for other servers to become followers within 40ms + logger.go:44: 10:44:56.703359 [DEBUG] raft-test: server 2: transport: append to 0: 1 entry [Noop:term=3,index=5] + logger.go:44: 10:44:56.703278 [DEBUG] raft-test: server 2: transport: append to 1: 1 entry [Noop:term=3,index=5] + logger.go:44: 10:44:56.703454 [INFO] raft: pipelining replication to peer {Voter 0 0} + logger.go:44: 10:44:56.703505 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 10:44:56.703511 [DEBUG] raft-test: fsm 2: applied 2 + logger.go:44: 10:44:56.704562 [DEBUG] raft-test: elect: server 0: became follower + logger.go:44: 10:44:56.704594 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 10:44:56.704610 [DEBUG] raft-test: elect: done + logger.go:44: 10:44:56.704713 [DEBUG] raft-test: server 2: pipeline: append to 0: 1 entry [Command:term=3,index=6] + logger.go:44: 10:44:56.704747 [DEBUG] raft-test: server 2: pipeline: append to 1: 1 entry [Command:term=3,index=6] + logger.go:44: 10:44:56.704830 [DEBUG] raft-test: fsm 2: applied 3 + logger.go:44: 10:44:56.704841 [DEBUG] raft-test: fsm 0: applied 2 + logger.go:44: 10:44:56.704893 [DEBUG] raft-test: close: start + logger.go:44: 10:44:56.704926 [DEBUG] raft-test: fsm 1: applied 2 + logger.go:44: 10:44:56.704966 [INFO] raft: aborting pipeline replication to peer {Voter 0 0} + logger.go:44: 10:44:56.704988 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 10:44:56.705047 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 10:44:56.705187 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 10:44:56.705292 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=2,index=4] + logger.go:44: 10:44:56.705398 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.705427 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 10:44:56.705359 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=4] + logger.go:44: 10:44:56.705490 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.705522 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 10:44:56.705602 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 10:44:56.705629 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 10:44:56.705651 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 10:44:56.705705 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 10:44:56.705739 [DEBUG] raft-test: close: done === RUN TestControl_DeposeAfterCommandCommitted --- PASS: TestControl_DeposeAfterCommandCommitted (0.04s) - logger.go:44: 17:00:30.046667 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 17:00:30.046713 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 17:00:30.046730 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 17:00:30.046741 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 17:00:30.046752 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 17:00:30.046775 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:30.046783 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 17:00:30.046804 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:30.046811 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 17:00:30.046826 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:30.046833 [DEBUG] raft-test: setup: done - logger.go:44: 17:00:30.046839 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 17:00:30.046846 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 17:00:30.046862 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 17:00:30.046848 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:30.046862 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:30.046864 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:30.065772 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 17:00:30.065790 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 2 - logger.go:44: 17:00:30.065808 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:30.065818 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 1 - logger.go:44: 17:00:30.065826 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 17:00:30.065829 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 17:00:30.066118 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 17:00:30.066131 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 17:00:30.066144 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:30.066153 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 17:00:30.066168 [INFO] raft: Duplicate RequestVote for same term: 2 - logger.go:44: 17:00:30.066172 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 2 - logger.go:44: 17:00:30.066191 [INFO] raft: Election won. Tally: 2 - logger.go:44: 17:00:30.066200 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 17:00:30.066222 [INFO] raft: Added peer 1, starting replication - logger.go:44: 17:00:30.066235 [INFO] raft: Added peer 2, starting replication - logger.go:44: 17:00:30.066274 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] - logger.go:44: 17:00:30.066296 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 17:00:30.066314 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 17:00:30.066330 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 17:00:30.066342 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] - logger.go:44: 17:00:30.066374 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "0") - logger.go:44: 17:00:30.066398 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 17:00:30.067515 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 17:00:30.067529 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 17:00:30.067536 [DEBUG] raft-test: elect: done - logger.go:44: 17:00:30.067575 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] - logger.go:44: 17:00:30.067584 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] - logger.go:44: 17:00:30.067633 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 17:00:30.067648 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 17:00:30.067661 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 17:00:30.068372 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:30.068385 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:30.068392 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 17:00:30.068604 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:30.068613 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:30.068618 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 17:00:30.069068 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 17:00:30.069080 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 17:00:30.069088 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 17:00:30.069101 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 17:00:30.069294 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 17:00:30.069310 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected - logger.go:44: 17:00:30.069322 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 17:00:30.069342 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 17:00:30.070623 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:30.070637 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:30.070646 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 17:00:30.070956 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:30.070981 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:30.070988 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 17:00:30.080956 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:30.080976 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:30.080986 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 17:00:30.081942 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:30.081972 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:30.081989 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 17:00:30.082572 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:30.082812 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:30.082917 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:30.082938 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 17:00:30.082900 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:30.082979 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 17:00:30.084376 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 17:00:30.084395 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 3 - logger.go:44: 17:00:30.084414 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:30.084422 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 1 - logger.go:44: 17:00:30.084434 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 17:00:30.084438 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 17:00:30.086530 [WARN] raft: Failed to contact 1 in 18.837666ms - logger.go:44: 17:00:30.086549 [WARN] raft: Failed to contact 2 in 18.898784ms - logger.go:44: 17:00:30.086555 [WARN] raft: Failed to contact quorum of nodes, stepping down - logger.go:44: 17:00:30.086572 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:30.086582 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 17:00:30.086588 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 17:00:30.086622 [DEBUG] raft-test: close: start - logger.go:44: 17:00:30.086647 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:30.086658 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:30.086682 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:30.086693 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 17:00:30.086658 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:30.086723 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 17:00:30.086735 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 17:00:30.086750 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 17:00:30.086778 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 17:00:30.086802 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 17:00:30.086810 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 17:00:30.086827 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 17:00:30.086840 [DEBUG] raft-test: close: done + logger.go:44: 10:44:56.706004 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 10:44:56.706148 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 10:44:56.706208 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 10:44:56.706260 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 10:44:56.706303 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 10:44:56.706385 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:56.706437 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 10:44:56.706496 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.706503 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:56.706586 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 10:44:56.706648 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:56.706682 [DEBUG] raft-test: setup: done + logger.go:44: 10:44:56.706707 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 10:44:56.706735 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 10:44:56.706767 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 10:44:56.706825 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.706884 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.722733 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 10:44:56.722774 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 2 + logger.go:44: 10:44:56.722831 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.722852 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 1 + logger.go:44: 10:44:56.722884 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 10:44:56.722893 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 10:44:56.726805 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 10:44:56.726864 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 10:44:56.726924 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.726945 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 10:44:56.727001 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 10:44:56.727020 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 2 + logger.go:44: 10:44:56.727047 [INFO] raft: Election won. Tally: 2 + logger.go:44: 10:44:56.727070 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 10:44:56.727094 [INFO] raft: Added peer 1, starting replication + logger.go:44: 10:44:56.727132 [INFO] raft: Added peer 2, starting replication + logger.go:44: 10:44:56.727230 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 10:44:56.727238 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] + logger.go:44: 10:44:56.727289 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 10:44:56.727328 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] + logger.go:44: 10:44:56.727355 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "0") + logger.go:44: 10:44:56.727399 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 10:44:56.727425 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 10:44:56.728529 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 10:44:56.728562 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 10:44:56.728588 [DEBUG] raft-test: elect: done + logger.go:44: 10:44:56.728705 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] + logger.go:44: 10:44:56.728718 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] + logger.go:44: 10:44:56.728836 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 10:44:56.728879 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 10:44:56.728916 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 10:44:56.729351 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.729395 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.729417 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 10:44:56.729872 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.729899 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.729923 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 10:44:56.730079 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 10:44:56.730128 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected + logger.go:44: 10:44:56.730151 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 10:44:56.730197 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 10:44:56.730329 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 10:44:56.730360 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 10:44:56.730386 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 10:44:56.730438 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 10:44:56.731680 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.731707 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.731730 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 10:44:56.732053 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.732111 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.732136 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 10:44:56.741589 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.741631 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.741651 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 10:44:56.741834 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.741856 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.741873 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 10:44:56.743257 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.743282 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.743305 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 10:44:56.743567 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.743588 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.743614 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 10:44:56.747617 [WARN] raft: Failed to contact 1 in 18.791228ms + logger.go:44: 10:44:56.747651 [WARN] raft: Failed to contact 2 in 18.769999ms + logger.go:44: 10:44:56.747671 [WARN] raft: Failed to contact quorum of nodes, stepping down + logger.go:44: 10:44:56.747741 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.747774 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 10:44:56.747797 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 10:44:56.747868 [DEBUG] raft-test: close: start + logger.go:44: 10:44:56.747950 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.747979 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.748006 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 10:44:56.747951 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.748093 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.748119 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 10:44:56.748158 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 10:44:56.748218 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 10:44:56.748254 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 10:44:56.748273 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 10:44:56.748295 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 10:44:56.748337 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 10:44:56.748366 [DEBUG] raft-test: close: done === RUN TestControl_DeposeAfterCommandCommittedThenElect === RUN TestControl_DeposeAfterCommandCommittedThenElect/0 === RUN TestControl_DeposeAfterCommandCommittedThenElect/1 === RUN TestControl_DeposeAfterCommandCommittedThenElect/2 ---- PASS: TestControl_DeposeAfterCommandCommittedThenElect (0.23s) - --- PASS: TestControl_DeposeAfterCommandCommittedThenElect/0 (0.06s) - logger.go:44: 17:00:30.086893 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 17:00:30.086939 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 17:00:30.086959 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 17:00:30.086973 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 17:00:30.086985 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 17:00:30.087013 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:30.087030 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 17:00:30.087054 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:30.087056 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:30.087077 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 17:00:30.087104 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:30.087112 [DEBUG] raft-test: setup: done - logger.go:44: 17:00:30.087119 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 17:00:30.087130 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 17:00:30.087136 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:30.087140 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 17:00:30.087170 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:30.107146 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 17:00:30.107162 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 2 - logger.go:44: 17:00:30.107178 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:30.107186 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 1 - logger.go:44: 17:00:30.107197 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 17:00:30.107202 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 17:00:30.110447 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 17:00:30.110460 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 17:00:30.110479 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:30.110486 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 17:00:30.110500 [INFO] raft: Duplicate RequestVote for same term: 2 - logger.go:44: 17:00:30.110514 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 2 - logger.go:44: 17:00:30.110523 [INFO] raft: Election won. Tally: 2 - logger.go:44: 17:00:30.110529 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 17:00:30.110537 [INFO] raft: Added peer 1, starting replication - logger.go:44: 17:00:30.110547 [INFO] raft: Added peer 2, starting replication - logger.go:44: 17:00:30.110594 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 17:00:30.110620 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 17:00:30.110605 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] - logger.go:44: 17:00:30.110679 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 17:00:30.110626 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] - logger.go:44: 17:00:30.110723 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "0") - logger.go:44: 17:00:30.110744 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 17:00:30.111735 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 17:00:30.111749 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 17:00:30.111757 [DEBUG] raft-test: elect: done - logger.go:44: 17:00:30.111807 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] - logger.go:44: 17:00:30.111827 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] - logger.go:44: 17:00:30.111866 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 17:00:30.111899 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 17:00:30.111924 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 17:00:30.112357 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:30.112395 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:30.112408 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 17:00:30.113691 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 17:00:30.113714 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 17:00:30.113745 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 17:00:30.113779 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 17:00:30.113752 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:30.113843 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:30.113850 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 17:00:30.113926 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 17:00:30.113937 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected - logger.go:44: 17:00:30.113945 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 17:00:30.113959 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 17:00:30.115821 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:30.115850 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:30.115858 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 17:00:30.115933 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:30.115942 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:30.115954 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 17:00:30.125801 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:30.125818 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:30.125826 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 17:00:30.126806 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:30.126837 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:30.126844 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 17:00:30.127655 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:30.127668 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:30.127677 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 17:00:30.128116 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:30.128131 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:30.128140 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 17:00:30.130868 [WARN] raft: Failed to contact 1 in 18.989736ms - logger.go:44: 17:00:30.130884 [WARN] raft: Failed to contact 2 in 19.008712ms - logger.go:44: 17:00:30.130890 [WARN] raft: Failed to contact quorum of nodes, stepping down - logger.go:44: 17:00:30.130909 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:30.130918 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 17:00:30.130941 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 17:00:30.130974 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 17:00:30.130986 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 17:00:30.130995 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 17:00:30.134718 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 17:00:30.134735 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 - logger.go:44: 17:00:30.134753 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:30.134761 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 - logger.go:44: 17:00:30.134770 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 17:00:30.134805 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 17:00:30.138874 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:30.139196 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:30.139248 [DEBUG] raft-test: fsm 1: applied 1 - logger.go:44: 17:00:30.139268 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:30.139309 [DEBUG] raft-test: server 0: transport: append to 2: newer term - logger.go:44: 17:00:30.140085 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:30.140144 [DEBUG] raft-test: server 0: transport: append to 2: newer term - logger.go:44: 17:00:30.140177 [ERR] raft: peer {Voter 2 2} has newer term, stopping replication - logger.go:44: 17:00:30.147925 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 17:00:30.147976 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 3 - logger.go:44: 17:00:30.147994 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:30.148006 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 1 - logger.go:44: 17:00:30.148042 [INFO] raft: Duplicate RequestVote for same term: 3 - logger.go:44: 17:00:30.148043 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 2 - logger.go:44: 17:00:30.148073 [INFO] raft: Election won. Tally: 2 - logger.go:44: 17:00:30.148083 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 17:00:30.148100 [INFO] raft: Added peer 1, starting replication - logger.go:44: 17:00:30.148108 [INFO] raft: Added peer 2, starting replication - logger.go:44: 17:00:30.148143 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=3,index=4] - logger.go:44: 17:00:30.148161 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "0") - logger.go:44: 17:00:30.148175 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 17:00:30.148208 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 17:00:30.148210 [DEBUG] raft-test: fsm 2: applied 1 - logger.go:44: 17:00:30.148218 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=3,index=4] - logger.go:44: 17:00:30.148279 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 17:00:30.148226 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 17:00:30.148329 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 17:00:30.148336 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 17:00:30.148340 [DEBUG] raft-test: elect: done - logger.go:44: 17:00:30.148367 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=3,index=5] - logger.go:44: 17:00:30.148383 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=3,index=5] - logger.go:44: 17:00:30.148414 [DEBUG] raft-test: fsm 0: applied 2 - logger.go:44: 17:00:30.148442 [DEBUG] raft-test: close: start - logger.go:44: 17:00:30.148473 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 17:00:30.148482 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 17:00:30.148508 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 17:00:30.148523 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 17:00:30.148534 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 17:00:30.148545 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 17:00:30.148554 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 17:00:30.148578 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 17:00:30.148598 [DEBUG] raft-test: close: done +--- PASS: TestControl_DeposeAfterCommandCommittedThenElect (0.34s) + --- PASS: TestControl_DeposeAfterCommandCommittedThenElect/0 (0.17s) + logger.go:44: 10:44:56.748506 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 10:44:56.748617 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 10:44:56.748673 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 10:44:56.748722 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 10:44:56.748766 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 10:44:56.748832 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:56.748879 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 10:44:56.748940 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.750641 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:56.750829 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 10:44:56.750940 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:56.751027 [DEBUG] raft-test: setup: done + logger.go:44: 10:44:56.751082 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 10:44:56.751095 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.751150 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 10:44:56.751665 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.751678 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 10:44:56.775515 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 10:44:56.775637 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 10:44:56.775730 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.775765 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 10:44:56.775846 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 2 + logger.go:44: 10:44:56.775876 [INFO] raft: Election won. Tally: 2 + logger.go:44: 10:44:56.775901 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 10:44:56.775974 [INFO] raft: Added peer 1, starting replication + logger.go:44: 10:44:56.776003 [INFO] raft: Added peer 2, starting replication + logger.go:44: 10:44:56.776046 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 10:44:56.776114 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] + logger.go:44: 10:44:56.776128 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 10:44:56.776175 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] + logger.go:44: 10:44:56.776185 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 10:44:56.776352 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 10:44:56.777379 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 10:44:56.777405 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 10:44:56.777420 [DEBUG] raft-test: elect: done + logger.go:44: 10:44:56.777524 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] + logger.go:44: 10:44:56.777570 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] + logger.go:44: 10:44:56.777621 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 10:44:56.777663 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 10:44:56.777702 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 10:44:56.778665 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.778688 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.778712 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 10:44:56.779172 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 10:44:56.779242 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected + logger.go:44: 10:44:56.779253 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.779282 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 10:44:56.779299 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.779330 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 10:44:56.779356 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 10:44:56.779724 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 10:44:56.779748 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 10:44:56.779771 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 10:44:56.779817 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 10:44:56.781988 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.782049 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.782081 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 10:44:56.782017 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.782142 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.782168 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 10:44:56.791334 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.791416 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.791450 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 10:44:56.791339 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.791522 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.791546 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 10:44:56.793940 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.793981 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.794012 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 10:44:56.793964 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.794094 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.794121 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 10:44:56.796526 [WARN] raft: Failed to contact 1 in 18.772762ms + logger.go:44: 10:44:56.796562 [WARN] raft: Failed to contact 2 in 18.91257ms + logger.go:44: 10:44:56.796589 [WARN] raft: Failed to contact quorum of nodes, stepping down + logger.go:44: 10:44:56.796660 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.796694 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 10:44:56.796718 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 10:44:56.796792 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 10:44:56.796846 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 10:44:56.796891 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 10:44:56.800453 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 10:44:56.800496 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 + logger.go:44: 10:44:56.800563 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.800590 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 + logger.go:44: 10:44:56.800621 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 10:44:56.800630 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 10:44:56.804004 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.804418 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.804468 [DEBUG] raft-test: server 0: transport: append to 2: newer term + logger.go:44: 10:44:56.805890 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.806002 [DEBUG] raft-test: fsm 1: applied 1 + logger.go:44: 10:44:56.806548 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.806601 [DEBUG] raft-test: server 0: transport: append to 2: newer term + logger.go:44: 10:44:56.806626 [ERR] raft: peer {Voter 2 2} has newer term, stopping replication + logger.go:44: 10:44:56.822186 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 10:44:56.822234 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 4 + logger.go:44: 10:44:56.822313 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.822340 [DEBUG] raft: Vote granted from 2 in term 4. Tally: 1 + logger.go:44: 10:44:56.822374 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 10:44:56.822383 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 10:44:56.823492 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 10:44:56.823532 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 3 + logger.go:44: 10:44:56.823595 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.823622 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 1 + logger.go:44: 10:44:56.823702 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 2 + logger.go:44: 10:44:56.823729 [INFO] raft: Election won. Tally: 2 + logger.go:44: 10:44:56.823752 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 10:44:56.823782 [INFO] raft: Added peer 1, starting replication + logger.go:44: 10:44:56.823806 [INFO] raft: Added peer 2, starting replication + logger.go:44: 10:44:56.823913 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=3,index=4] + logger.go:44: 10:44:56.823922 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 10:44:56.823955 [DEBUG] raft-test: server 0: transport: append to 2: newer term + logger.go:44: 10:44:56.823970 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 10:44:56.824052 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=3,index=4] + logger.go:44: 10:44:56.823981 [ERR] raft: peer {Voter 2 2} has newer term, stopping replication + logger.go:44: 10:44:56.824127 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.824161 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 10:44:56.825273 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 10:44:56.825316 [DEBUG] raft-test: elect: server 0: lost leadership + logger.go:44: 10:44:56.825339 [DEBUG] raft-test: elect: server 0: retry 1 + logger.go:44: 10:44:56.825394 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 10:44:56.825427 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 10:44:56.841391 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 10:44:56.841444 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 5 + logger.go:44: 10:44:56.841517 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.841540 [DEBUG] raft: Vote granted from 2 in term 5. Tally: 1 + logger.go:44: 10:44:56.841591 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 10:44:56.841600 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 10:44:56.854322 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 10:44:56.854370 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 4 + logger.go:44: 10:44:56.854500 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.854832 [DEBUG] raft: Vote granted from 0 in term 4. Tally: 1 + logger.go:44: 10:44:56.854861 [DEBUG] raft: Vote granted from 1 in term 4. Tally: 2 + logger.go:44: 10:44:56.854887 [INFO] raft: Election won. Tally: 2 + logger.go:44: 10:44:56.854921 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 10:44:56.855015 [INFO] raft: Added peer 1, starting replication + logger.go:44: 10:44:56.855051 [INFO] raft: Added peer 2, starting replication + logger.go:44: 10:44:56.855177 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 10:44:56.855239 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 10:44:56.855273 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=4,index=5] + logger.go:44: 10:44:56.855194 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=4,index=5] + logger.go:44: 10:44:56.855350 [DEBUG] raft-test: server 0: transport: append to 2: newer term + logger.go:44: 10:44:56.855385 [ERR] raft: peer {Voter 2 2} has newer term, stopping replication + logger.go:44: 10:44:56.855365 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 10:44:56.855446 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.855456 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 10:44:56.855537 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 10:44:56.856529 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 10:44:56.856580 [DEBUG] raft-test: elect: server 0: lost leadership + logger.go:44: 10:44:56.856609 [DEBUG] raft-test: elect: server 0: retry 2 + logger.go:44: 10:44:56.856676 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 10:44:56.856722 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 10:44:56.869397 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 10:44:56.869511 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 6 + logger.go:44: 10:44:56.869585 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.869611 [DEBUG] raft: Vote granted from 2 in term 6. Tally: 1 + logger.go:44: 10:44:56.869656 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 10:44:56.869710 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 10:44:56.881452 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 10:44:56.881499 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 5 + logger.go:44: 10:44:56.881572 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.881600 [DEBUG] raft: Vote granted from 0 in term 5. Tally: 1 + logger.go:44: 10:44:56.881647 [DEBUG] raft: Vote granted from 1 in term 5. Tally: 2 + logger.go:44: 10:44:56.881671 [INFO] raft: Election won. Tally: 2 + logger.go:44: 10:44:56.881697 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 10:44:56.881729 [INFO] raft: Added peer 1, starting replication + logger.go:44: 10:44:56.881756 [INFO] raft: Added peer 2, starting replication + logger.go:44: 10:44:56.881889 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=5,index=6] + logger.go:44: 10:44:56.881916 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 10:44:56.881992 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 10:44:56.881951 [DEBUG] raft-test: server 0: transport: append to 2: newer term + logger.go:44: 10:44:56.882072 [ERR] raft: peer {Voter 2 2} has newer term, stopping replication + logger.go:44: 10:44:56.882018 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=5,index=6] + logger.go:44: 10:44:56.882164 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.882198 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 10:44:56.883229 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 10:44:56.883264 [DEBUG] raft-test: elect: server 0: lost leadership + logger.go:44: 10:44:56.883286 [DEBUG] raft-test: elect: server 0: retry 3 + logger.go:44: 10:44:56.883309 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 10:44:56.883337 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 10:44:56.897962 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 10:44:56.898009 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 7 + logger.go:44: 10:44:56.898053 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.898081 [DEBUG] raft: Vote granted from 2 in term 7. Tally: 1 + logger.go:44: 10:44:56.898118 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 10:44:56.898185 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 10:44:56.898228 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 6 + logger.go:44: 10:44:56.898262 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.898282 [DEBUG] raft: Vote granted from 0 in term 6. Tally: 1 + logger.go:44: 10:44:56.898186 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 10:44:56.898337 [DEBUG] raft: Newer term discovered, fallback to follower + logger.go:44: 10:44:56.898399 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.915275 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 10:44:56.915330 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 8 + logger.go:44: 10:44:56.915398 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.915426 [DEBUG] raft: Vote granted from 0 in term 8. Tally: 1 + logger.go:44: 10:44:56.915491 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.915520 [DEBUG] raft: Vote granted from 1 in term 8. Tally: 2 + logger.go:44: 10:44:56.915563 [INFO] raft: Election won. Tally: 2 + logger.go:44: 10:44:56.915604 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 10:44:56.915671 [INFO] raft: Added peer 1, starting replication + logger.go:44: 10:44:56.915719 [INFO] raft: Added peer 2, starting replication + logger.go:44: 10:44:56.915742 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 10:44:56.915784 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 10:44:56.915876 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=8,index=7] + logger.go:44: 10:44:56.915878 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=8,index=7] + logger.go:44: 10:44:56.915999 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 10:44:56.916071 [WARN] raft: Failed to get previous log: 6 log not found (last: 3) + logger.go:44: 10:44:56.916111 [WARN] raft: AppendEntries to {Voter 2 2} rejected, sending older logs (next: 4) + logger.go:44: 10:44:56.916156 [DEBUG] raft-test: server 0: transport: append to 2: 4 entries [Noop:term=3,index=4 Noop:term=4,index=5 Noop:term=5,index=6 Noop:term=8,index=7] + logger.go:44: 10:44:56.916278 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 10:44:56.916322 [DEBUG] raft-test: fsm 2: applied 1 + logger.go:44: 10:44:56.917013 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 10:44:56.917044 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 10:44:56.917070 [DEBUG] raft-test: elect: done + logger.go:44: 10:44:56.917264 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=8,index=8] + logger.go:44: 10:44:56.917275 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=8,index=8] + logger.go:44: 10:44:56.917395 [DEBUG] raft-test: fsm 0: applied 2 + logger.go:44: 10:44:56.917455 [DEBUG] raft-test: close: start + logger.go:44: 10:44:56.917593 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 10:44:56.917647 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 10:44:56.917709 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 10:44:56.917776 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 10:44:56.917844 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 10:44:56.917877 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 10:44:56.917953 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 10:44:56.918009 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 10:44:56.918134 [DEBUG] raft-test: close: done --- PASS: TestControl_DeposeAfterCommandCommittedThenElect/1 (0.09s) - logger.go:44: 17:00:30.148692 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 17:00:30.148754 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 17:00:30.148782 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 17:00:30.148800 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 17:00:30.148820 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 17:00:30.148857 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:30.148875 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 17:00:30.148900 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:30.148906 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:30.148928 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 17:00:30.148955 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:30.148961 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:30.148987 [DEBUG] raft-test: setup: done - logger.go:44: 17:00:30.148995 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 17:00:30.149002 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 17:00:30.149012 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 17:00:30.149027 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:30.169504 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 17:00:30.169524 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 2 - logger.go:44: 17:00:30.169545 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:30.169553 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 1 - logger.go:44: 17:00:30.169564 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 17:00:30.169565 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 17:00:30.173464 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 17:00:30.173480 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 2 - logger.go:44: 17:00:30.173497 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:30.173504 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 1 - logger.go:44: 17:00:30.173515 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 17:00:30.173518 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 17:00:30.175110 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 17:00:30.175126 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 17:00:30.175146 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:30.175159 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 17:00:30.175170 [INFO] raft: Duplicate RequestVote for same term: 2 - logger.go:44: 17:00:30.175177 [INFO] raft: Duplicate RequestVote for same term: 2 - logger.go:44: 17:00:30.197879 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 17:00:30.197915 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 3 - logger.go:44: 17:00:30.197953 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:30.197970 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 1 - logger.go:44: 17:00:30.198016 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:30.198035 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:30.198042 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 2 - logger.go:44: 17:00:30.198058 [INFO] raft: Election won. Tally: 2 - logger.go:44: 17:00:30.198065 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 17:00:30.198074 [INFO] raft: Added peer 1, starting replication - logger.go:44: 17:00:30.198083 [INFO] raft: Added peer 2, starting replication - logger.go:44: 17:00:30.198105 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 17:00:30.198130 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 17:00:30.198142 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=3,index=2] - logger.go:44: 17:00:30.198182 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 17:00:30.198112 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=3,index=2] - logger.go:44: 17:00:30.198227 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 17:00:30.199330 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 17:00:30.199348 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 17:00:30.199356 [DEBUG] raft-test: elect: done - logger.go:44: 17:00:30.199403 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=3,index=3] - logger.go:44: 17:00:30.199412 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=3,index=3] - logger.go:44: 17:00:30.199470 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 17:00:30.199486 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 17:00:30.199505 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 17:00:30.200803 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 17:00:30.200814 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 17:00:30.200822 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 17:00:30.200831 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 17:00:30.201223 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:30.201245 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:30.201254 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 17:00:30.201230 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:30.201273 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:30.201278 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 17:00:30.201237 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 17:00:30.201291 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected - logger.go:44: 17:00:30.201299 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 17:00:30.201307 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 17:00:30.202387 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:30.202401 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:30.202409 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 17:00:30.203191 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:30.203205 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:30.203214 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 17:00:30.213811 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:30.213843 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:30.213850 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 17:00:30.214635 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:30.214653 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:30.214661 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 17:00:30.214641 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:30.214676 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:30.214683 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 17:00:30.215366 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:30.215380 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:30.215390 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 17:00:30.216008 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 17:00:30.216024 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 4 - logger.go:44: 17:00:30.216035 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:30.216042 [DEBUG] raft: Vote granted from 1 in term 4. Tally: 1 - logger.go:44: 17:00:30.216044 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 17:00:30.216058 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 17:00:30.218475 [WARN] raft: Failed to contact 2 in 18.946269ms - logger.go:44: 17:00:30.218489 [WARN] raft: Failed to contact 1 in 19.006344ms - logger.go:44: 17:00:30.218495 [WARN] raft: Failed to contact quorum of nodes, stepping down - logger.go:44: 17:00:30.218516 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:30.218525 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 17:00:30.218532 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 17:00:30.218552 [DEBUG] raft-test: elect: start (server 1) - logger.go:44: 17:00:30.218559 [DEBUG] raft-test: server 1: establish outbound connection to all other nodes - logger.go:44: 17:00:30.218569 [DEBUG] raft-test: elect: server 1: wait to become leader within 1.5s - logger.go:44: 17:00:30.221427 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 17:00:30.221444 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 4 - logger.go:44: 17:00:30.221455 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:30.221462 [DEBUG] raft: Vote granted from 2 in term 4. Tally: 1 - logger.go:44: 17:00:30.221474 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 17:00:30.221477 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 17:00:30.226210 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:30.226226 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:30.226236 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 17:00:30.227564 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:30.227588 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:30.227600 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 17:00:30.227568 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:30.227615 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:30.227621 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 17:00:30.237114 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 17:00:30.237149 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 5 - logger.go:44: 17:00:30.237168 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:30.237176 [DEBUG] raft: Vote granted from 1 in term 5. Tally: 1 - logger.go:44: 17:00:30.237205 [DEBUG] raft: Vote granted from 0 in term 5. Tally: 2 - logger.go:44: 17:00:30.237219 [INFO] raft: Election won. Tally: 2 - logger.go:44: 17:00:30.237227 [INFO] raft: Node at 1 [Leader] entering Leader state - logger.go:44: 17:00:30.237236 [INFO] raft: Added peer 0, starting replication - logger.go:44: 17:00:30.237244 [INFO] raft: Added peer 2, starting replication - logger.go:44: 17:00:30.237273 [DEBUG] raft-test: server 1: leadership: acquired - logger.go:44: 17:00:30.237289 [DEBUG] raft-test: server 1: transport: append to 2: 1 entry [Noop:term=5,index=4] - logger.go:44: 17:00:30.237209 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:30.237322 [DEBUG] raft-test: server 1: transport: append to 0: 1 entry [Noop:term=5,index=4] - logger.go:44: 17:00:30.237301 [DEBUG] raft-test: elect: server 1: wait for other servers to become followers within 40ms - logger.go:44: 17:00:30.237351 [DEBUG] raft-test: fsm 1: applied 1 - logger.go:44: 17:00:30.237351 [INFO] raft: pipelining replication to peer {Voter 0 0} - logger.go:44: 17:00:30.237342 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 17:00:30.237363 [DEBUG] raft-test: elect: server 0: became follower - logger.go:44: 17:00:30.237438 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 17:00:30.237460 [DEBUG] raft-test: elect: done - logger.go:44: 17:00:30.237488 [DEBUG] raft-test: server 1: pipeline: append to 0: 1 entry [Command:term=5,index=5] - logger.go:44: 17:00:30.237497 [DEBUG] raft-test: server 1: pipeline: append to 2: 1 entry [Command:term=5,index=5] - logger.go:44: 17:00:30.237536 [DEBUG] raft-test: fsm 2: applied 1 - logger.go:44: 17:00:30.237542 [DEBUG] raft-test: fsm 1: applied 2 - logger.go:44: 17:00:30.237578 [DEBUG] raft-test: close: start - logger.go:44: 17:00:30.237596 [INFO] raft: aborting pipeline replication to peer {Voter 0 0} - logger.go:44: 17:00:30.237605 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 17:00:30.237629 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 17:00:30.237654 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 17:00:30.237673 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 17:00:30.237683 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 17:00:30.237693 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 17:00:30.237705 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 17:00:30.237723 [DEBUG] raft-test: close: done + logger.go:44: 10:44:56.918613 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 10:44:56.918763 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 10:44:56.918838 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 10:44:56.918901 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 10:44:56.918957 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 10:44:56.919059 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:56.919098 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 10:44:56.919182 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:56.919215 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 10:44:56.919299 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:56.919330 [DEBUG] raft-test: setup: done + logger.go:44: 10:44:56.919362 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 10:44:56.919407 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 10:44:56.919444 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 10:44:56.919512 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.919582 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.919659 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.941223 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 10:44:56.941274 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 2 + logger.go:44: 10:44:56.941332 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.941354 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 1 + logger.go:44: 10:44:56.941390 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 10:44:56.941445 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 10:44:56.944659 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 10:44:56.944726 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 10:44:56.944766 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.944797 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 10:44:56.944846 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 10:44:56.944661 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 10:44:56.944915 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 2 + logger.go:44: 10:44:56.944951 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.944977 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 10:44:56.945010 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 1 + logger.go:44: 10:44:56.945052 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 10:44:56.945146 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 10:44:56.964007 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 10:44:56.964057 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 3 + logger.go:44: 10:44:56.964118 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.964145 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 1 + logger.go:44: 10:44:56.964275 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.964302 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 2 + logger.go:44: 10:44:56.964341 [INFO] raft: Election won. Tally: 2 + logger.go:44: 10:44:56.964377 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 10:44:56.964406 [INFO] raft: Added peer 1, starting replication + logger.go:44: 10:44:56.964434 [INFO] raft: Added peer 2, starting replication + logger.go:44: 10:44:56.964524 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=3,index=2] + logger.go:44: 10:44:56.964596 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 10:44:56.964280 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.964662 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 10:44:56.964703 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 10:44:56.964761 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=3,index=2] + logger.go:44: 10:44:56.964820 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 10:44:56.964879 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 10:44:56.964897 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 10:44:56.964913 [DEBUG] raft-test: elect: done + logger.go:44: 10:44:56.964998 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=3,index=3] + logger.go:44: 10:44:56.965150 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 10:44:56.965212 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 10:44:56.965263 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 10:44:56.965299 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=3,index=3] + logger.go:44: 10:44:56.965416 [DEBUG] raft-test: fsm 1: applied 1 + logger.go:44: 10:44:56.966513 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 10:44:56.966545 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected + logger.go:44: 10:44:56.966571 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 10:44:56.966597 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 10:44:56.967137 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 10:44:56.967159 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 10:44:56.967181 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 10:44:56.967223 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 10:44:56.967871 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.967917 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.967936 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 10:44:56.968260 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.968291 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.968312 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 10:44:56.968314 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.968353 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.968370 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 10:44:56.968748 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.968796 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.968824 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 10:44:56.975381 [WARN] raft: Failed to contact 2 in 10.281352ms + logger.go:44: 10:44:56.980368 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.980408 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.980436 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 10:44:56.981187 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.981235 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.981262 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 10:44:56.982466 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 10:44:56.982504 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 4 + logger.go:44: 10:44:56.982535 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.982557 [DEBUG] raft: Vote granted from 1 in term 4. Tally: 1 + logger.go:44: 10:44:56.982716 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 10:44:56.982785 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 10:44:56.983096 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 10:44:56.983120 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 4 + logger.go:44: 10:44:56.983147 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:56.983179 [DEBUG] raft: Vote granted from 2 in term 4. Tally: 1 + logger.go:44: 10:44:56.983208 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 10:44:56.983257 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 10:44:56.983354 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.983390 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.983398 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.983416 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 10:44:56.983441 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.983485 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 10:44:56.985616 [WARN] raft: Failed to contact 2 in 20.522874ms + logger.go:44: 10:44:56.985645 [WARN] raft: Failed to contact 1 in 20.199729ms + logger.go:44: 10:44:56.985668 [WARN] raft: Failed to contact quorum of nodes, stepping down + logger.go:44: 10:44:56.985724 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:56.985752 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 10:44:56.985771 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 10:44:56.985955 [DEBUG] raft-test: elect: start (server 1) + logger.go:44: 10:44:56.986000 [DEBUG] raft-test: server 1: establish outbound connection to all other nodes + logger.go:44: 10:44:56.986026 [DEBUG] raft-test: elect: server 1: wait to become leader within 1.5s + logger.go:44: 10:44:56.993662 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:56.993716 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:56.993767 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 10:44:56.995289 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:56.995324 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:56.995349 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 10:44:57.004822 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 10:44:57.004889 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 5 + logger.go:44: 10:44:57.004955 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:57.004978 [DEBUG] raft: Vote granted from 1 in term 5. Tally: 1 + logger.go:44: 10:44:57.005036 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:57.005133 [DEBUG] raft: Vote granted from 2 in term 5. Tally: 2 + logger.go:44: 10:44:57.005160 [INFO] raft: Election won. Tally: 2 + logger.go:44: 10:44:57.005186 [INFO] raft: Node at 1 [Leader] entering Leader state + logger.go:44: 10:44:57.005219 [INFO] raft: Added peer 0, starting replication + logger.go:44: 10:44:57.005252 [INFO] raft: Added peer 2, starting replication + logger.go:44: 10:44:57.005407 [DEBUG] raft-test: server 1: transport: append to 2: 1 entry [Noop:term=5,index=4] + logger.go:44: 10:44:57.005503 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 10:44:57.005518 [DEBUG] raft-test: server 1: transport: append to 0: 1 entry [Noop:term=5,index=4] + logger.go:44: 10:44:57.005561 [DEBUG] raft-test: fsm 2: applied 1 + logger.go:44: 10:44:57.005613 [INFO] raft: pipelining replication to peer {Voter 0 0} + logger.go:44: 10:44:57.005410 [DEBUG] raft-test: server 1: leadership: acquired + logger.go:44: 10:44:57.005683 [DEBUG] raft-test: elect: server 1: wait for other servers to become followers within 40ms + logger.go:44: 10:44:57.005723 [DEBUG] raft-test: elect: server 0: became follower + logger.go:44: 10:44:57.005744 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 10:44:57.005762 [DEBUG] raft-test: elect: done + logger.go:44: 10:44:57.005845 [DEBUG] raft-test: server 1: pipeline: append to 2: 1 entry [Command:term=5,index=5] + logger.go:44: 10:44:57.005898 [DEBUG] raft-test: server 1: pipeline: append to 0: 1 entry [Command:term=5,index=5] + logger.go:44: 10:44:57.005952 [DEBUG] raft-test: fsm 1: applied 2 + logger.go:44: 10:44:57.005995 [DEBUG] raft-test: close: start + logger.go:44: 10:44:57.006073 [INFO] raft: aborting pipeline replication to peer {Voter 0 0} + logger.go:44: 10:44:57.006114 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 10:44:57.006165 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 10:44:57.006222 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 10:44:57.006263 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 10:44:57.006285 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 10:44:57.006303 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 10:44:57.006322 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 10:44:57.006372 [DEBUG] raft-test: close: done --- PASS: TestControl_DeposeAfterCommandCommittedThenElect/2 (0.08s) - logger.go:44: 17:00:30.237821 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 17:00:30.237886 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 17:00:30.237913 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 17:00:30.237935 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 17:00:30.237955 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 17:00:30.237992 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:30.238011 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 17:00:30.238027 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:30.238042 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:30.238055 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 17:00:30.238083 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:30.238094 [DEBUG] raft-test: setup: done - logger.go:44: 17:00:30.238105 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:30.238106 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 17:00:30.238138 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 17:00:30.238148 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 17:00:30.238112 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:30.267582 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 17:00:30.267605 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 2 - logger.go:44: 17:00:30.267625 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:30.267632 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 1 - logger.go:44: 17:00:30.267660 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 17:00:30.267661 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 17:00:30.267816 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 17:00:30.267830 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 2 - logger.go:44: 17:00:30.267845 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:30.267852 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 1 - logger.go:44: 17:00:30.267861 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 17:00:30.267863 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 17:00:30.268060 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 17:00:30.268071 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 17:00:30.268086 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:30.268093 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 17:00:30.268102 [INFO] raft: Duplicate RequestVote for same term: 2 - logger.go:44: 17:00:30.268105 [INFO] raft: Duplicate RequestVote for same term: 2 - logger.go:44: 17:00:30.287558 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 17:00:30.287575 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 3 - logger.go:44: 17:00:30.287587 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:30.287599 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 1 - logger.go:44: 17:00:30.287624 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:30.287634 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 2 - logger.go:44: 17:00:30.287645 [INFO] raft: Election won. Tally: 2 - logger.go:44: 17:00:30.287654 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 17:00:30.287663 [INFO] raft: Added peer 1, starting replication - logger.go:44: 17:00:30.287670 [INFO] raft: Added peer 2, starting replication - logger.go:44: 17:00:30.287697 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 17:00:30.287723 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 17:00:30.287627 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:30.287699 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=3,index=2] - logger.go:44: 17:00:30.287779 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 17:00:30.287713 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=3,index=2] - logger.go:44: 17:00:30.287807 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 17:00:30.288893 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 17:00:30.288902 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 17:00:30.288907 [DEBUG] raft-test: elect: done - logger.go:44: 17:00:30.288943 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=3,index=3] - logger.go:44: 17:00:30.288954 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=3,index=3] - logger.go:44: 17:00:30.288981 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 17:00:30.289002 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 17:00:30.289019 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 17:00:30.290181 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 17:00:30.290192 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 17:00:30.290200 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 17:00:30.290208 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 17:00:30.290376 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:30.290385 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:30.290392 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 17:00:30.290521 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:30.290529 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:30.290534 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 17:00:30.291120 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 17:00:30.291134 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected - logger.go:44: 17:00:30.291146 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 17:00:30.291161 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 17:00:30.292344 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:30.292363 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:30.292370 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 17:00:30.293315 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:30.293329 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:30.293338 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 17:00:30.302560 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:30.302574 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:30.302581 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 17:00:30.303171 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:30.303186 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:30.303193 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 17:00:30.303922 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:30.303936 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:30.303945 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 17:00:30.305453 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:30.305468 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:30.305477 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 17:00:30.308121 [WARN] raft: Failed to contact 1 in 19.136183ms - logger.go:44: 17:00:30.308161 [WARN] raft: Failed to contact 2 in 19.122918ms - logger.go:44: 17:00:30.308167 [WARN] raft: Failed to contact quorum of nodes, stepping down - logger.go:44: 17:00:30.308185 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:30.308195 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 17:00:30.308202 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 17:00:30.308222 [DEBUG] raft-test: elect: start (server 2) - logger.go:44: 17:00:30.308229 [DEBUG] raft-test: server 2: establish outbound connection to all other nodes - logger.go:44: 17:00:30.308238 [DEBUG] raft-test: elect: server 2: wait to become leader within 1.5s - logger.go:44: 17:00:30.310348 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 17:00:30.310364 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 4 - logger.go:44: 17:00:30.310380 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:30.310387 [DEBUG] raft: Vote granted from 1 in term 4. Tally: 1 - logger.go:44: 17:00:30.310398 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 17:00:30.310401 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 17:00:30.312392 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 17:00:30.312409 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 4 - logger.go:44: 17:00:30.312420 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:30.312427 [DEBUG] raft: Vote granted from 2 in term 4. Tally: 1 - logger.go:44: 17:00:30.312439 [INFO] raft: Duplicate RequestVote for same term: 4 - logger.go:44: 17:00:30.312457 [DEBUG] raft: Vote granted from 0 in term 4. Tally: 2 - logger.go:44: 17:00:30.312469 [INFO] raft: Election won. Tally: 2 - logger.go:44: 17:00:30.312477 [INFO] raft: Node at 2 [Leader] entering Leader state - logger.go:44: 17:00:30.312485 [INFO] raft: Added peer 0, starting replication - logger.go:44: 17:00:30.312494 [INFO] raft: Added peer 1, starting replication - logger.go:44: 17:00:30.312523 [DEBUG] raft-test: server 2: leadership: acquired - logger.go:44: 17:00:30.312543 [DEBUG] raft-test: server 2: transport: append to 1: 1 entry [Noop:term=4,index=4] - logger.go:44: 17:00:30.312553 [DEBUG] raft-test: server 2: transport: append to 0: 1 entry [Noop:term=4,index=4] - logger.go:44: 17:00:30.312569 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "2") - logger.go:44: 17:00:30.312579 [INFO] raft: pipelining replication to peer {Voter 0 0} - logger.go:44: 17:00:30.312586 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 17:00:30.312596 [DEBUG] raft-test: fsm 2: applied 1 - logger.go:44: 17:00:30.312549 [DEBUG] raft-test: elect: server 2: wait for other servers to become followers within 40ms - logger.go:44: 17:00:30.312623 [DEBUG] raft-test: elect: server 0: became follower - logger.go:44: 17:00:30.312628 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 17:00:30.312636 [DEBUG] raft-test: elect: done - logger.go:44: 17:00:30.312654 [DEBUG] raft-test: server 2: pipeline: append to 1: 1 entry [Command:term=4,index=5] - logger.go:44: 17:00:30.312674 [DEBUG] raft-test: server 2: pipeline: append to 0: 1 entry [Command:term=4,index=5] - logger.go:44: 17:00:30.312698 [DEBUG] raft-test: fsm 2: applied 2 - logger.go:44: 17:00:30.312711 [DEBUG] raft-test: close: start - logger.go:44: 17:00:30.312713 [DEBUG] raft-test: fsm 1: applied 1 - logger.go:44: 17:00:30.312748 [INFO] raft: aborting pipeline replication to peer {Voter 0 0} - logger.go:44: 17:00:30.312762 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 17:00:30.312780 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 17:00:30.312797 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:30.312803 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 17:00:30.312810 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 17:00:30.312819 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:30.312825 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:30.312831 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 17:00:30.312840 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 17:00:30.312852 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 17:00:30.312859 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 17:00:30.312864 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 17:00:30.312876 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 17:00:30.312880 [DEBUG] raft-test: close: done + logger.go:44: 10:44:57.006531 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 10:44:57.006632 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 10:44:57.006696 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 10:44:57.006739 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 10:44:57.006817 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 10:44:57.006917 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:57.006942 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 10:44:57.006992 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:57.007014 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 10:44:57.007090 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:57.007110 [DEBUG] raft-test: setup: done + logger.go:44: 10:44:57.007130 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 10:44:57.007164 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 10:44:57.007195 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 10:44:57.007246 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:57.007295 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:57.007340 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:57.029857 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 10:44:57.029954 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 2 + logger.go:44: 10:44:57.030008 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:57.030062 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 10:44:57.030066 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 1 + logger.go:44: 10:44:57.030088 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 10:44:57.031033 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 10:44:57.031070 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 2 + logger.go:44: 10:44:57.031123 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:57.031144 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 1 + logger.go:44: 10:44:57.031174 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 10:44:57.031189 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 10:44:57.031298 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 10:44:57.031327 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 10:44:57.031384 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:57.031411 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 10:44:57.031443 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 10:44:57.031459 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 10:44:57.049851 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 10:44:57.049906 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 + logger.go:44: 10:44:57.049970 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:57.049992 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 + logger.go:44: 10:44:57.050024 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 10:44:57.050032 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 10:44:57.055116 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 10:44:57.055159 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 3 + logger.go:44: 10:44:57.055195 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:57.055216 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 1 + logger.go:44: 10:44:57.055257 [INFO] raft: Duplicate RequestVote for same term: 3 + logger.go:44: 10:44:57.055316 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:57.055344 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 2 + logger.go:44: 10:44:57.055361 [INFO] raft: Election won. Tally: 2 + logger.go:44: 10:44:57.055380 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 10:44:57.055410 [INFO] raft: Added peer 1, starting replication + logger.go:44: 10:44:57.055435 [INFO] raft: Added peer 2, starting replication + logger.go:44: 10:44:57.055529 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 10:44:57.055593 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 10:44:57.055547 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=3,index=2] + logger.go:44: 10:44:57.055600 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=3,index=2] + logger.go:44: 10:44:57.055691 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 10:44:57.055663 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "0") + logger.go:44: 10:44:57.055743 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 10:44:57.056760 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 10:44:57.056785 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 10:44:57.056800 [DEBUG] raft-test: elect: done + logger.go:44: 10:44:57.056878 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=3,index=3] + logger.go:44: 10:44:57.056926 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=3,index=3] + logger.go:44: 10:44:57.057009 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 10:44:57.057052 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 10:44:57.057122 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 10:44:57.057538 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:57.057565 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:57.057586 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 10:44:57.057836 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:57.057876 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:57.057895 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 10:44:57.058734 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 10:44:57.058741 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 10:44:57.058779 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 10:44:57.058831 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 10:44:57.058857 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 10:44:57.058780 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected + logger.go:44: 10:44:57.058923 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 10:44:57.058952 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 10:44:57.060310 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:57.060337 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:57.060361 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 10:44:57.060594 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:57.060616 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:57.060642 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 10:44:57.070131 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:57.070197 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:57.070220 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 10:44:57.070251 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:57.070270 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:57.070287 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 10:44:57.072136 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:57.072176 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:57.072202 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 10:44:57.072389 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:57.072423 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:57.072441 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 10:44:57.075903 [WARN] raft: Failed to contact 1 in 18.610697ms + logger.go:44: 10:44:57.075935 [WARN] raft: Failed to contact 2 in 18.890468ms + logger.go:44: 10:44:57.075953 [WARN] raft: Failed to contact quorum of nodes, stepping down + logger.go:44: 10:44:57.075996 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:57.076025 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 10:44:57.076045 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 10:44:57.076110 [DEBUG] raft-test: elect: start (server 2) + logger.go:44: 10:44:57.076134 [DEBUG] raft-test: server 2: establish outbound connection to all other nodes + logger.go:44: 10:44:57.076161 [DEBUG] raft-test: elect: server 2: wait to become leader within 1.5s + logger.go:44: 10:44:57.081675 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 10:44:57.081732 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 4 + logger.go:44: 10:44:57.081766 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:57.081786 [DEBUG] raft: Vote granted from 2 in term 4. Tally: 1 + logger.go:44: 10:44:57.081857 [WARN] raft: Rejecting vote request from 2 since we have a leader: 0 + logger.go:44: 10:44:57.081915 [DEBUG] raft: Vote granted from 0 in term 4. Tally: 2 + logger.go:44: 10:44:57.081931 [INFO] raft: Election won. Tally: 2 + logger.go:44: 10:44:57.081951 [INFO] raft: Node at 2 [Leader] entering Leader state + logger.go:44: 10:44:57.081978 [INFO] raft: Added peer 0, starting replication + logger.go:44: 10:44:57.082003 [INFO] raft: Added peer 1, starting replication + logger.go:44: 10:44:57.082118 [DEBUG] raft-test: server 2: transport: append to 1: 1 entry [Noop:term=4,index=4] + logger.go:44: 10:44:57.082129 [DEBUG] raft-test: server 2: leadership: acquired + logger.go:44: 10:44:57.082186 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 10:44:57.082192 [DEBUG] raft-test: elect: server 2: wait for other servers to become followers within 40ms + logger.go:44: 10:44:57.082279 [DEBUG] raft-test: fsm 2: applied 1 + logger.go:44: 10:44:57.082344 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:57.082383 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:57.082408 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 10:44:57.082357 [DEBUG] raft-test: server 2: transport: append to 0: 1 entry [Noop:term=4,index=4] + logger.go:44: 10:44:57.082594 [INFO] raft: pipelining replication to peer {Voter 0 0} + logger.go:44: 10:44:57.083447 [DEBUG] raft-test: elect: server 0: became follower + logger.go:44: 10:44:57.083482 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 10:44:57.083533 [DEBUG] raft-test: elect: done + logger.go:44: 10:44:57.083462 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:57.083585 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:57.083602 [DEBUG] raft-test: server 2: pipeline: append to 1: 1 entry [Command:term=4,index=5] + logger.go:44: 10:44:57.083630 [DEBUG] raft-test: server 2: pipeline: append to 0: 1 entry [Command:term=4,index=5] + logger.go:44: 10:44:57.083604 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 10:44:57.083697 [DEBUG] raft-test: fsm 2: applied 2 + logger.go:44: 10:44:57.083734 [DEBUG] raft-test: close: start + logger.go:44: 10:44:57.083738 [DEBUG] raft-test: fsm 1: applied 1 + logger.go:44: 10:44:57.083792 [INFO] raft: aborting pipeline replication to peer {Voter 0 0} + logger.go:44: 10:44:57.083803 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 10:44:57.083906 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 10:44:57.083951 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 10:44:57.083990 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:57.084009 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:57.084031 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 10:44:57.084069 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:57.084089 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:57.084110 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 10:44:57.084146 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 10:44:57.084185 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 10:44:57.084227 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 10:44:57.084277 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 10:44:57.084363 [DEBUG] raft-test: close: done === RUN TestControl_SnapshotAfterCommandCommitted --- PASS: TestControl_SnapshotAfterCommandCommitted (0.02s) - logger.go:44: 17:00:30.312986 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 17:00:30.313028 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 17:00:30.313045 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 17:00:30.313059 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 17:00:30.313070 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 17:00:30.313093 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:30.313103 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 17:00:30.313121 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:30.313128 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 17:00:30.313145 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:30.313148 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:30.313154 [DEBUG] raft-test: setup: done - logger.go:44: 17:00:30.313184 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 17:00:30.313198 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 17:00:30.313128 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:30.313185 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:30.313212 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 17:00:30.333832 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 17:00:30.333850 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 2 - logger.go:44: 17:00:30.333868 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:30.333876 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 1 - logger.go:44: 17:00:30.333887 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 17:00:30.333887 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 17:00:30.335045 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 17:00:30.335060 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 17:00:30.335072 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:30.335079 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 17:00:30.335091 [INFO] raft: Duplicate RequestVote for same term: 2 - logger.go:44: 17:00:30.335108 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 2 - logger.go:44: 17:00:30.335123 [INFO] raft: Election won. Tally: 2 - logger.go:44: 17:00:30.335136 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 17:00:30.335153 [INFO] raft: Added peer 1, starting replication - logger.go:44: 17:00:30.335161 [INFO] raft: Added peer 2, starting replication - logger.go:44: 17:00:30.335196 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] - logger.go:44: 17:00:30.335212 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "0") - logger.go:44: 17:00:30.335226 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 17:00:30.335255 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 17:00:30.335284 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 17:00:30.335256 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] - logger.go:44: 17:00:30.335344 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 17:00:30.336494 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 17:00:30.336508 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 17:00:30.336518 [DEBUG] raft-test: elect: done - logger.go:44: 17:00:30.336558 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] - logger.go:44: 17:00:30.336566 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] - logger.go:44: 17:00:30.336599 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 17:00:30.336635 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 17:00:30.336650 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=4] - logger.go:44: 17:00:30.336673 [DEBUG] raft-test: fsm 0: applied 2 - logger.go:44: 17:00:30.336678 [DEBUG] raft-test: server 0: control: take snapshot - logger.go:44: 17:00:30.336695 [INFO] raft: Starting snapshot up to 4 - logger.go:44: 17:00:30.336730 [INFO] raft: Compacting logs from 1 to 3 - logger.go:44: 17:00:30.336737 [INFO] raft: Snapshot to 4 complete - logger.go:44: 17:00:30.336755 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Other:term=2,index=5] - logger.go:44: 17:00:30.336695 [DEBUG] raft-test: fsm 2: applied 1 - logger.go:44: 17:00:30.336697 [DEBUG] raft-test: fsm 1: applied 1 - logger.go:44: 17:00:30.336794 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Other:term=2,index=5] - logger.go:44: 17:00:30.336804 [DEBUG] raft-test: fsm 2: applied 2 - logger.go:44: 17:00:30.336821 [DEBUG] raft-test: fsm 1: applied 2 - logger.go:44: 17:00:30.336839 [DEBUG] raft-test: close: start - logger.go:44: 17:00:30.336872 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 17:00:30.336892 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 17:00:30.336910 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 17:00:30.336927 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 17:00:30.336938 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 17:00:30.336944 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 17:00:30.336956 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 17:00:30.336970 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 17:00:30.336976 [DEBUG] raft-test: close: done + logger.go:44: 10:44:57.084635 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 10:44:57.084746 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 10:44:57.084796 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 10:44:57.084881 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 10:44:57.084915 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 10:44:57.084982 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:57.085025 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 10:44:57.085078 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:57.085084 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:57.085126 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 10:44:57.085181 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:57.085251 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:57.085308 [DEBUG] raft-test: setup: done + logger.go:44: 10:44:57.085354 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:57.085355 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 10:44:57.085413 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 10:44:57.085440 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 10:44:57.102529 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 10:44:57.102595 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 10:44:57.102624 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:57.102639 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 10:44:57.102681 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 2 + logger.go:44: 10:44:57.102692 [INFO] raft: Election won. Tally: 2 + logger.go:44: 10:44:57.102706 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 10:44:57.102730 [INFO] raft: Added peer 1, starting replication + logger.go:44: 10:44:57.102747 [INFO] raft: Added peer 2, starting replication + logger.go:44: 10:44:57.102864 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] + logger.go:44: 10:44:57.103030 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 10:44:57.103074 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 10:44:57.103128 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 10:44:57.103189 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] + logger.go:44: 10:44:57.103251 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 10:44:57.104235 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 10:44:57.104376 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 10:44:57.104413 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 10:44:57.104430 [DEBUG] raft-test: elect: done + logger.go:44: 10:44:57.104535 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] + logger.go:44: 10:44:57.104607 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] + logger.go:44: 10:44:57.104634 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 10:44:57.104717 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=4] + logger.go:44: 10:44:57.104717 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=4] + logger.go:44: 10:44:57.104844 [DEBUG] raft-test: fsm 0: applied 2 + logger.go:44: 10:44:57.104856 [DEBUG] raft-test: fsm 1: applied 1 + logger.go:44: 10:44:57.104862 [DEBUG] raft-test: server 0: control: take snapshot + logger.go:44: 10:44:57.104870 [DEBUG] raft-test: fsm 2: applied 1 + logger.go:44: 10:44:57.105006 [INFO] raft: Starting snapshot up to 4 + logger.go:44: 10:44:57.105084 [INFO] raft: Compacting logs from 1 to 3 + logger.go:44: 10:44:57.105138 [INFO] raft: Snapshot to 4 complete + logger.go:44: 10:44:57.105266 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Other:term=2,index=5] + logger.go:44: 10:44:57.105335 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Other:term=2,index=5] + logger.go:44: 10:44:57.105411 [DEBUG] raft-test: fsm 1: applied 2 + logger.go:44: 10:44:57.105435 [DEBUG] raft-test: fsm 2: applied 2 + logger.go:44: 10:44:57.105470 [DEBUG] raft-test: close: start + logger.go:44: 10:44:57.105495 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:57.105522 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 10:44:57.105522 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 10:44:57.105603 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 10:44:57.105646 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 10:44:57.105758 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 10:44:57.105776 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 10:44:57.105801 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 10:44:57.105840 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 10:44:57.105851 [DEBUG] raft-test: close: done === RUN TestControl_RestoreAfterDisconnection ---- PASS: TestControl_RestoreAfterDisconnection (0.04s) - logger.go:44: 17:00:30.337017 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 17:00:30.337061 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 17:00:30.337078 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 17:00:30.337089 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 17:00:30.337102 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 17:00:30.337126 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:30.337138 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 17:00:30.337157 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:30.337162 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:30.337177 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:30.337163 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 17:00:30.337203 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 17:00:30.337211 [DEBUG] raft-test: setup: done - logger.go:44: 17:00:30.337217 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 17:00:30.337224 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 17:00:30.337235 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 17:00:30.337228 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:30.357618 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 17:00:30.357643 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 2 - logger.go:44: 17:00:30.357665 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:30.357690 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 1 - logger.go:44: 17:00:30.357694 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 17:00:30.357702 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 17:00:30.360422 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 17:00:30.360438 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 17:00:30.360452 [DEBUG] raft: Votes needed: 2 - logger.go:44: 17:00:30.360459 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 17:00:30.360472 [INFO] raft: Duplicate RequestVote for same term: 2 - logger.go:44: 17:00:30.360510 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 2 - logger.go:44: 17:00:30.360521 [INFO] raft: Election won. Tally: 2 - logger.go:44: 17:00:30.360530 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 17:00:30.360541 [INFO] raft: Added peer 1, starting replication - logger.go:44: 17:00:30.360549 [INFO] raft: Added peer 2, starting replication - logger.go:44: 17:00:30.360585 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] - logger.go:44: 17:00:30.360602 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "0") - logger.go:44: 17:00:30.360616 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 17:00:30.360635 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 17:00:30.360655 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 17:00:30.360648 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] - logger.go:44: 17:00:30.360700 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 17:00:30.361916 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 17:00:30.361950 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 17:00:30.361956 [DEBUG] raft-test: elect: done - logger.go:44: 17:00:30.361997 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] - logger.go:44: 17:00:30.362005 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] - logger.go:44: 17:00:30.362025 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 17:00:30.362047 [DEBUG] raft-test: term: disconnect 1 - logger.go:44: 17:00:30.362052 [DEBUG] raft-test: server 0: disconnecting follower 1 - logger.go:44: 17:00:30.362076 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 17:00:30.362088 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=4] - logger.go:44: 17:00:30.362099 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 17:00:30.362107 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 17:00:30.362115 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 17:00:30.362123 [DEBUG] raft-test: fsm 0: applied 2 - logger.go:44: 17:00:30.362126 [DEBUG] raft-test: fsm 2: applied 1 - logger.go:44: 17:00:30.362156 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=5] - logger.go:44: 17:00:30.362162 [DEBUG] raft-test: server 0: transport: append to 1: 2 entries [Command:term=2,index=4 Command:term=2,index=5] - logger.go:44: 17:00:30.362213 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:30.362221 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 17:00:30.362230 [DEBUG] raft-test: fsm 2: applied 2 - logger.go:44: 17:00:30.362242 [DEBUG] raft-test: fsm 0: applied 3 - logger.go:44: 17:00:30.362268 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=6] - logger.go:44: 17:00:30.362292 [DEBUG] raft-test: fsm 0: applied 4 - logger.go:44: 17:00:30.362298 [DEBUG] raft-test: server 0: control: take snapshot - logger.go:44: 17:00:30.362305 [DEBUG] raft-test: fsm 2: applied 3 - logger.go:44: 17:00:30.362317 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=7] - logger.go:44: 17:00:30.362329 [INFO] raft: Starting snapshot up to 6 - logger.go:44: 17:00:30.362364 [INFO] raft: Compacting logs from 1 to 6 - logger.go:44: 17:00:30.362370 [INFO] raft: Snapshot to 6 complete - logger.go:44: 17:00:30.362392 [DEBUG] raft-test: fsm 2: applied 4 - logger.go:44: 17:00:30.362395 [DEBUG] raft-test: fsm 0: applied 5 - logger.go:44: 17:00:30.362419 [DEBUG] raft-test: server 0: reconnecting follower 1 - logger.go:44: 17:00:30.362427 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 17:00:30.362439 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 17:00:30.362444 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 17:00:30.362450 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:30.362465 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=8] - logger.go:44: 17:00:30.362491 [DEBUG] raft-test: fsm 0: applied 6 - logger.go:44: 17:00:30.362509 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Other:term=2,index=9] - logger.go:44: 17:00:30.362524 [DEBUG] raft-test: fsm 2: applied 5 - logger.go:44: 17:00:30.362555 [DEBUG] raft-test: fsm 2: applied 6 - logger.go:44: 17:00:30.363539 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:30.364133 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 17:00:30.364395 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:30.365833 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 17:00:30.366131 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:30.366635 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:30.367080 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 17:00:30.367966 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:30.369136 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 17:00:30.369776 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:30.369793 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:30.370993 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 17:00:30.371758 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 17:00:30.372469 [ERR] raft: Failed to get log at index 3: log not found - logger.go:44: 17:00:30.372479 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 17:00:30.372526 [INFO] raft: Copied 8 bytes to local snapshot - logger.go:44: 17:00:30.372543 [INFO] raft: Compacting logs from 1 to 2 - logger.go:44: 17:00:30.372549 [INFO] raft: Installed remote snapshot - logger.go:44: 17:00:30.372580 [DEBUG] raft-test: server 0: transport: append to 1: 3 entries [Command:term=2,index=7 Command:term=2,index=8 Other:term=2,index=9] - logger.go:44: 17:00:30.372615 [DEBUG] raft-test: fsm 1: applied 5 - logger.go:44: 17:00:30.372644 [DEBUG] raft-test: fsm 1: applied 6 - logger.go:44: 17:00:30.372662 [DEBUG] raft-test: close: start - logger.go:44: 17:00:30.372615 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 17:00:30.372693 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:30.372704 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 17:00:30.372701 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 17:00:30.372731 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 17:00:30.372753 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 17:00:30.372765 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 17:00:30.372774 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 17:00:30.372780 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 17:00:30.372792 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 17:00:30.372805 [DEBUG] raft-test: close: done +--- PASS: TestControl_RestoreAfterDisconnection (0.15s) + logger.go:44: 10:44:57.105942 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 10:44:57.106027 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 10:44:57.106064 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 10:44:57.106126 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 10:44:57.106185 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 10:44:57.106238 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:57.106268 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 10:44:57.106307 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:57.106322 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 10:44:57.106321 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:57.106359 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 10:44:57.106409 [DEBUG] raft-test: setup: done + logger.go:44: 10:44:57.106431 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 10:44:57.106459 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 10:44:57.106487 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 10:44:57.106505 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:57.106382 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:57.123635 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 10:44:57.123682 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 2 + logger.go:44: 10:44:57.123743 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:57.123762 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 1 + logger.go:44: 10:44:57.123794 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 10:44:57.123854 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 10:44:57.125153 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 10:44:57.125179 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 2 + logger.go:44: 10:44:57.125211 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:57.125232 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 1 + logger.go:44: 10:44:57.125271 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 10:44:57.125315 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 10:44:57.136414 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 10:44:57.136453 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 10:44:57.136500 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:57.136516 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 10:44:57.136560 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 10:44:57.136580 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 10:44:57.141457 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 10:44:57.141513 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 3 + logger.go:44: 10:44:57.141556 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:57.141584 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 1 + logger.go:44: 10:44:57.141632 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 10:44:57.141699 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 10:44:57.153183 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 10:44:57.153240 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 + logger.go:44: 10:44:57.153291 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:57.153316 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 + logger.go:44: 10:44:57.153370 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 10:44:57.153440 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 10:44:57.158048 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 10:44:57.158094 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 3 + logger.go:44: 10:44:57.158157 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:57.158184 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 1 + logger.go:44: 10:44:57.158234 [INFO] raft: Duplicate RequestVote for same term: 3 + logger.go:44: 10:44:57.158262 [INFO] raft: Duplicate RequestVote for same term: 3 + logger.go:44: 10:44:57.169650 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 10:44:57.169739 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 4 + logger.go:44: 10:44:57.169830 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:57.169850 [DEBUG] raft: Vote granted from 1 in term 4. Tally: 1 + logger.go:44: 10:44:57.169868 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 10:44:57.169887 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 10:44:57.176659 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 10:44:57.176696 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 4 + logger.go:44: 10:44:57.176737 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:57.176758 [DEBUG] raft: Vote granted from 2 in term 4. Tally: 1 + logger.go:44: 10:44:57.176791 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 10:44:57.176798 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 10:44:57.185991 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 10:44:57.186029 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 4 + logger.go:44: 10:44:57.186095 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:57.186120 [DEBUG] raft: Vote granted from 0 in term 4. Tally: 1 + logger.go:44: 10:44:57.186164 [INFO] raft: Duplicate RequestVote for same term: 4 + logger.go:44: 10:44:57.186215 [INFO] raft: Duplicate RequestVote for same term: 4 + logger.go:44: 10:44:57.192425 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 10:44:57.192485 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 5 + logger.go:44: 10:44:57.192547 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:57.192576 [DEBUG] raft: Vote granted from 1 in term 5. Tally: 1 + logger.go:44: 10:44:57.192614 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 10:44:57.192616 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 10:44:57.206766 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 10:44:57.206812 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 5 + logger.go:44: 10:44:57.206936 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:57.206964 [DEBUG] raft: Vote granted from 2 in term 5. Tally: 1 + logger.go:44: 10:44:57.207004 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 10:44:57.207052 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 10:44:57.208906 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 10:44:57.208943 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 6 + logger.go:44: 10:44:57.209111 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:57.209131 [DEBUG] raft: Vote granted from 1 in term 6. Tally: 1 + logger.go:44: 10:44:57.209153 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 10:44:57.209178 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 10:44:57.211772 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 10:44:57.211842 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 5 + logger.go:44: 10:44:57.211906 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:57.211929 [DEBUG] raft: Vote granted from 0 in term 5. Tally: 1 + logger.go:44: 10:44:57.211970 [DEBUG] raft: Newer term discovered, fallback to follower + logger.go:44: 10:44:57.211984 [INFO] raft: Duplicate RequestVote for same term: 5 + logger.go:44: 10:44:57.212004 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:57.232879 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 10:44:57.232934 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 6 + logger.go:44: 10:44:57.233011 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:57.233038 [DEBUG] raft: Vote granted from 2 in term 6. Tally: 1 + logger.go:44: 10:44:57.233080 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 10:44:57.233129 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 10:44:57.236190 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 10:44:57.236239 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 7 + logger.go:44: 10:44:57.236325 [DEBUG] raft: Votes needed: 2 + logger.go:44: 10:44:57.236359 [DEBUG] raft: Vote granted from 0 in term 7. Tally: 1 + logger.go:44: 10:44:57.236439 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:57.236456 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:57.236483 [DEBUG] raft: Vote granted from 1 in term 7. Tally: 2 + logger.go:44: 10:44:57.236573 [INFO] raft: Election won. Tally: 2 + logger.go:44: 10:44:57.236606 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 10:44:57.236694 [INFO] raft: Added peer 1, starting replication + logger.go:44: 10:44:57.236735 [INFO] raft: Added peer 2, starting replication + logger.go:44: 10:44:57.236748 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 10:44:57.236827 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 10:44:57.236914 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=7,index=2] + logger.go:44: 10:44:57.237001 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=7,index=2] + logger.go:44: 10:44:57.237020 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 10:44:57.237166 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 10:44:57.238113 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 10:44:57.238146 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 10:44:57.238171 [DEBUG] raft-test: elect: done + logger.go:44: 10:44:57.238321 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=7,index=3] + logger.go:44: 10:44:57.238323 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=7,index=3] + logger.go:44: 10:44:57.238474 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 10:44:57.238605 [DEBUG] raft-test: term: disconnect 1 + logger.go:44: 10:44:57.238632 [DEBUG] raft-test: server 0: disconnecting follower 1 + logger.go:44: 10:44:57.238740 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=7,index=4] + logger.go:44: 10:44:57.238743 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=7,index=4] + logger.go:44: 10:44:57.238855 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 10:44:57.238889 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 10:44:57.238927 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 10:44:57.238972 [DEBUG] raft-test: fsm 0: applied 2 + logger.go:44: 10:44:57.238990 [DEBUG] raft-test: fsm 2: applied 1 + logger.go:44: 10:44:57.239130 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=7,index=5] + logger.go:44: 10:44:57.239174 [DEBUG] raft-test: server 0: transport: append to 1: 2 entries [Command:term=7,index=4 Command:term=7,index=5] + logger.go:44: 10:44:57.239223 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:57.239258 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 10:44:57.239278 [DEBUG] raft-test: fsm 0: applied 3 + logger.go:44: 10:44:57.239309 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:57.239293 [DEBUG] raft-test: fsm 2: applied 2 + logger.go:44: 10:44:57.239231 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:57.239453 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:57.239478 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 10:44:57.239438 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=7,index=6] + logger.go:44: 10:44:57.239618 [DEBUG] raft-test: fsm 0: applied 4 + logger.go:44: 10:44:57.239651 [DEBUG] raft-test: server 0: control: take snapshot + logger.go:44: 10:44:57.239724 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=7,index=7] + logger.go:44: 10:44:57.239776 [DEBUG] raft-test: fsm 2: applied 3 + logger.go:44: 10:44:57.239827 [INFO] raft: Starting snapshot up to 6 + logger.go:44: 10:44:57.239915 [INFO] raft: Compacting logs from 1 to 6 + logger.go:44: 10:44:57.239939 [INFO] raft: Snapshot to 6 complete + logger.go:44: 10:44:57.240020 [DEBUG] raft-test: fsm 0: applied 5 + logger.go:44: 10:44:57.240058 [DEBUG] raft-test: server 0: reconnecting follower 1 + logger.go:44: 10:44:57.240083 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 10:44:57.240120 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 10:44:57.240142 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 10:44:57.240196 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=7,index=8] + logger.go:44: 10:44:57.240245 [DEBUG] raft-test: fsm 2: applied 4 + logger.go:44: 10:44:57.240324 [DEBUG] raft-test: fsm 0: applied 6 + logger.go:44: 10:44:57.240414 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Other:term=7,index=9] + logger.go:44: 10:44:57.240465 [DEBUG] raft-test: fsm 2: applied 5 + logger.go:44: 10:44:57.240563 [DEBUG] raft-test: fsm 2: applied 6 + logger.go:44: 10:44:57.241929 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:57.242041 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 10:44:57.243714 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 10:44:57.244705 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:57.247040 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 10:44:57.248605 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:57.248642 [WARN] raft: Failed to contact 1 in 10.169371ms + logger.go:44: 10:44:57.248836 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 10:44:57.250550 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 10:44:57.250732 [ERR] raft: Failed to get log at index 3: log not found + logger.go:44: 10:44:57.250876 [INFO] raft: Copied 8 bytes to local snapshot + logger.go:44: 10:44:57.250935 [INFO] raft: Compacting logs from 1 to 2 + logger.go:44: 10:44:57.250954 [INFO] raft: Installed remote snapshot + logger.go:44: 10:44:57.251036 [DEBUG] raft-test: server 0: transport: append to 1: 3 entries [Command:term=7,index=7 Command:term=7,index=8 Other:term=7,index=9] + logger.go:44: 10:44:57.251135 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 10:44:57.251147 [DEBUG] raft-test: fsm 1: applied 5 + logger.go:44: 10:44:57.251213 [DEBUG] raft-test: fsm 1: applied 6 + logger.go:44: 10:44:57.251266 [DEBUG] raft-test: close: start + logger.go:44: 10:44:57.251330 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 10:44:57.251343 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 10:44:57.251356 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 10:44:57.251456 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 10:44:57.251509 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 10:44:57.251556 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 10:44:57.251583 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 10:44:57.251605 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 10:44:57.251647 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 10:44:57.251725 [DEBUG] raft-test: close: done === RUN TestFSM_Restore --- PASS: TestFSM_Restore (0.00s) === RUN TestServer_StartAndShutdown --- PASS: TestServer_StartAndShutdown (0.02s) - logger.go:44: 17:00:30.372873 [DEBUG] raft-test: setup: start (1 servers) - logger.go:44: 17:00:30.372901 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 17:00:30.372916 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 17:00:30.372935 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0}] - logger.go:44: 17:00:30.372943 [DEBUG] raft-test: setup: done - logger.go:44: 17:00:30.372949 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 17:00:30.372958 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 17:00:30.372965 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 17:00:30.372991 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 17:00:30.393509 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 17:00:30.393526 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 17:00:30.393536 [DEBUG] raft: Votes needed: 1 - logger.go:44: 17:00:30.393543 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 17:00:30.393547 [INFO] raft: Election won. Tally: 1 - logger.go:44: 17:00:30.393554 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 17:00:30.393584 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 17:00:30.393600 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 17:00:30.393611 [DEBUG] raft-test: elect: done - logger.go:44: 17:00:30.393633 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 17:00:30.393649 [DEBUG] raft-test: close: start - logger.go:44: 17:00:30.393677 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 17:00:30.393697 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 17:00:30.393703 [DEBUG] raft-test: close: done + logger.go:44: 10:44:57.251967 [DEBUG] raft-test: setup: start (1 servers) + logger.go:44: 10:44:57.252066 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 10:44:57.252113 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 10:44:57.252180 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0}] + logger.go:44: 10:44:57.252215 [DEBUG] raft-test: setup: done + logger.go:44: 10:44:57.252240 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 10:44:57.252272 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 10:44:57.252298 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 10:44:57.252248 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 10:44:57.274613 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 10:44:57.274662 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 10:44:57.274698 [DEBUG] raft: Votes needed: 1 + logger.go:44: 10:44:57.274729 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 10:44:57.274750 [INFO] raft: Election won. Tally: 1 + logger.go:44: 10:44:57.274773 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 10:44:57.274878 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 10:44:57.274958 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 10:44:57.275052 [DEBUG] raft-test: elect: done + logger.go:44: 10:44:57.275171 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 10:44:57.275248 [DEBUG] raft-test: close: start + logger.go:44: 10:44:57.275349 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 10:44:57.275426 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 10:44:57.275460 [DEBUG] raft-test: close: done === RUN ExampleControl_Commands ---- PASS: ExampleControl_Commands (0.17s) +--- PASS: ExampleControl_Commands (0.15s) === RUN ExampleControl_Snapshots --- PASS: ExampleControl_Snapshots (0.04s) PASS -ok github.com/CanonicalLtd/raft-test 1.180s +ok github.com/CanonicalLtd/raft-test 1.416s === RUN TestNotifier_Acquired --- PASS: TestNotifier_Acquired (0.00s) - logger.go:44: 17:00:29.357204 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 17:00:29.357284 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 10:44:57.010297 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 10:44:57.010496 [DEBUG] raft-test: server 0: leadership: stop watching === RUN TestNotifier_AcquiredTimeout --- PASS: TestNotifier_AcquiredTimeout (0.00s) - logger.go:44: 17:00:29.357404 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 10:44:57.010939 [DEBUG] raft-test: server 0: leadership: stop watching === RUN TestNotifier_LeaderhsipLost --- PASS: TestNotifier_LeaderhsipLost (0.00s) - logger.go:44: 17:00:29.357464 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 17:00:29.357484 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 17:00:29.357490 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 10:44:57.011238 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 10:44:57.011353 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 10:44:57.011396 [DEBUG] raft-test: server 0: leadership: stop watching === RUN TestNotifier_LostTwice --- PASS: TestNotifier_LostTwice (0.00s) - logger.go:44: 17:00:29.357521 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 17:00:29.357536 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 10:44:57.011595 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 10:44:57.011640 [DEBUG] raft-test: server 0: leadership: stop watching === RUN TestNotifier_UnexpectedLeadershipChange --- PASS: TestNotifier_UnexpectedLeadershipChange (0.00s) === RUN TestNotifier_InconsistentNotifications --- PASS: TestNotifier_InconsistentNotifications (0.01s) - logger.go:44: 17:00:29.367708 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 10:44:57.022156 [DEBUG] raft-test: server 0: leadership: acquired === RUN TestNotifier_DoubleAcquiredRequest --- PASS: TestNotifier_DoubleAcquiredRequest (0.00s) === RUN TestNotifier_DoubleNotification --- PASS: TestNotifier_DoubleNotification (0.01s) - logger.go:44: 17:00:29.377964 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 10:44:57.032711 [DEBUG] raft-test: server 0: leadership: acquired === RUN TestNotifier_OneLeadershipAfterTheOther --- PASS: TestNotifier_OneLeadershipAfterTheOther (0.00s) - logger.go:44: 17:00:29.378022 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 17:00:29.378041 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 17:00:29.378049 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 17:00:29.378057 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 10:44:57.033075 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 10:44:57.033293 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 10:44:57.033342 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 10:44:57.033376 [DEBUG] raft-test: server 0: leadership: stop watching === RUN TestTracker_Acquired --- PASS: TestTracker_Acquired (0.00s) - logger.go:44: 17:00:29.378183 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 17:00:29.378209 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 17:00:29.378218 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 17:00:29.378230 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 10:44:57.033706 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 10:44:57.033769 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 10:44:57.033791 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 10:44:57.033823 [DEBUG] raft-test: server 2: leadership: stop watching === RUN TestTracker_AcquiredTimeout --- PASS: TestTracker_AcquiredTimeout (0.00s) - logger.go:44: 17:00:29.378333 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 10:44:57.034155 [DEBUG] raft-test: server 0: leadership: stop watching === RUN TestTracker_AcquiredTwice --- PASS: TestTracker_AcquiredTwice (0.00s) - logger.go:44: 17:00:29.378415 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 17:00:29.378447 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 17:00:29.378469 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 17:00:29.378485 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 10:44:57.034357 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 10:44:57.034410 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 10:44:57.034437 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 10:44:57.034504 [DEBUG] raft-test: server 2: leadership: stop watching === RUN TestTracker_AcquiredAfterLost --- PASS: TestTracker_AcquiredAfterLost (0.00s) - logger.go:44: 17:00:29.378589 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 17:00:29.378634 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 17:00:29.378649 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 17:00:29.378663 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 17:00:29.378673 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 10:44:57.034684 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 10:44:57.034743 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 10:44:57.034764 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 10:44:57.034784 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 10:44:57.034814 [DEBUG] raft-test: server 2: leadership: stop watching === RUN TestTracker_AddSameServerID --- PASS: TestTracker_AddSameServerID (0.00s) - logger.go:44: 17:00:29.378726 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 10:44:57.034965 [DEBUG] raft-test: server 0: leadership: stop watching === RUN TestTracker_AddAfterObserving --- PASS: TestTracker_AddAfterObserving (0.00s) - logger.go:44: 17:00:29.378767 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 10:44:57.035081 [DEBUG] raft-test: server 0: leadership: stop watching PASS -ok github.com/CanonicalLtd/raft-test/internal/election 0.028s +ok github.com/CanonicalLtd/raft-test/internal/election 0.041s === RUN TestEvent --- PASS: TestEvent (0.00s) PASS -ok github.com/CanonicalLtd/raft-test/internal/event 0.007s +ok github.com/CanonicalLtd/raft-test/internal/event 0.010s ? github.com/CanonicalLtd/raft-test/internal/fsms [no test files] === RUN TestNew --- PASS: TestNew (0.00s) - logger.go:44: 17:00:28.340929 [TRACE] raft-test: hello + logger.go:44: 10:44:50.779213 [TRACE] raft-test: hello PASS -ok github.com/CanonicalLtd/raft-test/internal/logging 0.003s +ok github.com/CanonicalLtd/raft-test/internal/logging 0.020s === RUN TestStringifyLogs === RUN TestStringifyLogs/0_entries === RUN TestStringifyLogs/1_entry_[Noop:term=1,index=1] @@ -2573,43 +2728,43 @@ --- PASS: TestPeer_CommandLogsCount (0.00s) === RUN TestFaultyTransport_AppendEntries_Default --- PASS: TestFaultyTransport_AppendEntries_Default (0.00s) - logger.go:44: 17:00:29.310830 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 17:00:29.310885 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:57.164798 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 10:44:57.164906 [DEBUG] raft-test: server 0: transport: append to 1: not connected === RUN TestFaultyTransport_AppendEntries_Connected --- PASS: TestFaultyTransport_AppendEntries_Connected (0.00s) - logger.go:44: 17:00:29.310928 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=1,index=1] + logger.go:44: 10:44:57.165031 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=1,index=1] === RUN TestFaultyTransport_PipelineAppendEntries_Disconnected --- PASS: TestFaultyTransport_PipelineAppendEntries_Disconnected (0.00s) - logger.go:44: 17:00:29.311031 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 17:00:29.311052 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 10:44:57.165289 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 10:44:57.165339 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected === RUN TestFaultyTransport_PipelineAppendEntries_Connected --- PASS: TestFaultyTransport_PipelineAppendEntries_Connected (0.00s) - logger.go:44: 17:00:29.311093 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 10:44:57.165488 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries === RUN TestFaultyTransport_Deposing --- PASS: TestFaultyTransport_Deposing (0.00s) - logger.go:44: 17:00:29.311151 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=1,index=1] - logger.go:44: 17:00:29.311172 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=1,index=1] - logger.go:44: 17:00:29.311187 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=1,index=3] - logger.go:44: 17:00:29.311215 [DEBUG] raft-test: server 0: transport: append to 2: syncing logs - logger.go:44: 17:00:29.311224 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=1,index=3] - logger.go:44: 17:00:29.311260 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=1,index=3] - logger.go:44: 17:00:29.311272 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 17:00:29.311282 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=1,index=3] - logger.go:44: 17:00:29.311287 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected - logger.go:44: 17:00:29.311294 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=1,index=3] - logger.go:44: 17:00:29.311299 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 17:00:29.311308 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=1,index=3] - logger.go:44: 17:00:29.311313 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 10:44:57.165862 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=1,index=1] + logger.go:44: 10:44:57.165975 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=1,index=1] + logger.go:44: 10:44:57.166026 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=1,index=3] + logger.go:44: 10:44:57.166092 [DEBUG] raft-test: server 0: transport: append to 2: syncing logs + logger.go:44: 10:44:57.166123 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=1,index=3] + logger.go:44: 10:44:57.166176 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=1,index=3] + logger.go:44: 10:44:57.166194 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 10:44:57.166226 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=1,index=3] + logger.go:44: 10:44:57.166243 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected + logger.go:44: 10:44:57.166263 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=1,index=3] + logger.go:44: 10:44:57.166291 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 10:44:57.166320 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=1,index=3] + logger.go:44: 10:44:57.166338 [DEBUG] raft-test: server 0: transport: append to 2: not connected === RUN TestNetwork_FaultyEnqueue --- PASS: TestNetwork_FaultyEnqueue (0.00s) - logger.go:44: 17:00:29.311368 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 17:00:29.311390 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=1,index=1] - logger.go:44: 17:00:29.311401 [DEBUG] raft-test: server 0: transport: append to 1: newer term - logger.go:44: 17:00:29.311419 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=0,index=0] - logger.go:44: 17:00:29.311424 [DEBUG] raft-test: server 0: pipeline: append to: 1: enqueue fault: command 1 - logger.go:44: 17:00:29.311433 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 10:44:57.166460 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 10:44:57.166515 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=1,index=1] + logger.go:44: 10:44:57.166547 [DEBUG] raft-test: server 0: transport: append to 1: newer term + logger.go:44: 10:44:57.166602 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=0,index=0] + logger.go:44: 10:44:57.166623 [DEBUG] raft-test: server 0: pipeline: append to: 1: enqueue fault: command 1 + logger.go:44: 10:44:57.166657 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes PASS -ok github.com/CanonicalLtd/raft-test/internal/network 0.006s +ok github.com/CanonicalLtd/raft-test/internal/network 0.017s create-stamp debian/debhelper-build-stamp fakeroot debian/rules binary dh binary --buildsystem=golang --with=golang @@ -2637,12 +2792,14 @@ dpkg-source --after-build . dpkg-buildpackage: info: binary-only upload (no source included) I: copying local configuration +I: user script /srv/workspace/pbuilder/5110/tmp/hooks/B01_cleanup starting +I: user script /srv/workspace/pbuilder/5110/tmp/hooks/B01_cleanup finished I: unmounting dev/ptmx filesystem I: unmounting dev/pts filesystem I: unmounting dev/shm filesystem I: unmounting proc filesystem I: unmounting sys filesystem I: cleaning the build env -I: removing directory /srv/workspace/pbuilder/13551 and its subdirectories -I: Current time: Mon Jul 19 05:00:33 -12 2021 -I: pbuilder-time-stamp: 1626714033 +I: removing directory /srv/workspace/pbuilder/5110 and its subdirectories +I: Current time: Wed Jun 17 00:45:06 +14 2020 +I: pbuilder-time-stamp: 1592304306