Diff of the two buildlogs: -- --- b1/build.log 2021-02-15 19:00:29.894196219 +0000 +++ b2/build.log 2021-02-15 19:03:04.801241900 +0000 @@ -1,6 +1,6 @@ I: pbuilder: network access will be disabled during build -I: Current time: Mon Feb 15 06:55:47 -12 2021 -I: pbuilder-time-stamp: 1613415347 +I: Current time: Tue Feb 16 09:00:39 +14 2021 +I: pbuilder-time-stamp: 1613415639 I: Building the build Environment I: extracting base tarball [/var/cache/pbuilder/buster-reproducible-base.tgz] I: copying local configuration @@ -17,7 +17,7 @@ I: Extracting source gpgv: unknown type of key resource 'trustedkeys.kbx' gpgv: keyblock resource '/root/.gnupg/trustedkeys.kbx': General error -gpgv: Signature made Thu Aug 2 20:35:10 2018 -12 +gpgv: Signature made Fri Aug 3 22:35:10 2018 +14 gpgv: using RSA key 0740778A2DFC4A39C0C8BC8C8F2B113C6535C5A7 gpgv: Can't check signature: No public key dpkg-source: warning: failed to verify signature on ./golang-github-canonicalltd-raft-test_0.0~git20180628.c3345b5-1.dsc @@ -26,136 +26,170 @@ dpkg-source: info: unpacking golang-github-canonicalltd-raft-test_0.0~git20180628.c3345b5-1.debian.tar.xz I: using fakeroot in build. I: Installing the build-deps -I: user script /srv/workspace/pbuilder/3458/tmp/hooks/D02_print_environment starting +I: user script /srv/workspace/pbuilder/6245/tmp/hooks/D01_modify_environment starting +debug: Running on ff2a. +I: Changing host+domainname to test build reproducibility +I: Adding a custom variable just for the fun of it... +I: Changing /bin/sh to bash +Removing 'diversion of /bin/sh to /bin/sh.distrib by dash' +Adding 'diversion of /bin/sh to /bin/sh.distrib by bash' +Removing 'diversion of /usr/share/man/man1/sh.1.gz to /usr/share/man/man1/sh.distrib.1.gz by dash' +Adding 'diversion of /usr/share/man/man1/sh.1.gz to /usr/share/man/man1/sh.distrib.1.gz by bash' +I: Setting pbuilder2's login shell to /bin/bash +I: Setting pbuilder2's GECOS to second user,second room,second work-phone,second home-phone,second other +I: user script /srv/workspace/pbuilder/6245/tmp/hooks/D01_modify_environment finished +I: user script /srv/workspace/pbuilder/6245/tmp/hooks/D02_print_environment starting I: set - BUILDDIR='/build' - BUILDUSERGECOS='first user,first room,first work-phone,first home-phone,first other' - BUILDUSERNAME='pbuilder1' - BUILD_ARCH='armhf' - DEBIAN_FRONTEND='noninteractive' - DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all parallel=3' - DISTRIBUTION='' - HOME='/root' - HOST_ARCH='armhf' + BASH=/bin/sh + BASHOPTS=checkwinsize:cmdhist:complete_fullquote:extquote:force_fignore:globasciiranges:hostcomplete:interactive_comments:progcomp:promptvars:sourcepath + BASH_ALIASES=() + BASH_ARGC=() + BASH_ARGV=() + BASH_CMDS=() + BASH_LINENO=([0]="12" [1]="0") + BASH_SOURCE=([0]="/tmp/hooks/D02_print_environment" [1]="/tmp/hooks/D02_print_environment") + BASH_VERSINFO=([0]="5" [1]="0" [2]="3" [3]="1" [4]="release" [5]="arm-unknown-linux-gnueabihf") + BASH_VERSION='5.0.3(1)-release' + BUILDDIR=/build + BUILDUSERGECOS='second user,second room,second work-phone,second home-phone,second other' + BUILDUSERNAME=pbuilder2 + BUILD_ARCH=armhf + DEBIAN_FRONTEND=noninteractive + DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all parallel=4' + DIRSTACK=() + DISTRIBUTION= + EUID=0 + FUNCNAME=([0]="Echo" [1]="main") + GROUPS=() + HOME=/root + HOSTNAME=i-capture-the-hostname + HOSTTYPE=arm + HOST_ARCH=armhf IFS=' ' - INVOCATION_ID='a16d2085c38847afa26afc8e3a59e0d9' - LANG='C' - LANGUAGE='en_US:en' - LC_ALL='C' - MAIL='/var/mail/root' - OPTIND='1' - PATH='/usr/sbin:/usr/bin:/sbin:/bin:/usr/games' - PBCURRENTCOMMANDLINEOPERATION='build' - PBUILDER_OPERATION='build' - PBUILDER_PKGDATADIR='/usr/share/pbuilder' - PBUILDER_PKGLIBDIR='/usr/lib/pbuilder' - PBUILDER_SYSCONFDIR='/etc' - PPID='3458' - PS1='# ' - PS2='> ' + INVOCATION_ID=80d81d8158964e3aaa6958eb520cb37c + LANG=C + LANGUAGE=it_CH:it + LC_ALL=C + MACHTYPE=arm-unknown-linux-gnueabihf + MAIL=/var/mail/root + OPTERR=1 + OPTIND=1 + OSTYPE=linux-gnueabihf + PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path + PBCURRENTCOMMANDLINEOPERATION=build + PBUILDER_OPERATION=build + PBUILDER_PKGDATADIR=/usr/share/pbuilder + PBUILDER_PKGLIBDIR=/usr/lib/pbuilder + PBUILDER_SYSCONFDIR=/etc + PIPESTATUS=([0]="0") + POSIXLY_CORRECT=y + PPID=6245 PS4='+ ' - PWD='/' - SHELL='/bin/bash' - SHLVL='2' - SUDO_COMMAND='/usr/bin/timeout -k 18.1h 18h /usr/bin/ionice -c 3 /usr/bin/nice /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/tmp.3omoJnJzku/pbuilderrc_jTNV --hookdir /etc/pbuilder/first-build-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/buster-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/tmp.3omoJnJzku/b1 --logfile b1/build.log golang-github-canonicalltd-raft-test_0.0~git20180628.c3345b5-1.dsc' - SUDO_GID='115' - SUDO_UID='110' - SUDO_USER='jenkins' - TERM='unknown' - TZ='/usr/share/zoneinfo/Etc/GMT+12' - USER='root' - _='/usr/bin/systemd-run' - http_proxy='http://10.0.0.15:8000/' + PWD=/ + SHELL=/bin/bash + SHELLOPTS=braceexpand:errexit:hashall:interactive-comments:posix + SHLVL=3 + SUDO_COMMAND='/usr/bin/timeout -k 24.1h 24h /usr/bin/ionice -c 3 /usr/bin/nice -n 11 /usr/bin/unshare --uts -- /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/tmp.3omoJnJzku/pbuilderrc_lAqZ --hookdir /etc/pbuilder/rebuild-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/buster-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/tmp.3omoJnJzku/b2 --logfile b2/build.log golang-github-canonicalltd-raft-test_0.0~git20180628.c3345b5-1.dsc' + SUDO_GID=114 + SUDO_UID=109 + SUDO_USER=jenkins + TERM=unknown + TZ=/usr/share/zoneinfo/Etc/GMT-14 + UID=0 + USER=root + _='I: set' + http_proxy=http://10.0.0.15:8000/ I: uname -a - Linux p64b 4.19.0-14-arm64 #1 SMP Debian 4.19.171-2 (2021-01-30) aarch64 GNU/Linux + Linux i-capture-the-hostname 4.19.0-14-armmp-lpae #1 SMP Debian 4.19.171-2 (2021-01-30) armv7l GNU/Linux I: ls -l /bin total 3328 - -rwxr-xr-x 1 root root 767656 Apr 17 2019 bash - -rwxr-xr-x 3 root root 26052 Jul 10 2019 bunzip2 - -rwxr-xr-x 3 root root 26052 Jul 10 2019 bzcat - lrwxrwxrwx 1 root root 6 Jul 10 2019 bzcmp -> bzdiff - -rwxr-xr-x 1 root root 2227 Jul 10 2019 bzdiff - lrwxrwxrwx 1 root root 6 Jul 10 2019 bzegrep -> bzgrep - -rwxr-xr-x 1 root root 4877 Jun 24 2019 bzexe - lrwxrwxrwx 1 root root 6 Jul 10 2019 bzfgrep -> bzgrep - -rwxr-xr-x 1 root root 3641 Jul 10 2019 bzgrep - -rwxr-xr-x 3 root root 26052 Jul 10 2019 bzip2 - -rwxr-xr-x 1 root root 9636 Jul 10 2019 bzip2recover - lrwxrwxrwx 1 root root 6 Jul 10 2019 bzless -> bzmore - -rwxr-xr-x 1 root root 1297 Jul 10 2019 bzmore - -rwxr-xr-x 1 root root 22432 Feb 28 2019 cat - -rwxr-xr-x 1 root root 38868 Feb 28 2019 chgrp - -rwxr-xr-x 1 root root 38836 Feb 28 2019 chmod - -rwxr-xr-x 1 root root 42972 Feb 28 2019 chown - -rwxr-xr-x 1 root root 88376 Feb 28 2019 cp - -rwxr-xr-x 1 root root 75516 Jan 17 2019 dash - -rwxr-xr-x 1 root root 71648 Feb 28 2019 date - -rwxr-xr-x 1 root root 51212 Feb 28 2019 dd - -rwxr-xr-x 1 root root 55672 Feb 28 2019 df - -rwxr-xr-x 1 root root 88444 Feb 28 2019 dir - -rwxr-xr-x 1 root root 54872 Jan 9 2019 dmesg - lrwxrwxrwx 1 root root 8 Sep 26 2018 dnsdomainname -> hostname - lrwxrwxrwx 1 root root 8 Sep 26 2018 domainname -> hostname - -rwxr-xr-x 1 root root 22364 Feb 28 2019 echo - -rwxr-xr-x 1 root root 28 Jan 7 2019 egrep - -rwxr-xr-x 1 root root 18260 Feb 28 2019 false - -rwxr-xr-x 1 root root 28 Jan 7 2019 fgrep - -rwxr-xr-x 1 root root 47356 Jan 9 2019 findmnt - -rwsr-xr-x 1 root root 21980 Apr 22 2020 fusermount - -rwxr-xr-x 1 root root 124508 Jan 7 2019 grep - -rwxr-xr-x 2 root root 2345 Jan 5 2019 gunzip - -rwxr-xr-x 1 root root 6375 Jan 5 2019 gzexe - -rwxr-xr-x 1 root root 64232 Jan 5 2019 gzip - -rwxr-xr-x 1 root root 13784 Sep 26 2018 hostname - -rwxr-xr-x 1 root root 43044 Feb 28 2019 ln - -rwxr-xr-x 1 root root 34932 Jul 26 2018 login - -rwxr-xr-x 1 root root 88444 Feb 28 2019 ls - -rwxr-xr-x 1 root root 67036 Jan 9 2019 lsblk - -rwxr-xr-x 1 root root 47168 Feb 28 2019 mkdir - -rwxr-xr-x 1 root root 43040 Feb 28 2019 mknod - -rwxr-xr-x 1 root root 26552 Feb 28 2019 mktemp - -rwxr-xr-x 1 root root 26024 Jan 9 2019 more - -rwsr-xr-x 1 root root 34268 Jan 9 2019 mount - -rwxr-xr-x 1 root root 9688 Jan 9 2019 mountpoint - -rwxr-xr-x 1 root root 84284 Feb 28 2019 mv - lrwxrwxrwx 1 root root 8 Sep 26 2018 nisdomainname -> hostname - lrwxrwxrwx 1 root root 14 Feb 14 2019 pidof -> /sbin/killall5 - -rwxr-xr-x 1 root root 22416 Feb 28 2019 pwd - lrwxrwxrwx 1 root root 4 Apr 17 2019 rbash -> bash - -rwxr-xr-x 1 root root 26504 Feb 28 2019 readlink - -rwxr-xr-x 1 root root 42968 Feb 28 2019 rm - -rwxr-xr-x 1 root root 26496 Feb 28 2019 rmdir - -rwxr-xr-x 1 root root 14136 Jan 21 2019 run-parts - -rwxr-xr-x 1 root root 76012 Dec 22 2018 sed - lrwxrwxrwx 1 root root 4 Feb 6 20:27 sh -> dash - -rwxr-xr-x 1 root root 22384 Feb 28 2019 sleep - -rwxr-xr-x 1 root root 51124 Feb 28 2019 stty - -rwsr-xr-x 1 root root 42472 Jan 9 2019 su - -rwxr-xr-x 1 root root 22392 Feb 28 2019 sync - -rwxr-xr-x 1 root root 283324 Apr 23 2019 tar - -rwxr-xr-x 1 root root 9808 Jan 21 2019 tempfile - -rwxr-xr-x 1 root root 63464 Feb 28 2019 touch - -rwxr-xr-x 1 root root 18260 Feb 28 2019 true - -rwxr-xr-x 1 root root 9636 Apr 22 2020 ulockmgr_server - -rwsr-xr-x 1 root root 21976 Jan 9 2019 umount - -rwxr-xr-x 1 root root 22380 Feb 28 2019 uname - -rwxr-xr-x 2 root root 2345 Jan 5 2019 uncompress - -rwxr-xr-x 1 root root 88444 Feb 28 2019 vdir - -rwxr-xr-x 1 root root 21980 Jan 9 2019 wdctl - -rwxr-xr-x 1 root root 946 Jan 21 2019 which - lrwxrwxrwx 1 root root 8 Sep 26 2018 ypdomainname -> hostname - -rwxr-xr-x 1 root root 1983 Jan 5 2019 zcat - -rwxr-xr-x 1 root root 1677 Jan 5 2019 zcmp - -rwxr-xr-x 1 root root 5879 Jan 5 2019 zdiff - -rwxr-xr-x 1 root root 29 Jan 5 2019 zegrep - -rwxr-xr-x 1 root root 29 Jan 5 2019 zfgrep - -rwxr-xr-x 1 root root 2080 Jan 5 2019 zforce - -rwxr-xr-x 1 root root 7584 Jan 5 2019 zgrep - -rwxr-xr-x 1 root root 2205 Jan 5 2019 zless - -rwxr-xr-x 1 root root 1841 Jan 5 2019 zmore - -rwxr-xr-x 1 root root 4552 Jan 5 2019 znew -I: user script /srv/workspace/pbuilder/3458/tmp/hooks/D02_print_environment finished + -rwxr-xr-x 1 root root 767656 Apr 18 2019 bash + -rwxr-xr-x 3 root root 26052 Jul 11 2019 bunzip2 + -rwxr-xr-x 3 root root 26052 Jul 11 2019 bzcat + lrwxrwxrwx 1 root root 6 Jul 11 2019 bzcmp -> bzdiff + -rwxr-xr-x 1 root root 2227 Jul 11 2019 bzdiff + lrwxrwxrwx 1 root root 6 Jul 11 2019 bzegrep -> bzgrep + -rwxr-xr-x 1 root root 4877 Jun 25 2019 bzexe + lrwxrwxrwx 1 root root 6 Jul 11 2019 bzfgrep -> bzgrep + -rwxr-xr-x 1 root root 3641 Jul 11 2019 bzgrep + -rwxr-xr-x 3 root root 26052 Jul 11 2019 bzip2 + -rwxr-xr-x 1 root root 9636 Jul 11 2019 bzip2recover + lrwxrwxrwx 1 root root 6 Jul 11 2019 bzless -> bzmore + -rwxr-xr-x 1 root root 1297 Jul 11 2019 bzmore + -rwxr-xr-x 1 root root 22432 Mar 1 2019 cat + -rwxr-xr-x 1 root root 38868 Mar 1 2019 chgrp + -rwxr-xr-x 1 root root 38836 Mar 1 2019 chmod + -rwxr-xr-x 1 root root 42972 Mar 1 2019 chown + -rwxr-xr-x 1 root root 88376 Mar 1 2019 cp + -rwxr-xr-x 1 root root 75516 Jan 18 2019 dash + -rwxr-xr-x 1 root root 71648 Mar 1 2019 date + -rwxr-xr-x 1 root root 51212 Mar 1 2019 dd + -rwxr-xr-x 1 root root 55672 Mar 1 2019 df + -rwxr-xr-x 1 root root 88444 Mar 1 2019 dir + -rwxr-xr-x 1 root root 54872 Jan 10 2019 dmesg + lrwxrwxrwx 1 root root 8 Sep 27 2018 dnsdomainname -> hostname + lrwxrwxrwx 1 root root 8 Sep 27 2018 domainname -> hostname + -rwxr-xr-x 1 root root 22364 Mar 1 2019 echo + -rwxr-xr-x 1 root root 28 Jan 8 2019 egrep + -rwxr-xr-x 1 root root 18260 Mar 1 2019 false + -rwxr-xr-x 1 root root 28 Jan 8 2019 fgrep + -rwxr-xr-x 1 root root 47356 Jan 10 2019 findmnt + -rwsr-xr-x 1 root root 21980 Apr 23 2020 fusermount + -rwxr-xr-x 1 root root 124508 Jan 8 2019 grep + -rwxr-xr-x 2 root root 2345 Jan 6 2019 gunzip + -rwxr-xr-x 1 root root 6375 Jan 6 2019 gzexe + -rwxr-xr-x 1 root root 64232 Jan 6 2019 gzip + -rwxr-xr-x 1 root root 13784 Sep 27 2018 hostname + -rwxr-xr-x 1 root root 43044 Mar 1 2019 ln + -rwxr-xr-x 1 root root 34932 Jul 27 2018 login + -rwxr-xr-x 1 root root 88444 Mar 1 2019 ls + -rwxr-xr-x 1 root root 67036 Jan 10 2019 lsblk + -rwxr-xr-x 1 root root 47168 Mar 1 2019 mkdir + -rwxr-xr-x 1 root root 43040 Mar 1 2019 mknod + -rwxr-xr-x 1 root root 26552 Mar 1 2019 mktemp + -rwxr-xr-x 1 root root 26024 Jan 10 2019 more + -rwsr-xr-x 1 root root 34268 Jan 10 2019 mount + -rwxr-xr-x 1 root root 9688 Jan 10 2019 mountpoint + -rwxr-xr-x 1 root root 84284 Mar 1 2019 mv + lrwxrwxrwx 1 root root 8 Sep 27 2018 nisdomainname -> hostname + lrwxrwxrwx 1 root root 14 Feb 15 2019 pidof -> /sbin/killall5 + -rwxr-xr-x 1 root root 22416 Mar 1 2019 pwd + lrwxrwxrwx 1 root root 4 Apr 18 2019 rbash -> bash + -rwxr-xr-x 1 root root 26504 Mar 1 2019 readlink + -rwxr-xr-x 1 root root 42968 Mar 1 2019 rm + -rwxr-xr-x 1 root root 26496 Mar 1 2019 rmdir + -rwxr-xr-x 1 root root 14136 Jan 22 2019 run-parts + -rwxr-xr-x 1 root root 76012 Dec 23 2018 sed + lrwxrwxrwx 1 root root 4 Feb 16 09:00 sh -> bash + lrwxrwxrwx 1 root root 4 Feb 7 22:24 sh.distrib -> dash + -rwxr-xr-x 1 root root 22384 Mar 1 2019 sleep + -rwxr-xr-x 1 root root 51124 Mar 1 2019 stty + -rwsr-xr-x 1 root root 42472 Jan 10 2019 su + -rwxr-xr-x 1 root root 22392 Mar 1 2019 sync + -rwxr-xr-x 1 root root 283324 Apr 24 2019 tar + -rwxr-xr-x 1 root root 9808 Jan 22 2019 tempfile + -rwxr-xr-x 1 root root 63464 Mar 1 2019 touch + -rwxr-xr-x 1 root root 18260 Mar 1 2019 true + -rwxr-xr-x 1 root root 9636 Apr 23 2020 ulockmgr_server + -rwsr-xr-x 1 root root 21976 Jan 10 2019 umount + -rwxr-xr-x 1 root root 22380 Mar 1 2019 uname + -rwxr-xr-x 2 root root 2345 Jan 6 2019 uncompress + -rwxr-xr-x 1 root root 88444 Mar 1 2019 vdir + -rwxr-xr-x 1 root root 21980 Jan 10 2019 wdctl + -rwxr-xr-x 1 root root 946 Jan 22 2019 which + lrwxrwxrwx 1 root root 8 Sep 27 2018 ypdomainname -> hostname + -rwxr-xr-x 1 root root 1983 Jan 6 2019 zcat + -rwxr-xr-x 1 root root 1677 Jan 6 2019 zcmp + -rwxr-xr-x 1 root root 5879 Jan 6 2019 zdiff + -rwxr-xr-x 1 root root 29 Jan 6 2019 zegrep + -rwxr-xr-x 1 root root 29 Jan 6 2019 zfgrep + -rwxr-xr-x 1 root root 2080 Jan 6 2019 zforce + -rwxr-xr-x 1 root root 7584 Jan 6 2019 zgrep + -rwxr-xr-x 1 root root 2205 Jan 6 2019 zless + -rwxr-xr-x 1 root root 1841 Jan 6 2019 zmore + -rwxr-xr-x 1 root root 4552 Jan 6 2019 znew +I: user script /srv/workspace/pbuilder/6245/tmp/hooks/D02_print_environment finished -> Attempting to satisfy build-dependencies -> Creating pbuilder-satisfydepends-dummy package Package: pbuilder-satisfydepends-dummy @@ -312,7 +346,7 @@ Get: 107 http://deb.debian.org/debian buster/main armhf golang-github-hashicorp-go-msgpack-dev all 0.0~git20150518.0.fa3f638-6 [42.7 kB] Get: 108 http://deb.debian.org/debian buster/main armhf golang-github-hashicorp-logutils-dev all 0.0~git20150609.0.0dc08b1-1 [8150 B] Get: 109 http://deb.debian.org/debian buster/main armhf golang-github-hashicorp-raft-dev all 1.0.0+git20180823.82694fb-3 [85.9 kB] -Fetched 85.2 MB in 10s (8346 kB/s) +Fetched 85.2 MB in 11s (7667 kB/s) debconf: delaying package configuration, since apt-utils is not installed Selecting previously unselected package libbsd0:armhf. (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 18932 files and directories currently installed.) @@ -771,7 +805,7 @@ fakeroot is already the newest version (1.23-1). 0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded. I: Building the package -I: Running cd /build/golang-github-canonicalltd-raft-test-0.0~git20180628.c3345b5/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-buildpackage -us -uc -b +I: Running cd /build/golang-github-canonicalltd-raft-test-0.0~git20180628.c3345b5/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-buildpackage -us -uc -b dpkg-buildpackage: info: source package golang-github-canonicalltd-raft-test dpkg-buildpackage: info: source version 0.0~git20180628.c3345b5-1 dpkg-buildpackage: info: source distribution unstable @@ -789,21 +823,20 @@ dh_autoreconf -O--buildsystem=golang dh_auto_configure -O--buildsystem=golang dh_auto_build -O--buildsystem=golang - cd obj-arm-linux-gnueabihf && go install -gcflags=all=\"-trimpath=/build/golang-github-canonicalltd-raft-test-0.0\~git20180628.c3345b5/obj-arm-linux-gnueabihf/src\" -asmflags=all=\"-trimpath=/build/golang-github-canonicalltd-raft-test-0.0\~git20180628.c3345b5/obj-arm-linux-gnueabihf/src\" -v -p 3 github.com/CanonicalLtd/raft-test github.com/CanonicalLtd/raft-test/internal/election github.com/CanonicalLtd/raft-test/internal/event github.com/CanonicalLtd/raft-test/internal/fsms github.com/CanonicalLtd/raft-test/internal/logging github.com/CanonicalLtd/raft-test/internal/network -internal/race + cd obj-arm-linux-gnueabihf && go install -gcflags=all=\"-trimpath=/build/golang-github-canonicalltd-raft-test-0.0\~git20180628.c3345b5/obj-arm-linux-gnueabihf/src\" -asmflags=all=\"-trimpath=/build/golang-github-canonicalltd-raft-test-0.0\~git20180628.c3345b5/obj-arm-linux-gnueabihf/src\" -v -p 4 github.com/CanonicalLtd/raft-test github.com/CanonicalLtd/raft-test/internal/election github.com/CanonicalLtd/raft-test/internal/event github.com/CanonicalLtd/raft-test/internal/fsms github.com/CanonicalLtd/raft-test/internal/logging github.com/CanonicalLtd/raft-test/internal/network internal/cpu -errors runtime/internal/sys +internal/race +errors sync/atomic -runtime/internal/atomic internal/bytealg +runtime/internal/atomic math internal/testlog math/bits -runtime unicode/utf8 +runtime unicode -strconv container/list crypto/internal/subtle crypto/subtle @@ -813,93 +846,94 @@ vendor/golang_org/x/net/dns/dnsmessage internal/nettrace runtime/cgo -crypto/rc4 +strconv vendor/golang_org/x/crypto/cryptobyte/asn1 github.com/CanonicalLtd/raft-test/internal/event +crypto/rc4 sync io +math/rand reflect syscall +internal/singleflight bytes -bufio crypto/cipher crypto/aes -time -internal/syscall/unix -math/rand +bufio strings -internal/singleflight hash hash/crc32 crypto crypto/internal/randutil -internal/poll crypto/sha512 +crypto/hmac +crypto/md5 +time +internal/syscall/unix +crypto/sha1 +crypto/sha256 +vendor/golang_org/x/text/transform +path +html +hash/crc64 +github.com/hashicorp/logutils +internal/poll encoding/binary sort -os github.com/hashicorp/go-immutable-radix +os encoding/base64 crypto/des -crypto/hmac -crypto/md5 -crypto/sha1 -crypto/sha256 -fmt -encoding/pem -path/filepath vendor/golang_org/x/crypto/internal/chacha20 vendor/golang_org/x/crypto/poly1305 -io/ioutil -vendor/golang_org/x/crypto/chacha20poly1305 +encoding/pem vendor/golang_org/x/crypto/curve25519 -vendor/golang_org/x/text/transform -path +vendor/golang_org/x/crypto/chacha20poly1305 os/signal -hash/crc64 -html -github.com/hashicorp/logutils +path/filepath +runtime/debug +fmt +io/ioutil context -math/big encoding/json log -net +math/big compress/flate -crypto/rand +net compress/gzip -crypto/elliptic -encoding/asn1 -crypto/rsa -crypto/ecdsa -crypto/dsa encoding/hex -vendor/golang_org/x/crypto/cryptobyte -crypto/x509/pkix net/url vendor/golang_org/x/text/unicode/bidi vendor/golang_org/x/text/unicode/norm vendor/golang_org/x/text/secure/bidirule vendor/golang_org/x/net/http2/hpack -mime +crypto/rand +crypto/elliptic +encoding/asn1 +crypto/rsa +crypto/dsa +crypto/ecdsa +crypto/x509/pkix +vendor/golang_org/x/crypto/cryptobyte vendor/golang_org/x/net/idna +mime mime/quotedprintable net/http/internal encoding/gob text/template/parse -crypto/x509 -net/textproto -vendor/golang_org/x/net/http/httpguts -vendor/golang_org/x/net/http/httpproxy -mime/multipart -crypto/tls -text/template github.com/pkg/errors flag -runtime/debug runtime/trace testing -html/template +text/template github.com/CanonicalLtd/raft-test/internal/logging +net/textproto +vendor/golang_org/x/net/http/httpproxy +crypto/x509 +vendor/golang_org/x/net/http/httpguts +mime/multipart +html/template +crypto/tls net/http/httptrace net/http net/rpc @@ -911,1640 +945,1758 @@ github.com/CanonicalLtd/raft-test/internal/network github.com/CanonicalLtd/raft-test dh_auto_test -O--buildsystem=golang - cd obj-arm-linux-gnueabihf && go test -vet=off -v -p 3 github.com/CanonicalLtd/raft-test github.com/CanonicalLtd/raft-test/internal/election github.com/CanonicalLtd/raft-test/internal/event github.com/CanonicalLtd/raft-test/internal/fsms github.com/CanonicalLtd/raft-test/internal/logging github.com/CanonicalLtd/raft-test/internal/network + cd obj-arm-linux-gnueabihf && go test -vet=off -v -p 4 github.com/CanonicalLtd/raft-test github.com/CanonicalLtd/raft-test/internal/election github.com/CanonicalLtd/raft-test/internal/event github.com/CanonicalLtd/raft-test/internal/fsms github.com/CanonicalLtd/raft-test/internal/logging github.com/CanonicalLtd/raft-test/internal/network === RUN TestCluster_Default ---- PASS: TestCluster_Default (0.01s) - logger.go:44: 18:59:53.951830 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 18:59:53.953116 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 18:59:53.953665 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 18:59:53.954048 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 18:59:53.954233 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 18:59:53.954881 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:53.955085 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 18:59:53.955754 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:53.955876 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 18:59:53.956098 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:53.956229 [DEBUG] raft-test: setup: done - logger.go:44: 18:59:53.956406 [DEBUG] raft-test: close: start - logger.go:44: 18:59:53.958012 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:53.962097 [INFO] raft: Node at 0 [Shutdown] entering Follower state (Leader: "") - logger.go:44: 18:59:53.963265 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 18:59:53.962716 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:53.963727 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 18:59:53.963988 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 18:59:53.964065 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 18:59:53.964222 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 18:59:53.964332 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 18:59:53.964427 [DEBUG] raft-test: close: done +--- PASS: TestCluster_Default (0.00s) + logger.go:44: 19:02:50.577606 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 19:02:50.577984 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 19:02:50.578186 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 19:02:50.578267 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 19:02:50.578332 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 19:02:50.578617 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:50.578674 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 19:02:50.578784 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:50.578827 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 19:02:50.578924 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:50.578971 [DEBUG] raft-test: setup: done + logger.go:44: 19:02:50.579054 [DEBUG] raft-test: close: start + logger.go:44: 19:02:50.579275 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:50.579313 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:50.579393 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 19:02:50.579472 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 19:02:50.579553 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 19:02:50.579595 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 19:02:50.579663 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 19:02:50.579741 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 19:02:50.579798 [DEBUG] raft-test: close: done === RUN TestControl_ElectAndDepose ---- PASS: TestControl_ElectAndDepose (0.04s) - logger.go:44: 18:59:53.964837 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 18:59:53.965398 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 18:59:53.965835 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 18:59:53.966003 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 18:59:53.966125 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 18:59:53.966474 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 18:59:53.966790 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 18:59:53.967014 [DEBUG] raft-test: setup: done - logger.go:44: 18:59:53.967089 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 18:59:53.967189 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 18:59:53.967316 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 18:59:53.986533 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 18:59:53.986722 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 18:59:53.986789 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] - logger.go:44: 18:59:53.986636 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] - logger.go:44: 18:59:53.987976 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 18:59:53.988069 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 18:59:53.988121 [DEBUG] raft-test: elect: done - logger.go:44: 18:59:53.988227 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 18:59:53.988365 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 18:59:53.988963 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:53.989261 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 18:59:53.989367 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected - logger.go:44: 18:59:53.989274 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:53.989016 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 18:59:53.989795 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 18:59:53.989726 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:53.990123 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:53.991041 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:53.991160 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:53.991498 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:53.991596 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:53.998125 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 18:59:53.998244 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 18:59:53.998407 [DEBUG] raft-test: close: start - logger.go:44: 18:59:53.998587 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 18:59:53.998723 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 18:59:53.998861 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:53.999015 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:53.999203 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:53.999271 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:53.999915 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 18:59:54.000014 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 18:59:54.000152 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 18:59:54.000260 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 18:59:54.000384 [DEBUG] raft-test: close: done +--- PASS: TestControl_ElectAndDepose (0.05s) + logger.go:44: 19:02:50.579967 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 19:02:50.580133 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 19:02:50.580214 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 19:02:50.580291 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 19:02:50.580357 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 19:02:50.580518 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 19:02:50.580605 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 19:02:50.580686 [DEBUG] raft-test: setup: done + logger.go:44: 19:02:50.580714 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 19:02:50.580755 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 19:02:50.580796 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 19:02:50.608552 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 19:02:50.608918 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 19:02:50.608773 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] + logger.go:44: 19:02:50.608795 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] + logger.go:44: 19:02:50.610330 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 19:02:50.610514 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 19:02:50.610617 [DEBUG] raft-test: elect: done + logger.go:44: 19:02:50.610523 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:50.610758 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 19:02:50.610786 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 19:02:50.611422 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 19:02:50.611008 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:50.611709 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:50.611536 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 19:02:50.611926 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 19:02:50.612874 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:50.613003 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:50.612891 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 19:02:50.613226 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected + logger.go:44: 19:02:50.613675 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:50.613807 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:50.615015 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:50.616496 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:50.623737 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:50.623802 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:50.626221 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:50.626277 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:50.627013 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:50.627124 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:50.628925 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:50.629082 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:50.629095 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 19:02:50.629504 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 19:02:50.629678 [DEBUG] raft-test: close: start + logger.go:44: 19:02:50.629943 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 19:02:50.630224 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 19:02:50.630448 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 19:02:50.630582 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 19:02:50.630699 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 19:02:50.630889 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 19:02:50.631076 [DEBUG] raft-test: close: done === RUN TestControl_DeposeAfterCommandEnqueued ---- PASS: TestControl_DeposeAfterCommandEnqueued (0.05s) - logger.go:44: 18:59:54.000660 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 18:59:54.001029 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 18:59:54.001205 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 18:59:54.001375 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 18:59:54.001497 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 18:59:54.001864 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:54.001954 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 18:59:54.002057 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.002177 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:54.002252 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 18:59:54.002426 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:54.002604 [DEBUG] raft-test: setup: done - logger.go:44: 18:59:54.002677 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 18:59:54.002775 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 18:59:54.002864 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 18:59:54.003028 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.003889 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.025975 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 18:59:54.026142 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 18:59:54.026262 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.026336 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 18:59:54.026546 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 2 - logger.go:44: 18:59:54.026600 [INFO] raft: Election won. Tally: 2 - logger.go:44: 18:59:54.026667 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 18:59:54.026800 [INFO] raft: Added peer 1, starting replication - logger.go:44: 18:59:54.026878 [INFO] raft: Added peer 2, starting replication - logger.go:44: 18:59:54.027067 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 18:59:54.027177 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] - logger.go:44: 18:59:54.027399 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 18:59:54.027400 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] - logger.go:44: 18:59:54.027225 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 18:59:54.027728 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 18:59:54.027769 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 18:59:54.027923 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 18:59:54.027973 [DEBUG] raft-test: elect: done - logger.go:44: 18:59:54.028237 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] - logger.go:44: 18:59:54.028395 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] - logger.go:44: 18:59:54.028657 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 18:59:54.028920 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 18:59:54.028985 [DEBUG] raft-test: server 0: pipeline: append to: 2: enqueue fault: command 2 - logger.go:44: 18:59:54.029232 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=4] - logger.go:44: 18:59:54.029305 [DEBUG] raft-test: server 0: pipeline: append to: 1: enqueue fault: command 2 - logger.go:44: 18:59:54.029440 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 18:59:54.029643 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 18:59:54.029935 [DEBUG] raft-test: fsm 1: applied 1 - logger.go:44: 18:59:54.030043 [DEBUG] raft-test: fsm 2: applied 1 - logger.go:44: 18:59:54.030267 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.037894 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.037961 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 18:59:54.030293 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.038140 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.038252 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 18:59:54.050021 [WARN] raft: Failed to contact 1 in 19.857044ms - logger.go:44: 18:59:54.050164 [WARN] raft: Failed to contact 2 in 19.924463ms - logger.go:44: 18:59:54.050221 [WARN] raft: Failed to contact quorum of nodes, stepping down - logger.go:44: 18:59:54.050345 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.050524 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 18:59:54.050594 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 18:59:54.050679 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 18:59:54.050769 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 18:59:54.050973 [DEBUG] raft-test: close: start - logger.go:44: 18:59:54.051099 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 18:59:54.051168 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 18:59:54.051321 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 18:59:54.051465 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 18:59:54.051600 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 18:59:54.051669 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 18:59:54.051733 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 18:59:54.051878 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 18:59:54.052013 [DEBUG] raft-test: close: done +--- PASS: TestControl_DeposeAfterCommandEnqueued (0.06s) + logger.go:44: 19:02:50.631482 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 19:02:50.631994 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 19:02:50.632259 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 19:02:50.632469 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 19:02:50.632657 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 19:02:50.632991 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:50.633132 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 19:02:50.633250 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:50.633631 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:50.633774 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 19:02:50.633893 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:50.634056 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:50.634218 [DEBUG] raft-test: setup: done + logger.go:44: 19:02:50.634332 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 19:02:50.634459 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 19:02:50.634596 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 19:02:50.634375 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:50.651624 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 19:02:50.651881 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 2 + logger.go:44: 19:02:50.652243 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:50.652635 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 1 + logger.go:44: 19:02:50.652367 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 19:02:50.652450 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:50.660080 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 19:02:50.660235 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 19:02:50.660404 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:50.660492 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 19:02:50.660526 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 19:02:50.660762 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 2 + logger.go:44: 19:02:50.660839 [INFO] raft: Election won. Tally: 2 + logger.go:44: 19:02:50.660940 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 19:02:50.661093 [INFO] raft: Added peer 1, starting replication + logger.go:44: 19:02:50.661239 [INFO] raft: Added peer 2, starting replication + logger.go:44: 19:02:50.661281 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 19:02:50.661549 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:50.661667 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] + logger.go:44: 19:02:50.662052 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 19:02:50.661753 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 19:02:50.661961 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "0") + logger.go:44: 19:02:50.662191 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 19:02:50.662221 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Noop:term=2,index=2] + logger.go:44: 19:02:50.663511 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 19:02:50.663653 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 19:02:50.663748 [DEBUG] raft-test: elect: done + logger.go:44: 19:02:50.663692 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:50.664155 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] + logger.go:44: 19:02:50.664103 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] + logger.go:44: 19:02:50.664611 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:50.664627 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 19:02:50.665218 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=4] + logger.go:44: 19:02:50.665371 [DEBUG] raft-test: server 0: pipeline: append to: 1: enqueue fault: command 2 + logger.go:44: 19:02:50.665579 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=4] + logger.go:44: 19:02:50.665679 [DEBUG] raft-test: server 0: pipeline: append to: 2: enqueue fault: command 2 + logger.go:44: 19:02:50.665813 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 19:02:50.665990 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 19:02:50.666353 [DEBUG] raft-test: fsm 2: applied 1 + logger.go:44: 19:02:50.666487 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:50.666625 [DEBUG] raft-test: fsm 1: applied 1 + logger.go:44: 19:02:50.666637 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:50.666942 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 19:02:50.667182 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:50.667295 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:50.667389 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 19:02:50.680303 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:50.680506 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:50.680799 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:50.681057 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 19:02:50.680518 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:50.681598 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 19:02:50.681714 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 19:02:50.682175 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 3 + logger.go:44: 19:02:50.682416 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 19:02:50.682616 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 19:02:50.681951 [WARN] raft: Failed to contact 1 in 15.61722ms + logger.go:44: 19:02:50.683050 [WARN] raft: Failed to contact 2 in 15.284436ms + logger.go:44: 19:02:50.683229 [WARN] raft: Failed to contact quorum of nodes, stepping down + logger.go:44: 19:02:50.682639 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:50.682496 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:50.684543 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 1 + logger.go:44: 19:02:50.682682 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 + logger.go:44: 19:02:50.683678 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:50.683739 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 19:02:50.684025 [DEBUG] raft-test: close: start + logger.go:44: 19:02:50.685233 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:50.686379 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 + logger.go:44: 19:02:50.685440 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 19:02:50.685456 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:50.685812 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 19:02:50.685905 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 19:02:50.685943 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 19:02:50.687612 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 19:02:50.687848 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 19:02:50.688528 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 19:02:50.688977 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 19:02:50.689528 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 19:02:50.689772 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 19:02:50.690066 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 19:02:50.690464 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 19:02:50.690901 [DEBUG] raft-test: close: done === RUN TestControl_DeposeAfterCommandEnqueuedThenElect --- PASS: TestControl_DeposeAfterCommandEnqueuedThenElect (0.00s) === RUN TestControl_DeposeAfterCommandEnqueuedThenElectSame --- PASS: TestControl_DeposeAfterCommandEnqueuedThenElectSame (0.18s) - logger.go:44: 18:59:54.052534 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 18:59:54.052899 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 18:59:54.053068 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 18:59:54.053190 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 18:59:54.053309 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 18:59:54.053570 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 18:59:54.058418 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 18:59:54.058873 [DEBUG] raft-test: setup: done - logger.go:44: 18:59:54.058962 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 18:59:54.059047 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 18:59:54.059140 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 18:59:54.074751 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 18:59:54.074957 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 18:59:54.074971 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] - logger.go:44: 18:59:54.074823 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] - logger.go:44: 18:59:54.082035 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 18:59:54.082274 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 18:59:54.082604 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.082821 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 18:59:54.082886 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 18:59:54.082955 [DEBUG] raft-test: elect: done - logger.go:44: 18:59:54.083289 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] - logger.go:44: 18:59:54.083444 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] - logger.go:44: 18:59:54.083697 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 18:59:54.083972 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 18:59:54.084038 [DEBUG] raft-test: server 0: pipeline: append to: 2: enqueue fault: command 2 - logger.go:44: 18:59:54.084287 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=4] - logger.go:44: 18:59:54.084363 [DEBUG] raft-test: server 0: pipeline: append to: 1: enqueue fault: command 2 - logger.go:44: 18:59:54.084460 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 18:59:54.084597 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 18:59:54.084825 [DEBUG] raft-test: fsm 1: applied 1 - logger.go:44: 18:59:54.085003 [DEBUG] raft-test: fsm 2: applied 1 - logger.go:44: 18:59:54.085256 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.085386 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.085938 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.086026 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.098227 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 18:59:54.098350 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 18:59:54.098599 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 18:59:54.098695 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 18:59:54.098785 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 18:59:54.228360 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=8,index=5] - logger.go:44: 18:59:54.228624 [DEBUG] raft-test: server 0: transport: append to 2: 2 entries [Command:term=2,index=4 Noop:term=8,index=5] - logger.go:44: 18:59:54.229106 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 18:59:54.229206 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 18:59:54.229497 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=8,index=5] - logger.go:44: 18:59:54.229803 [DEBUG] raft-test: server 0: transport: append to 1: 2 entries [Command:term=2,index=4 Noop:term=8,index=5] - logger.go:44: 18:59:54.230289 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 18:59:54.230383 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 18:59:54.230437 [DEBUG] raft-test: elect: done - logger.go:44: 18:59:54.230629 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=8,index=6] - logger.go:44: 18:59:54.230826 [DEBUG] raft-test: fsm 2: applied 2 - logger.go:44: 18:59:54.231048 [DEBUG] raft-test: fsm 0: applied 2 - logger.go:44: 18:59:54.231114 [DEBUG] raft-test: fsm 0: applied 3 - logger.go:44: 18:59:54.231230 [DEBUG] raft-test: close: start - logger.go:44: 18:59:54.231392 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=8,index=6] - logger.go:44: 18:59:54.231801 [DEBUG] raft-test: fsm 1: applied 2 - logger.go:44: 18:59:54.231941 [DEBUG] raft-test: fsm 1: applied 3 - logger.go:44: 18:59:54.232050 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 18:59:54.232207 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 18:59:54.232361 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 18:59:54.232441 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 18:59:54.232508 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 18:59:54.232673 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 18:59:54.232771 [DEBUG] raft-test: close: done + logger.go:44: 19:02:50.691231 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 19:02:50.691395 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 19:02:50.691487 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 19:02:50.691559 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 19:02:50.691629 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 19:02:50.691771 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 19:02:50.691895 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 19:02:50.692031 [DEBUG] raft-test: setup: done + logger.go:44: 19:02:50.692071 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 19:02:50.692121 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 19:02:50.692166 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 19:02:50.716191 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 19:02:50.716593 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] + logger.go:44: 19:02:50.716741 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 19:02:50.716618 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] + logger.go:44: 19:02:50.717424 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 19:02:50.718890 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:50.718892 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:50.718933 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 19:02:50.720125 [DEBUG] raft-test: elect: done + logger.go:44: 19:02:50.719416 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 19:02:50.721815 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] + logger.go:44: 19:02:50.720808 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] + logger.go:44: 19:02:50.722004 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 19:02:50.722228 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 19:02:50.722417 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=4] + logger.go:44: 19:02:50.722506 [DEBUG] raft-test: server 0: pipeline: append to: 2: enqueue fault: command 2 + logger.go:44: 19:02:50.721673 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:50.722758 [DEBUG] raft-test: fsm 2: applied 1 + logger.go:44: 19:02:50.722876 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=4] + logger.go:44: 19:02:50.722926 [DEBUG] raft-test: server 0: pipeline: append to: 1: enqueue fault: command 2 + logger.go:44: 19:02:50.722993 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 19:02:50.723083 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 19:02:50.721682 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:50.723178 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:50.723355 [DEBUG] raft-test: fsm 1: applied 1 + logger.go:44: 19:02:50.725523 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:50.725589 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:50.735809 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:50.735905 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:50.736706 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 19:02:50.736803 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 19:02:50.736733 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 19:02:50.736940 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 19:02:50.737052 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 19:02:50.866407 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 19:02:50.866723 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 19:02:50.866789 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=9,index=5] + logger.go:44: 19:02:50.866839 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=9,index=5] + logger.go:44: 19:02:50.867550 [DEBUG] raft-test: server 0: transport: append to 2: 2 entries [Command:term=2,index=4 Noop:term=9,index=5] + logger.go:44: 19:02:50.867914 [DEBUG] raft-test: server 0: transport: append to 1: 2 entries [Command:term=2,index=4 Noop:term=9,index=5] + logger.go:44: 19:02:50.868321 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 19:02:50.868640 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 19:02:50.868853 [DEBUG] raft-test: elect: done + logger.go:44: 19:02:50.869073 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:50.869496 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=9,index=6] + logger.go:44: 19:02:50.868364 [DEBUG] raft-test: fsm 0: applied 2 + logger.go:44: 19:02:50.869860 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=9,index=6] + logger.go:44: 19:02:50.870507 [DEBUG] raft-test: fsm 0: applied 3 + logger.go:44: 19:02:50.870891 [DEBUG] raft-test: close: start + logger.go:44: 19:02:50.871117 [DEBUG] raft-test: fsm 1: applied 2 + logger.go:44: 19:02:50.869923 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:50.871197 [DEBUG] raft-test: fsm 2: applied 2 + logger.go:44: 19:02:50.871430 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 19:02:50.872553 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 19:02:50.872606 [DEBUG] raft-test: fsm 2: applied 3 + logger.go:44: 19:02:50.873193 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 19:02:50.873731 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 19:02:50.874003 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 19:02:50.874452 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 19:02:50.875028 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 19:02:50.875302 [DEBUG] raft-test: close: done === RUN TestControl_DeposeAfterCommandAppended ---- PASS: TestControl_DeposeAfterCommandAppended (0.04s) - logger.go:44: 18:59:54.233136 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 18:59:54.233503 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 18:59:54.241959 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 18:59:54.242253 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 18:59:54.242383 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 18:59:54.242752 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 18:59:54.242967 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 18:59:54.243196 [DEBUG] raft-test: setup: done - logger.go:44: 18:59:54.243270 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 18:59:54.243369 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 18:59:54.243470 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 18:59:54.262744 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] - logger.go:44: 18:59:54.263267 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 18:59:54.263423 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 18:59:54.263623 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] - logger.go:44: 18:59:54.264040 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 18:59:54.264102 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 18:59:54.264151 [DEBUG] raft-test: elect: done - logger.go:44: 18:59:54.264446 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] - logger.go:44: 18:59:54.264517 [DEBUG] raft-test: server 0: pipeline: append to 2: append fault: command 1 - logger.go:44: 18:59:54.264810 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] - logger.go:44: 18:59:54.264892 [DEBUG] raft-test: server 0: pipeline: append to 1: append fault: command 1 - logger.go:44: 18:59:54.265086 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 18:59:54.265240 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 18:59:54.270068 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.270337 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.270488 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 18:59:54.270563 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected - logger.go:44: 18:59:54.270748 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.270869 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.271013 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 18:59:54.271081 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 18:59:54.274042 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=2,index=3] - logger.go:44: 18:59:54.274271 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.274607 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 18:59:54.274680 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 18:59:54.274866 [DEBUG] raft-test: close: start - logger.go:44: 18:59:54.275090 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=3] - logger.go:44: 18:59:54.275167 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.275323 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 18:59:54.275483 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 18:59:54.275624 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 18:59:54.275695 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 18:59:54.275832 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 18:59:54.276042 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 18:59:54.276101 [DEBUG] raft-test: close: done +--- PASS: TestControl_DeposeAfterCommandAppended (0.05s) + logger.go:44: 19:02:50.876017 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 19:02:50.876191 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 19:02:50.876277 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 19:02:50.876347 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 19:02:50.876410 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 19:02:50.876558 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 19:02:50.876681 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 19:02:50.876815 [DEBUG] raft-test: setup: done + logger.go:44: 19:02:50.876861 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 19:02:50.876898 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 19:02:50.876942 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 19:02:50.903372 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 19:02:50.903766 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 19:02:50.903918 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] + logger.go:44: 19:02:50.904489 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] + logger.go:44: 19:02:50.905473 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 19:02:50.905689 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 19:02:50.905890 [DEBUG] raft-test: elect: done + logger.go:44: 19:02:50.906636 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] + logger.go:44: 19:02:50.906884 [DEBUG] raft-test: server 0: pipeline: append to 2: append fault: command 1 + logger.go:44: 19:02:50.906645 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] + logger.go:44: 19:02:50.907370 [DEBUG] raft-test: server 0: pipeline: append to 1: append fault: command 1 + logger.go:44: 19:02:50.907768 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:50.907918 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 19:02:50.908505 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 19:02:50.908016 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:50.909052 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:50.909282 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 19:02:50.909647 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 19:02:50.908778 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 19:02:50.910502 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:50.910656 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected + logger.go:44: 19:02:50.910727 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:50.912175 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=2,index=3] + logger.go:44: 19:02:50.912213 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:50.913311 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=3] + logger.go:44: 19:02:50.913392 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:50.921093 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:50.921205 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:50.923725 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 19:02:50.923920 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 19:02:50.923780 [DEBUG] raft-test: close: start + logger.go:44: 19:02:50.924248 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=3] + logger.go:44: 19:02:50.924423 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:50.924265 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=2,index=3] + logger.go:44: 19:02:50.924676 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:50.924301 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:50.924913 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:50.925102 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 19:02:50.925367 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 19:02:50.925595 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 19:02:50.925719 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 19:02:50.925859 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 19:02:50.926045 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 19:02:50.926211 [DEBUG] raft-test: close: done === RUN TestControl_DeposeAfterCommandAppendedThenElect === RUN TestControl_DeposeAfterCommandAppendedThenElect/0 === RUN TestControl_DeposeAfterCommandAppendedThenElect/1 === RUN TestControl_DeposeAfterCommandAppendedThenElect/2 ---- PASS: TestControl_DeposeAfterCommandAppendedThenElect (0.40s) - --- PASS: TestControl_DeposeAfterCommandAppendedThenElect/0 (0.28s) - logger.go:44: 18:59:54.276481 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 18:59:54.276856 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 18:59:54.277035 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 18:59:54.277241 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 18:59:54.277369 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 18:59:54.285799 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:54.285989 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 18:59:54.286360 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:54.286464 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 18:59:54.286710 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:54.286788 [DEBUG] raft-test: setup: done - logger.go:44: 18:59:54.286860 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 18:59:54.286938 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 18:59:54.287031 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 18:59:54.287226 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.287400 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.287550 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.304461 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 18:59:54.304660 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 2 - logger.go:44: 18:59:54.304832 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.304904 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 1 - logger.go:44: 18:59:54.305013 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 18:59:54.305121 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 18:59:54.312707 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 18:59:54.312861 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 2 - logger.go:44: 18:59:54.312988 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.313058 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 1 - logger.go:44: 18:59:54.313162 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 18:59:54.313267 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 18:59:54.313383 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 18:59:54.313454 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 18:59:54.313628 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.313735 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 18:59:54.313850 [INFO] raft: Duplicate RequestVote for same term: 2 - logger.go:44: 18:59:54.313991 [INFO] raft: Duplicate RequestVote for same term: 2 - logger.go:44: 18:59:54.329996 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 18:59:54.330172 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 3 - logger.go:44: 18:59:54.330319 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.330394 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 1 - logger.go:44: 18:59:54.330570 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 18:59:54.330684 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 18:59:54.333907 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 18:59:54.334148 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 3 - logger.go:44: 18:59:54.334343 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.334420 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 1 - logger.go:44: 18:59:54.334576 [INFO] raft: Duplicate RequestVote for same term: 3 - logger.go:44: 18:59:54.334702 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 18:59:54.334776 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 - logger.go:44: 18:59:54.334875 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.334947 [INFO] raft: Duplicate RequestVote for same term: 3 - logger.go:44: 18:59:54.335018 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 - logger.go:44: 18:59:54.335218 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 18:59:54.335318 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 18:59:54.353892 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 18:59:54.354096 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 4 - logger.go:44: 18:59:54.354316 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.354389 [DEBUG] raft: Vote granted from 1 in term 4. Tally: 1 - logger.go:44: 18:59:54.354503 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 18:59:54.354607 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 18:59:54.355737 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 18:59:54.355845 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 4 - logger.go:44: 18:59:54.355988 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.356060 [DEBUG] raft: Vote granted from 2 in term 4. Tally: 1 - logger.go:44: 18:59:54.356286 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 18:59:54.356429 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 18:59:54.356919 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 4 - logger.go:44: 18:59:54.357037 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.357102 [DEBUG] raft: Vote granted from 0 in term 4. Tally: 1 - logger.go:44: 18:59:54.357312 [INFO] raft: Duplicate RequestVote for same term: 4 - logger.go:44: 18:59:54.356539 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 18:59:54.357575 [INFO] raft: Duplicate RequestVote for same term: 4 - logger.go:44: 18:59:54.378526 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 18:59:54.378722 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 5 - logger.go:44: 18:59:54.378850 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.378924 [DEBUG] raft: Vote granted from 1 in term 5. Tally: 1 - logger.go:44: 18:59:54.379033 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 18:59:54.379142 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 18:59:54.380421 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 18:59:54.380636 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 5 - logger.go:44: 18:59:54.380774 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.380845 [DEBUG] raft: Vote granted from 2 in term 5. Tally: 1 - logger.go:44: 18:59:54.380954 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 18:59:54.381061 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 18:59:54.383443 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 18:59:54.383611 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 5 - logger.go:44: 18:59:54.383740 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.383811 [DEBUG] raft: Vote granted from 0 in term 5. Tally: 1 - logger.go:44: 18:59:54.383950 [INFO] raft: Duplicate RequestVote for same term: 5 - logger.go:44: 18:59:54.384106 [INFO] raft: Duplicate RequestVote for same term: 5 - logger.go:44: 18:59:54.397757 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 18:59:54.397937 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 6 - logger.go:44: 18:59:54.398112 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.398186 [DEBUG] raft: Vote granted from 1 in term 6. Tally: 1 - logger.go:44: 18:59:54.398297 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 18:59:54.398406 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 18:59:54.408258 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 18:59:54.408435 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 6 - logger.go:44: 18:59:54.408683 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.408844 [DEBUG] raft: Vote granted from 0 in term 6. Tally: 1 - logger.go:44: 18:59:54.408782 [INFO] raft: Duplicate RequestVote for same term: 6 - logger.go:44: 18:59:54.408976 [DEBUG] raft: Vote granted from 2 in term 6. Tally: 2 - logger.go:44: 18:59:54.409039 [INFO] raft: Election won. Tally: 2 - logger.go:44: 18:59:54.409108 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 18:59:54.409192 [INFO] raft: Added peer 1, starting replication - logger.go:44: 18:59:54.409265 [INFO] raft: Added peer 2, starting replication - logger.go:44: 18:59:54.409765 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=6,index=2] - logger.go:44: 18:59:54.408939 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.410157 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 18:59:54.410319 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 18:59:54.410482 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 18:59:54.410860 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=6,index=2] - logger.go:44: 18:59:54.411070 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "0") - logger.go:44: 18:59:54.411438 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 18:59:54.411777 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 18:59:54.411910 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 18:59:54.411963 [DEBUG] raft-test: elect: done - logger.go:44: 18:59:54.412299 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=6,index=3] - logger.go:44: 18:59:54.412633 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 18:59:54.412786 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.412965 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=6,index=3] - logger.go:44: 18:59:54.413409 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=6,index=4] - logger.go:44: 18:59:54.413504 [DEBUG] raft-test: server 0: pipeline: append to 1: append fault: command 2 - logger.go:44: 18:59:54.413020 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=6,index=4] - logger.go:44: 18:59:54.413670 [DEBUG] raft-test: server 0: pipeline: append to 2: append fault: command 2 - logger.go:44: 18:59:54.414086 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 18:59:54.414300 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 18:59:54.414377 [DEBUG] raft-test: fsm 1: applied 1 - logger.go:44: 18:59:54.414603 [DEBUG] raft-test: fsm 2: applied 1 - logger.go:44: 18:59:54.415101 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.415722 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.415797 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 18:59:54.415136 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 18:59:54.415993 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 18:59:54.416076 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 18:59:54.416168 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 18:59:54.415524 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 18:59:54.416340 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected - logger.go:44: 18:59:54.416450 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 18:59:54.416532 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 18:59:54.416648 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.416732 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.416795 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 18:59:54.417511 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=6,index=4] - logger.go:44: 18:59:54.417656 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.417844 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 18:59:54.418182 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=6,index=4] - logger.go:44: 18:59:54.418263 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.418339 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 18:59:54.429228 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.429375 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.429459 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 18:59:54.429525 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.429648 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.429763 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 18:59:54.431048 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=6,index=4] - logger.go:44: 18:59:54.431177 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.431266 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 18:59:54.431328 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=6,index=4] - logger.go:44: 18:59:54.431407 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.431485 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 18:59:54.431421 [WARN] raft: Failed to contact 1 in 17.417925ms - logger.go:44: 18:59:54.431647 [WARN] raft: Failed to contact 2 in 18.232576ms - logger.go:44: 18:59:54.431720 [WARN] raft: Failed to contact quorum of nodes, stepping down - logger.go:44: 18:59:54.432159 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.432261 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 18:59:54.432329 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 18:59:54.431653 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 18:59:54.432462 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 7 - logger.go:44: 18:59:54.432769 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.432843 [DEBUG] raft: Vote granted from 1 in term 7. Tally: 1 - logger.go:44: 18:59:54.433002 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 18:59:54.433288 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 18:59:54.433387 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 18:59:54.433479 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 18:59:54.433583 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 18:59:54.435785 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 18:59:54.435973 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 7 - logger.go:44: 18:59:54.436118 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.436195 [DEBUG] raft: Vote granted from 2 in term 7. Tally: 1 - logger.go:44: 18:59:54.436310 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 18:59:54.436481 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 18:59:54.450402 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 18:59:54.450561 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 7 - logger.go:44: 18:59:54.450689 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.450764 [DEBUG] raft: Vote granted from 0 in term 7. Tally: 1 - logger.go:44: 18:59:54.450765 [INFO] raft: Duplicate RequestVote for same term: 7 - logger.go:44: 18:59:54.450850 [INFO] raft: Duplicate RequestVote for same term: 7 - logger.go:44: 18:59:54.451593 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 18:59:54.451693 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 8 - logger.go:44: 18:59:54.451819 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.451865 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 18:59:54.451889 [DEBUG] raft: Vote granted from 1 in term 8. Tally: 1 - logger.go:44: 18:59:54.451928 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 18:59:54.455261 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 18:59:54.455432 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 8 - logger.go:44: 18:59:54.455578 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.455652 [DEBUG] raft: Vote granted from 2 in term 8. Tally: 1 - logger.go:44: 18:59:54.455762 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 18:59:54.455868 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 18:59:54.478548 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 18:59:54.478729 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 8 - logger.go:44: 18:59:54.478878 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.478952 [DEBUG] raft: Vote granted from 0 in term 8. Tally: 1 - logger.go:44: 18:59:54.479003 [INFO] raft: Duplicate RequestVote for same term: 8 - logger.go:44: 18:59:54.479035 [INFO] raft: Duplicate RequestVote for same term: 8 - logger.go:44: 18:59:54.480272 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 18:59:54.480423 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 9 - logger.go:44: 18:59:54.480554 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.480681 [DEBUG] raft: Vote granted from 1 in term 9. Tally: 1 - logger.go:44: 18:59:54.480619 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 18:59:54.480697 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 18:59:54.484157 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 18:59:54.484317 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 9 - logger.go:44: 18:59:54.484466 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.484545 [DEBUG] raft: Vote granted from 2 in term 9. Tally: 1 - logger.go:44: 18:59:54.484566 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 18:59:54.484483 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 18:59:54.501839 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 18:59:54.502117 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 9 - logger.go:44: 18:59:54.502298 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.502374 [DEBUG] raft: Vote granted from 0 in term 9. Tally: 1 - logger.go:44: 18:59:54.502512 [INFO] raft: Duplicate RequestVote for same term: 9 - logger.go:44: 18:59:54.502666 [INFO] raft: Duplicate RequestVote for same term: 9 - logger.go:44: 18:59:54.510002 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 18:59:54.510183 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 10 - logger.go:44: 18:59:54.510320 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.510542 [DEBUG] raft: Vote granted from 1 in term 10. Tally: 1 - logger.go:44: 18:59:54.510689 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 18:59:54.510809 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 18:59:54.516606 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 18:59:54.516802 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 10 - logger.go:44: 18:59:54.516935 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.517007 [DEBUG] raft: Vote granted from 2 in term 10. Tally: 1 - logger.go:44: 18:59:54.517114 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 18:59:54.517223 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 18:59:54.534188 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 18:59:54.534370 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 11 - logger.go:44: 18:59:54.534500 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.534676 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 18:59:54.535193 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 18:59:54.535479 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 10 - logger.go:44: 18:59:54.535582 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.535648 [DEBUG] raft: Vote granted from 0 in term 10. Tally: 1 - logger.go:44: 18:59:54.535338 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 18:59:54.535359 [DEBUG] raft: Vote granted from 2 in term 11. Tally: 1 - logger.go:44: 18:59:54.535908 [DEBUG] raft: Newer term discovered, fallback to follower - logger.go:44: 18:59:54.535982 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.536199 [INFO] raft: Duplicate RequestVote for same term: 10 - logger.go:44: 18:59:54.536598 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 18:59:54.536688 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 11 - logger.go:44: 18:59:54.536810 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.536875 [DEBUG] raft: Vote granted from 1 in term 11. Tally: 1 - logger.go:44: 18:59:54.536970 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 18:59:54.537069 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 18:59:54.552410 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 18:59:54.552634 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 12 - logger.go:44: 18:59:54.552797 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.552871 [DEBUG] raft: Vote granted from 0 in term 12. Tally: 1 - logger.go:44: 18:59:54.552925 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.553016 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.553069 [DEBUG] raft: Vote granted from 1 in term 12. Tally: 2 - logger.go:44: 18:59:54.553134 [INFO] raft: Election won. Tally: 2 - logger.go:44: 18:59:54.553209 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 18:59:54.553290 [INFO] raft: Added peer 1, starting replication - logger.go:44: 18:59:54.553362 [INFO] raft: Added peer 2, starting replication - logger.go:44: 18:59:54.553641 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=12,index=5] - logger.go:44: 18:59:54.553877 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 18:59:54.553980 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 18:59:54.554064 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 18:59:54.554288 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=12,index=5] - logger.go:44: 18:59:54.554583 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 18:59:54.554792 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 18:59:54.554855 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 18:59:54.554906 [DEBUG] raft-test: elect: done - logger.go:44: 18:59:54.555087 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=12,index=6] - logger.go:44: 18:59:54.555335 [DEBUG] raft-test: fsm 2: applied 2 - logger.go:44: 18:59:54.555527 [DEBUG] raft-test: fsm 0: applied 2 - logger.go:44: 18:59:54.555593 [DEBUG] raft-test: fsm 0: applied 3 - logger.go:44: 18:59:54.555725 [DEBUG] raft-test: close: start - logger.go:44: 18:59:54.555897 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=12,index=6] - logger.go:44: 18:59:54.556279 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 18:59:54.556391 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 18:59:54.556648 [DEBUG] raft-test: fsm 1: applied 2 - logger.go:44: 18:59:54.556719 [DEBUG] raft-test: fsm 1: applied 3 - logger.go:44: 18:59:54.556860 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 18:59:54.557015 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 18:59:54.557159 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 18:59:54.557231 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 18:59:54.557303 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 18:59:54.557434 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 18:59:54.557530 [DEBUG] raft-test: close: done - --- PASS: TestControl_DeposeAfterCommandAppendedThenElect/1 (0.06s) - logger.go:44: 18:59:54.558780 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 18:59:54.559202 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 18:59:54.559440 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 18:59:54.559589 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 18:59:54.559711 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 18:59:54.559966 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:54.560058 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 18:59:54.560811 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.561536 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:54.561726 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 18:59:54.562027 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:54.562108 [DEBUG] raft-test: setup: done - logger.go:44: 18:59:54.562181 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 18:59:54.562326 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 18:59:54.562416 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 18:59:54.562610 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.562774 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.581387 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 18:59:54.581620 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 18:59:54.581774 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.581846 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 18:59:54.581936 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 2 - logger.go:44: 18:59:54.581985 [INFO] raft: Election won. Tally: 2 - logger.go:44: 18:59:54.582049 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 18:59:54.582175 [INFO] raft: Added peer 1, starting replication - logger.go:44: 18:59:54.582249 [INFO] raft: Added peer 2, starting replication - logger.go:44: 18:59:54.582513 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] - logger.go:44: 18:59:54.582513 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] - logger.go:44: 18:59:54.582715 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 18:59:54.582807 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 18:59:54.582258 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 18:59:54.583284 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 18:59:54.583455 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 18:59:54.583539 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 18:59:54.583622 [DEBUG] raft-test: elect: done - logger.go:44: 18:59:54.583916 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] - logger.go:44: 18:59:54.584101 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] - logger.go:44: 18:59:54.584240 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 18:59:54.584512 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 18:59:54.584580 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.584580 [DEBUG] raft-test: server 0: pipeline: append to 2: append fault: command 2 - logger.go:44: 18:59:54.584337 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.584627 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=4] - logger.go:44: 18:59:54.585059 [DEBUG] raft-test: server 0: pipeline: append to 1: append fault: command 2 - logger.go:44: 18:59:54.585345 [DEBUG] raft-test: fsm 2: applied 1 - logger.go:44: 18:59:54.585486 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 18:59:54.585624 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 18:59:54.585721 [DEBUG] raft-test: fsm 1: applied 1 - logger.go:44: 18:59:54.586288 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 18:59:54.586370 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected - logger.go:44: 18:59:54.586449 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 18:59:54.586529 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 18:59:54.586664 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 18:59:54.586737 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 18:59:54.586532 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 18:59:54.586816 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 18:59:54.587466 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.587598 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.587659 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.587755 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.587818 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 18:59:54.587664 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 18:59:54.588412 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 18:59:54.588483 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.588555 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 18:59:54.588724 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=2,index=4] - logger.go:44: 18:59:54.588790 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.588856 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 18:59:54.600260 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.600504 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.600578 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 18:59:54.600369 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 18:59:54.600717 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.600801 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 18:59:54.600650 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=2,index=4] - logger.go:44: 18:59:54.600952 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.601021 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 18:59:54.600977 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.601145 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.601202 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 18:59:54.602762 [WARN] raft: Failed to contact 1 in 17.952651ms - logger.go:44: 18:59:54.602893 [WARN] raft: Failed to contact 2 in 17.458385ms - logger.go:44: 18:59:54.602950 [WARN] raft: Failed to contact quorum of nodes, stepping down - logger.go:44: 18:59:54.603092 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.603190 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 18:59:54.603260 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 18:59:54.603447 [DEBUG] raft-test: elect: start (server 1) - logger.go:44: 18:59:54.603529 [DEBUG] raft-test: server 1: establish outbound connection to all other nodes - logger.go:44: 18:59:54.603617 [DEBUG] raft-test: elect: server 1: wait to become leader within 1.5s - logger.go:44: 18:59:54.607645 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 18:59:54.607836 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 - logger.go:44: 18:59:54.607955 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.608064 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 - logger.go:44: 18:59:54.608178 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 18:59:54.608214 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 18:59:54.612028 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 18:59:54.612237 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 3 - logger.go:44: 18:59:54.612334 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 18:59:54.612423 [INFO] raft: Duplicate RequestVote for same term: 3 - logger.go:44: 18:59:54.612423 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.612754 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 18:59:54.612347 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.612925 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 1 - logger.go:44: 18:59:54.612991 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 2 - logger.go:44: 18:59:54.613041 [INFO] raft: Election won. Tally: 2 - logger.go:44: 18:59:54.613109 [INFO] raft: Node at 1 [Leader] entering Leader state - logger.go:44: 18:59:54.613206 [INFO] raft: Added peer 0, starting replication - logger.go:44: 18:59:54.613308 [INFO] raft: Added peer 2, starting replication - logger.go:44: 18:59:54.613641 [DEBUG] raft-test: server 1: transport: append to 2: 1 entry [Noop:term=3,index=5] - logger.go:44: 18:59:54.613864 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "1") - logger.go:44: 18:59:54.614018 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 18:59:54.614122 [DEBUG] raft-test: server 1: leadership: acquired - logger.go:44: 18:59:54.614267 [DEBUG] raft-test: elect: server 1: wait for other servers to become followers within 40ms - logger.go:44: 18:59:54.614494 [DEBUG] raft-test: server 1: transport: append to 0: 1 entry [Noop:term=3,index=5] - logger.go:44: 18:59:54.614731 [INFO] raft: pipelining replication to peer {Voter 0 0} - logger.go:44: 18:59:54.614924 [DEBUG] raft-test: elect: server 0: became follower - logger.go:44: 18:59:54.614987 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 18:59:54.615037 [DEBUG] raft-test: elect: done - logger.go:44: 18:59:54.615235 [DEBUG] raft-test: server 1: pipeline: append to 2: 1 entry [Command:term=3,index=6] - logger.go:44: 18:59:54.615415 [DEBUG] raft-test: fsm 2: applied 2 - logger.go:44: 18:59:54.615708 [DEBUG] raft-test: fsm 1: applied 2 - logger.go:44: 18:59:54.615777 [DEBUG] raft-test: fsm 1: applied 3 - logger.go:44: 18:59:54.615915 [DEBUG] raft-test: close: start - logger.go:44: 18:59:54.616062 [DEBUG] raft-test: server 1: pipeline: append to 0: 1 entry [Command:term=3,index=6] - logger.go:44: 18:59:54.616391 [INFO] raft: aborting pipeline replication to peer {Voter 0 0} - logger.go:44: 18:59:54.616537 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 18:59:54.616615 [DEBUG] raft-test: server 1: transport: append to 2: 0 entries - logger.go:44: 18:59:54.616642 [DEBUG] raft-test: fsm 0: applied 2 - logger.go:44: 18:59:54.617083 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 18:59:54.617094 [DEBUG] raft-test: fsm 0: applied 3 - logger.go:44: 18:59:54.617304 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 18:59:54.617443 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 18:59:54.617521 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 18:59:54.617587 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 18:59:54.617654 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 18:59:54.617823 [DEBUG] raft-test: close: done - --- PASS: TestControl_DeposeAfterCommandAppendedThenElect/2 (0.05s) - logger.go:44: 18:59:54.618350 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 18:59:54.618729 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 18:59:54.618903 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 18:59:54.619025 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 18:59:54.619184 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 18:59:54.619443 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:54.619655 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 18:59:54.619870 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:54.620004 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 18:59:54.620207 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:54.620282 [DEBUG] raft-test: setup: done - logger.go:44: 18:59:54.620352 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 18:59:54.620467 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 18:59:54.620559 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 18:59:54.620720 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.620880 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.621097 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.636909 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 18:59:54.637074 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 18:59:54.637214 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.637324 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 18:59:54.637387 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 2 - logger.go:44: 18:59:54.637437 [INFO] raft: Election won. Tally: 2 - logger.go:44: 18:59:54.637502 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 18:59:54.637597 [INFO] raft: Added peer 1, starting replication - logger.go:44: 18:59:54.637724 [INFO] raft: Added peer 2, starting replication - logger.go:44: 18:59:54.637791 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 18:59:54.637969 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 18:59:54.638035 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] - logger.go:44: 18:59:54.638214 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] - logger.go:44: 18:59:54.638239 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 18:59:54.638422 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 18:59:54.639254 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 18:59:54.639477 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 18:59:54.639533 [DEBUG] raft-test: elect: done - logger.go:44: 18:59:54.639906 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] - logger.go:44: 18:59:54.639999 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] - logger.go:44: 18:59:54.640297 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 18:59:54.640382 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 18:59:54.640586 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.640795 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.640660 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=4] - logger.go:44: 18:59:54.641076 [DEBUG] raft-test: server 0: pipeline: append to 1: append fault: command 2 - logger.go:44: 18:59:54.640672 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 18:59:54.641283 [DEBUG] raft-test: server 0: pipeline: append to 2: append fault: command 2 - logger.go:44: 18:59:54.641533 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 18:59:54.641733 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 18:59:54.641807 [DEBUG] raft-test: fsm 1: applied 1 - logger.go:44: 18:59:54.642015 [DEBUG] raft-test: fsm 2: applied 1 - logger.go:44: 18:59:54.642751 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 18:59:54.642831 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected - logger.go:44: 18:59:54.642877 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.642954 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.643016 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 18:59:54.642913 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 18:59:54.642777 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 18:59:54.643370 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 18:59:54.643445 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 18:59:54.643527 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 18:59:54.643235 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 18:59:54.644142 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.644220 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.644282 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 18:59:54.645425 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=2,index=4] - logger.go:44: 18:59:54.645511 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.645446 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 18:59:54.645757 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.645600 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 18:59:54.645837 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 18:59:54.656120 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.656328 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.656404 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 18:59:54.657445 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.657551 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.657622 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 18:59:54.657581 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 18:59:54.657811 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.657889 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 18:59:54.657891 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=2,index=4] - logger.go:44: 18:59:54.658040 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.658111 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 18:59:54.658760 [WARN] raft: Failed to contact 1 in 17.753728ms - logger.go:44: 18:59:54.658872 [WARN] raft: Failed to contact 2 in 17.905441ms - logger.go:44: 18:59:54.658928 [WARN] raft: Failed to contact quorum of nodes, stepping down - logger.go:44: 18:59:54.659076 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.659169 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 18:59:54.659243 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 18:59:54.659393 [DEBUG] raft-test: elect: start (server 2) - logger.go:44: 18:59:54.659776 [DEBUG] raft-test: server 2: establish outbound connection to all other nodes - logger.go:44: 18:59:54.659980 [DEBUG] raft-test: elect: server 2: wait to become leader within 1.5s - logger.go:44: 18:59:54.667063 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 18:59:54.667219 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 - logger.go:44: 18:59:54.667377 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.667512 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 - logger.go:44: 18:59:54.667645 [WARN] raft: Rejecting vote request from 2 since we have a leader: 0 - logger.go:44: 18:59:54.667876 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 2 - logger.go:44: 18:59:54.667932 [INFO] raft: Election won. Tally: 2 - logger.go:44: 18:59:54.668057 [INFO] raft: Node at 2 [Leader] entering Leader state - logger.go:44: 18:59:54.668162 [INFO] raft: Added peer 0, starting replication - logger.go:44: 18:59:54.668238 [INFO] raft: Added peer 1, starting replication - logger.go:44: 18:59:54.668540 [DEBUG] raft-test: server 2: transport: append to 1: 1 entry [Noop:term=3,index=5] - logger.go:44: 18:59:54.668769 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 18:59:54.668957 [DEBUG] raft-test: fsm 2: applied 2 - logger.go:44: 18:59:54.669022 [DEBUG] raft-test: server 2: transport: append to 0: 1 entry [Noop:term=3,index=5] - logger.go:44: 18:59:54.669048 [DEBUG] raft-test: server 2: leadership: acquired - logger.go:44: 18:59:54.669636 [DEBUG] raft-test: elect: server 2: wait for other servers to become followers within 40ms - logger.go:44: 18:59:54.669828 [DEBUG] raft-test: elect: server 0: became follower - logger.go:44: 18:59:54.669889 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 18:59:54.669940 [DEBUG] raft-test: elect: done - logger.go:44: 18:59:54.670049 [DEBUG] raft-test: server 2: pipeline: append to 1: 0 entries - logger.go:44: 18:59:54.669242 [INFO] raft: pipelining replication to peer {Voter 0 0} - logger.go:44: 18:59:54.670367 [DEBUG] raft-test: server 2: pipeline: append to 0: 1 entry [Command:term=3,index=6] - logger.go:44: 18:59:54.670429 [DEBUG] raft-test: fsm 1: applied 2 - logger.go:44: 18:59:54.670262 [DEBUG] raft-test: server 2: pipeline: append to 1: 1 entry [Command:term=3,index=6] - logger.go:44: 18:59:54.670654 [DEBUG] raft-test: fsm 2: applied 3 - logger.go:44: 18:59:54.670780 [DEBUG] raft-test: close: start - logger.go:44: 18:59:54.670694 [DEBUG] raft-test: fsm 0: applied 2 - logger.go:44: 18:59:54.670955 [INFO] raft: aborting pipeline replication to peer {Voter 0 0} - logger.go:44: 18:59:54.671033 [DEBUG] raft-test: server 2: transport: append to 1: 0 entries - logger.go:44: 18:59:54.670991 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 18:59:54.671470 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 18:59:54.671627 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 18:59:54.671752 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 18:59:54.671823 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 18:59:54.671907 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 18:59:54.672030 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 18:59:54.672104 [DEBUG] raft-test: close: done +--- PASS: TestControl_DeposeAfterCommandAppendedThenElect (0.29s) + --- PASS: TestControl_DeposeAfterCommandAppendedThenElect/0 (0.14s) + logger.go:44: 19:02:50.926705 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 19:02:50.927262 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 19:02:50.927491 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 19:02:50.927699 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 19:02:50.927903 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 19:02:50.928242 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:50.928380 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 19:02:50.928517 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:50.928721 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:50.928843 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 19:02:50.928953 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:50.929112 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:50.929383 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:50.929427 [DEBUG] raft-test: setup: done + logger.go:44: 19:02:50.929651 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 19:02:50.929762 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 19:02:50.929895 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 19:02:50.946332 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 19:02:50.946637 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 2 + logger.go:44: 19:02:50.946997 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:50.947211 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:50.947364 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 19:02:50.947241 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 1 + logger.go:44: 19:02:50.953252 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 19:02:50.953588 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 19:02:50.953808 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:50.953905 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 19:02:50.953918 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 19:02:50.954175 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 2 + logger.go:44: 19:02:50.954256 [INFO] raft: Election won. Tally: 2 + logger.go:44: 19:02:50.954360 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 19:02:50.954493 [INFO] raft: Added peer 1, starting replication + logger.go:44: 19:02:50.954618 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 19:02:50.954648 [INFO] raft: Added peer 2, starting replication + logger.go:44: 19:02:50.954852 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 19:02:50.954856 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:50.955309 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] + logger.go:44: 19:02:50.955405 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "0") + logger.go:44: 19:02:50.955621 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 19:02:50.955794 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 19:02:50.955817 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Noop:term=2,index=2] + logger.go:44: 19:02:50.956339 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 19:02:50.956447 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 19:02:50.956527 [DEBUG] raft-test: elect: done + logger.go:44: 19:02:50.956914 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] + logger.go:44: 19:02:50.956951 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] + logger.go:44: 19:02:50.957444 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 19:02:50.957888 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=4] + logger.go:44: 19:02:50.958012 [DEBUG] raft-test: server 0: pipeline: append to 2: append fault: command 2 + logger.go:44: 19:02:50.958097 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:50.957607 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:50.958320 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=4] + logger.go:44: 19:02:50.958720 [DEBUG] raft-test: server 0: pipeline: append to 1: append fault: command 2 + logger.go:44: 19:02:50.958564 [DEBUG] raft-test: fsm 2: applied 1 + logger.go:44: 19:02:50.959006 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 19:02:50.959260 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 19:02:50.959038 [DEBUG] raft-test: fsm 1: applied 1 + logger.go:44: 19:02:50.960125 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 19:02:50.960251 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected + logger.go:44: 19:02:50.960377 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 19:02:50.960514 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 19:02:50.960822 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 19:02:50.960948 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 19:02:50.961078 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 19:02:50.961227 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 19:02:50.961307 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:50.961475 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:50.960985 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:50.961681 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:50.961863 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 19:02:50.961574 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 19:02:50.962068 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=4] + logger.go:44: 19:02:50.962177 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:50.962285 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 19:02:50.963195 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=2,index=4] + logger.go:44: 19:02:50.963309 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:50.963417 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 19:02:50.974108 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=4] + logger.go:44: 19:02:50.974299 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:50.974493 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 19:02:50.974533 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:50.974817 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:50.974954 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 19:02:50.975054 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:50.975214 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:50.975344 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 19:02:50.975434 [WARN] raft: Failed to contact 1 in 16.850941ms + logger.go:44: 19:02:50.975598 [WARN] raft: Failed to contact 2 in 16.873107ms + logger.go:44: 19:02:50.975713 [WARN] raft: Failed to contact quorum of nodes, stepping down + logger.go:44: 19:02:50.975879 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=2,index=4] + logger.go:44: 19:02:50.976071 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:50.976532 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 19:02:50.975939 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:50.975998 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 19:02:50.977055 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 19:02:50.976334 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 19:02:50.977414 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 19:02:50.977587 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 19:02:50.978562 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 19:02:50.978727 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 + logger.go:44: 19:02:50.978931 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:50.979079 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 + logger.go:44: 19:02:50.979122 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 19:02:50.979003 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:50.997657 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 19:02:50.997907 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 3 + logger.go:44: 19:02:50.998205 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:50.998422 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 1 + logger.go:44: 19:02:50.998445 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 19:02:50.998306 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:51.002018 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 19:02:51.002256 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 3 + logger.go:44: 19:02:51.002552 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.002701 [INFO] raft: Duplicate RequestVote for same term: 3 + logger.go:44: 19:02:51.002934 [INFO] raft: Duplicate RequestVote for same term: 3 + logger.go:44: 19:02:51.002758 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 1 + logger.go:44: 19:02:51.008173 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 19:02:51.008245 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 4 + logger.go:44: 19:02:51.008333 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.008392 [DEBUG] raft: Vote granted from 2 in term 4. Tally: 1 + logger.go:44: 19:02:51.008357 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:51.008471 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 19:02:51.027156 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 19:02:51.027323 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 4 + logger.go:44: 19:02:51.027538 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.027779 [DEBUG] raft: Vote granted from 1 in term 4. Tally: 1 + logger.go:44: 19:02:51.027843 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 19:02:51.027640 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:51.030023 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 19:02:51.030178 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 4 + logger.go:44: 19:02:51.030410 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.030562 [DEBUG] raft: Vote granted from 0 in term 4. Tally: 1 + logger.go:44: 19:02:51.030495 [INFO] raft: Duplicate RequestVote for same term: 4 + logger.go:44: 19:02:51.030970 [INFO] raft: Duplicate RequestVote for same term: 4 + logger.go:44: 19:02:51.035318 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 19:02:51.035470 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 5 + logger.go:44: 19:02:51.035670 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.035809 [DEBUG] raft: Vote granted from 2 in term 5. Tally: 1 + logger.go:44: 19:02:51.035993 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 19:02:51.036200 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:51.051492 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 19:02:51.051712 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 5 + logger.go:44: 19:02:51.052018 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.052218 [DEBUG] raft: Vote granted from 0 in term 5. Tally: 1 + logger.go:44: 19:02:51.052527 [INFO] raft: Duplicate RequestVote for same term: 5 + logger.go:44: 19:02:51.053077 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.053462 [DEBUG] raft: Vote granted from 1 in term 5. Tally: 2 + logger.go:44: 19:02:51.053627 [INFO] raft: Election won. Tally: 2 + logger.go:44: 19:02:51.053820 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 19:02:51.054035 [INFO] raft: Added peer 1, starting replication + logger.go:44: 19:02:51.054243 [INFO] raft: Added peer 2, starting replication + logger.go:44: 19:02:51.054914 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=5,index=5] + logger.go:44: 19:02:51.055323 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "0") + logger.go:44: 19:02:51.055753 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 19:02:51.055769 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 19:02:51.055825 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=5,index=5] + logger.go:44: 19:02:51.055934 [DEBUG] raft-test: fsm 0: applied 2 + logger.go:44: 19:02:51.056151 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 19:02:51.057016 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 19:02:51.057505 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 19:02:51.057979 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 19:02:51.058155 [DEBUG] raft-test: elect: done + logger.go:44: 19:02:51.058172 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 19:02:51.058488 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.058787 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=5,index=6] + logger.go:44: 19:02:51.058907 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=5,index=6] + logger.go:44: 19:02:51.058994 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.059319 [DEBUG] raft-test: fsm 2: applied 2 + logger.go:44: 19:02:51.059994 [DEBUG] raft-test: fsm 0: applied 3 + logger.go:44: 19:02:51.060876 [DEBUG] raft-test: close: start + logger.go:44: 19:02:51.060051 [DEBUG] raft-test: fsm 1: applied 2 + logger.go:44: 19:02:51.061629 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 19:02:51.061723 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.061911 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 19:02:51.062728 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.063146 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 19:02:51.063208 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 19:02:51.063267 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 19:02:51.063298 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 19:02:51.063330 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 19:02:51.063382 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 19:02:51.063425 [DEBUG] raft-test: close: done + --- PASS: TestControl_DeposeAfterCommandAppendedThenElect/1 (0.10s) + logger.go:44: 19:02:51.063664 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 19:02:51.063816 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 19:02:51.063888 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 19:02:51.063943 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 19:02:51.064002 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 19:02:51.064099 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:51.064136 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 19:02:51.064222 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:51.064256 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 19:02:51.064331 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:51.064363 [DEBUG] raft-test: setup: done + logger.go:44: 19:02:51.064395 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 19:02:51.064438 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 19:02:51.064478 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 19:02:51.064540 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.064608 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.064675 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.083876 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 19:02:51.084135 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 2 + logger.go:44: 19:02:51.084444 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.084663 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 1 + logger.go:44: 19:02:51.084857 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 19:02:51.084547 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:51.093801 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 19:02:51.094133 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 19:02:51.094245 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 19:02:51.094362 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 2 + logger.go:44: 19:02:51.094247 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.094499 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:51.094507 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 19:02:51.094465 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.094746 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 1 + logger.go:44: 19:02:51.094528 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 19:02:51.094289 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 19:02:51.094852 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 19:02:51.099609 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 19:02:51.099688 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 3 + logger.go:44: 19:02:51.099805 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.099893 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 1 + logger.go:44: 19:02:51.099933 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 19:02:51.099835 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:51.114973 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 19:02:51.115095 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 3 + logger.go:44: 19:02:51.115251 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.115357 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 1 + logger.go:44: 19:02:51.115597 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.115732 [INFO] raft: Duplicate RequestVote for same term: 3 + logger.go:44: 19:02:51.115818 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 2 + logger.go:44: 19:02:51.116038 [INFO] raft: Election won. Tally: 2 + logger.go:44: 19:02:51.116193 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 19:02:51.116394 [INFO] raft: Added peer 1, starting replication + logger.go:44: 19:02:51.116517 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 19:02:51.116790 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 19:02:51.116578 [INFO] raft: Added peer 2, starting replication + logger.go:44: 19:02:51.116866 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.117481 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "0") + logger.go:44: 19:02:51.117605 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=3,index=2] + logger.go:44: 19:02:51.117711 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 19:02:51.118119 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 19:02:51.118120 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Noop:term=3,index=2] + logger.go:44: 19:02:51.119731 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 19:02:51.119869 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 19:02:51.119973 [DEBUG] raft-test: elect: done + logger.go:44: 19:02:51.120064 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 19:02:51.120452 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=3,index=3] + logger.go:44: 19:02:51.120056 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.120986 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 19:02:51.121620 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=3,index=4] + logger.go:44: 19:02:51.121769 [DEBUG] raft-test: server 0: pipeline: append to 2: append fault: command 2 + logger.go:44: 19:02:51.120500 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=3,index=3] + logger.go:44: 19:02:51.122314 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=3,index=4] + logger.go:44: 19:02:51.122455 [DEBUG] raft-test: server 0: pipeline: append to 1: append fault: command 2 + logger.go:44: 19:02:51.120122 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.123128 [DEBUG] raft-test: fsm 2: applied 1 + logger.go:44: 19:02:51.123204 [DEBUG] raft-test: fsm 1: applied 1 + logger.go:44: 19:02:51.123534 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 19:02:51.123993 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 19:02:51.124061 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 19:02:51.124265 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected + logger.go:44: 19:02:51.124415 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 19:02:51.124639 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 19:02:51.123842 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.124977 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:51.125100 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 19:02:51.125471 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 19:02:51.125618 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 19:02:51.125765 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 19:02:51.125980 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 19:02:51.126074 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=3,index=4] + logger.go:44: 19:02:51.126351 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:51.126364 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.126841 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:51.126905 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 19:02:51.126911 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 19:02:51.128100 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=3,index=4] + logger.go:44: 19:02:51.128144 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:51.128215 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 19:02:51.137529 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.137688 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:51.137818 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 19:02:51.137559 [WARN] raft: Failed to contact 1 in 14.262751ms + logger.go:44: 19:02:51.138072 [WARN] raft: Failed to contact 2 in 14.09913ms + logger.go:44: 19:02:51.138185 [WARN] raft: Failed to contact quorum of nodes, stepping down + logger.go:44: 19:02:51.138405 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.138563 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=3,index=4] + logger.go:44: 19:02:51.138886 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:51.138576 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 19:02:51.138729 [DEBUG] raft-test: elect: start (server 1) + logger.go:44: 19:02:51.139038 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 19:02:51.139201 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 19:02:51.139259 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.139835 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:51.139522 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 19:02:51.140132 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 4 + logger.go:44: 19:02:51.140262 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=3,index=4] + logger.go:44: 19:02:51.140430 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 19:02:51.139658 [DEBUG] raft-test: server 1: establish outbound connection to all other nodes + logger.go:44: 19:02:51.140808 [DEBUG] raft-test: elect: server 1: wait to become leader within 1.5s + logger.go:44: 19:02:51.140442 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:51.141129 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 19:02:51.139973 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 19:02:51.140353 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.141663 [DEBUG] raft: Vote granted from 1 in term 4. Tally: 1 + logger.go:44: 19:02:51.140471 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:51.144474 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 19:02:51.144713 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 4 + logger.go:44: 19:02:51.144924 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.145073 [DEBUG] raft: Vote granted from 2 in term 4. Tally: 1 + logger.go:44: 19:02:51.145119 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 19:02:51.144985 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:51.156353 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 19:02:51.156419 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 4 + logger.go:44: 19:02:51.156497 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.156531 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 19:02:51.156581 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 19:02:51.156552 [DEBUG] raft: Vote granted from 0 in term 4. Tally: 1 + logger.go:44: 19:02:51.157087 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 19:02:51.157144 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 5 + logger.go:44: 19:02:51.157222 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.157276 [DEBUG] raft: Vote granted from 1 in term 5. Tally: 1 + logger.go:44: 19:02:51.157352 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.157353 [DEBUG] raft: Vote granted from 0 in term 5. Tally: 2 + logger.go:44: 19:02:51.157471 [INFO] raft: Election won. Tally: 2 + logger.go:44: 19:02:51.157525 [INFO] raft: Node at 1 [Leader] entering Leader state + logger.go:44: 19:02:51.157292 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.157594 [INFO] raft: Added peer 0, starting replication + logger.go:44: 19:02:51.157609 [DEBUG] raft-test: server 1: leadership: acquired + logger.go:44: 19:02:51.157815 [DEBUG] raft-test: server 1: transport: append to 0: 0 entries + logger.go:44: 19:02:51.157956 [INFO] raft: pipelining replication to peer {Voter 0 0} + logger.go:44: 19:02:51.157720 [INFO] raft: Added peer 2, starting replication + logger.go:44: 19:02:51.158166 [DEBUG] raft-test: server 1: pipeline: append to 0: 1 entry [Noop:term=5,index=5] + logger.go:44: 19:02:51.158361 [DEBUG] raft-test: fsm 1: applied 2 + logger.go:44: 19:02:51.157846 [DEBUG] raft-test: elect: server 1: wait for other servers to become followers within 40ms + logger.go:44: 19:02:51.158598 [DEBUG] raft-test: elect: server 0: became follower + logger.go:44: 19:02:51.158871 [DEBUG] raft-test: server 1: transport: append to 2: 1 entry [Noop:term=5,index=5] + logger.go:44: 19:02:51.159172 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 19:02:51.159375 [DEBUG] raft-test: fsm 2: applied 2 + logger.go:44: 19:02:51.159522 [DEBUG] raft-test: server 1: pipeline: append to 0: 0 entries + logger.go:44: 19:02:51.159865 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 19:02:51.159995 [DEBUG] raft-test: elect: done + logger.go:44: 19:02:51.159877 [DEBUG] raft-test: fsm 0: applied 2 + logger.go:44: 19:02:51.160254 [DEBUG] raft-test: server 1: pipeline: append to 2: 1 entry [Command:term=5,index=6] + logger.go:44: 19:02:51.160283 [DEBUG] raft-test: server 1: pipeline: append to 0: 1 entry [Command:term=5,index=6] + logger.go:44: 19:02:51.160676 [DEBUG] raft-test: fsm 1: applied 3 + logger.go:44: 19:02:51.160678 [DEBUG] raft-test: server 1: transport: append to 0: 0 entries + logger.go:44: 19:02:51.160894 [DEBUG] raft-test: close: start + logger.go:44: 19:02:51.161391 [INFO] raft: aborting pipeline replication to peer {Voter 0 0} + logger.go:44: 19:02:51.161558 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 19:02:51.161808 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 19:02:51.162080 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 19:02:51.162326 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 19:02:51.162454 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 19:02:51.162571 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 19:02:51.162683 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 19:02:51.162849 [DEBUG] raft-test: close: done + --- PASS: TestControl_DeposeAfterCommandAppendedThenElect/2 (0.06s) + logger.go:44: 19:02:51.163544 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 19:02:51.163996 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 19:02:51.164215 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 19:02:51.164404 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 19:02:51.164586 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 19:02:51.164881 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:51.165018 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 19:02:51.165135 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.165300 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:51.165603 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 19:02:51.165748 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.165935 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:51.166066 [DEBUG] raft-test: setup: done + logger.go:44: 19:02:51.166180 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.166194 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 19:02:51.166438 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 19:02:51.166567 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 19:02:51.182588 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 19:02:51.182763 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 19:02:51.182974 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.183114 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 19:02:51.183440 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 2 + logger.go:44: 19:02:51.183591 [INFO] raft: Election won. Tally: 2 + logger.go:44: 19:02:51.183749 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 19:02:51.183926 [INFO] raft: Added peer 1, starting replication + logger.go:44: 19:02:51.184127 [INFO] raft: Added peer 2, starting replication + logger.go:44: 19:02:51.184657 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] + logger.go:44: 19:02:51.184934 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] + logger.go:44: 19:02:51.185069 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 19:02:51.185298 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 19:02:51.184679 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 19:02:51.185845 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 19:02:51.186155 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 19:02:51.186287 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 19:02:51.186385 [DEBUG] raft-test: elect: done + logger.go:44: 19:02:51.186865 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] + logger.go:44: 19:02:51.186901 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] + logger.go:44: 19:02:51.187596 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 19:02:51.188086 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.188302 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=4] + logger.go:44: 19:02:51.188199 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=4] + logger.go:44: 19:02:51.188627 [DEBUG] raft-test: server 0: pipeline: append to 2: append fault: command 2 + logger.go:44: 19:02:51.188474 [DEBUG] raft-test: server 0: pipeline: append to 1: append fault: command 2 + logger.go:44: 19:02:51.189462 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 19:02:51.189714 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 19:02:51.189716 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.190078 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:51.190211 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 19:02:51.189771 [DEBUG] raft-test: fsm 1: applied 1 + logger.go:44: 19:02:51.189771 [DEBUG] raft-test: fsm 2: applied 1 + logger.go:44: 19:02:51.190621 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 19:02:51.191074 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected + logger.go:44: 19:02:51.191112 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 19:02:51.191159 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 19:02:51.191271 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.191321 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:51.191362 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 19:02:51.191296 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 19:02:51.191436 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 19:02:51.191474 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 19:02:51.191520 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 19:02:51.192825 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=2,index=4] + logger.go:44: 19:02:51.192868 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:51.192906 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 19:02:51.192919 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=4] + logger.go:44: 19:02:51.192989 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:51.193025 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 19:02:51.203133 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.203251 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:51.203426 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 19:02:51.204724 [WARN] raft: Failed to contact 1 in 16.282787ms + logger.go:44: 19:02:51.204849 [WARN] raft: Failed to contact 2 in 17.251098ms + logger.go:44: 19:02:51.204916 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=2,index=4] + logger.go:44: 19:02:51.205058 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:51.204766 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.205273 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:51.205420 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 19:02:51.204943 [WARN] raft: Failed to contact quorum of nodes, stepping down + logger.go:44: 19:02:51.205686 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.205011 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=2,index=4] + logger.go:44: 19:02:51.205943 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:51.205173 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 19:02:51.205854 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 19:02:51.206421 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 19:02:51.206060 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 19:02:51.206461 [DEBUG] raft-test: elect: start (server 2) + logger.go:44: 19:02:51.206842 [DEBUG] raft-test: server 2: establish outbound connection to all other nodes + logger.go:44: 19:02:51.206985 [DEBUG] raft-test: elect: server 2: wait to become leader within 1.5s + logger.go:44: 19:02:51.216606 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 19:02:51.216693 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 + logger.go:44: 19:02:51.216795 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.216866 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 + logger.go:44: 19:02:51.216920 [WARN] raft: Rejecting vote request from 2 since we have a leader: 0 + logger.go:44: 19:02:51.216927 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 2 + logger.go:44: 19:02:51.217061 [INFO] raft: Election won. Tally: 2 + logger.go:44: 19:02:51.217138 [INFO] raft: Node at 2 [Leader] entering Leader state + logger.go:44: 19:02:51.217223 [INFO] raft: Added peer 0, starting replication + logger.go:44: 19:02:51.217310 [INFO] raft: Added peer 1, starting replication + logger.go:44: 19:02:51.217359 [DEBUG] raft-test: server 2: leadership: acquired + logger.go:44: 19:02:51.217562 [DEBUG] raft-test: server 2: transport: append to 1: 1 entry [Noop:term=3,index=5] + logger.go:44: 19:02:51.217564 [DEBUG] raft-test: server 2: transport: append to 0: 1 entry [Noop:term=3,index=5] + logger.go:44: 19:02:51.217623 [DEBUG] raft-test: elect: server 2: wait for other servers to become followers within 40ms + logger.go:44: 19:02:51.217817 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 19:02:51.217857 [INFO] raft: pipelining replication to peer {Voter 0 0} + logger.go:44: 19:02:51.217915 [DEBUG] raft-test: elect: server 0: became follower + logger.go:44: 19:02:51.218058 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 19:02:51.218107 [DEBUG] raft-test: elect: done + logger.go:44: 19:02:51.217929 [DEBUG] raft-test: fsm 2: applied 2 + logger.go:44: 19:02:51.218274 [DEBUG] raft-test: server 2: pipeline: append to 0: 1 entry [Command:term=3,index=6] + logger.go:44: 19:02:51.218255 [DEBUG] raft-test: server 2: pipeline: append to 1: 1 entry [Command:term=3,index=6] + logger.go:44: 19:02:51.218549 [DEBUG] raft-test: fsm 0: applied 2 + logger.go:44: 19:02:51.218606 [DEBUG] raft-test: fsm 1: applied 2 + logger.go:44: 19:02:51.218611 [DEBUG] raft-test: fsm 2: applied 3 + logger.go:44: 19:02:51.218799 [DEBUG] raft-test: close: start + logger.go:44: 19:02:51.218936 [INFO] raft: aborting pipeline replication to peer {Voter 0 0} + logger.go:44: 19:02:51.218948 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 19:02:51.219144 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 19:02:51.219272 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 19:02:51.219400 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 19:02:51.219475 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 19:02:51.219544 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 19:02:51.219651 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 19:02:51.219708 [DEBUG] raft-test: close: done === RUN TestControl_DeposeAfterCommandCommitted --- PASS: TestControl_DeposeAfterCommandCommitted (0.04s) - logger.go:44: 18:59:54.672706 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 18:59:54.673233 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 18:59:54.673458 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 18:59:54.673585 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 18:59:54.673883 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 18:59:54.674139 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:54.674261 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 18:59:54.674468 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:54.674550 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 18:59:54.674732 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:54.674829 [DEBUG] raft-test: setup: done - logger.go:44: 18:59:54.674910 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 18:59:54.674972 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.674988 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 18:59:54.675181 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 18:59:54.674864 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.675376 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.691181 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 18:59:54.691566 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 18:59:54.691688 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.691759 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 18:59:54.691970 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 2 - logger.go:44: 18:59:54.692024 [INFO] raft: Election won. Tally: 2 - logger.go:44: 18:59:54.692108 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 18:59:54.692203 [INFO] raft: Added peer 1, starting replication - logger.go:44: 18:59:54.692379 [INFO] raft: Added peer 2, starting replication - logger.go:44: 18:59:54.692823 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 18:59:54.692923 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 2 - logger.go:44: 18:59:54.693023 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.693089 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 1 - logger.go:44: 18:59:54.693190 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 18:59:54.693292 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 18:59:54.693451 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 18:59:54.693644 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 18:59:54.693784 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] - logger.go:44: 18:59:54.693575 [INFO] raft: Duplicate RequestVote for same term: 2 - logger.go:44: 18:59:54.693835 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] - logger.go:44: 18:59:54.694053 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "0") - logger.go:44: 18:59:54.694232 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 18:59:54.694358 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 18:59:54.694933 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 18:59:54.695000 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 18:59:54.695071 [DEBUG] raft-test: elect: done - logger.go:44: 18:59:54.695344 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] - logger.go:44: 18:59:54.695345 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] - logger.go:44: 18:59:54.695681 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 18:59:54.695834 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 18:59:54.696028 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 18:59:54.696179 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.696296 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.696375 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 18:59:54.696480 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.696613 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.696673 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 18:59:54.696854 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 18:59:54.697018 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected - logger.go:44: 18:59:54.697109 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 18:59:54.697115 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 18:59:54.697266 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 18:59:54.697335 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 18:59:54.697443 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 18:59:54.697211 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 18:59:54.699542 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.699668 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.699800 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 18:59:54.699675 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.700008 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.700081 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 18:59:54.709354 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.709548 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.709621 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 18:59:54.709906 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.709997 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.710059 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 18:59:54.711578 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.711669 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.711747 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 18:59:54.712350 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.712454 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.712531 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 18:59:54.712901 [WARN] raft: Failed to contact 1 in 17.202127ms - logger.go:44: 18:59:54.712981 [WARN] raft: Failed to contact 2 in 17.257045ms - logger.go:44: 18:59:54.713035 [WARN] raft: Failed to contact quorum of nodes, stepping down - logger.go:44: 18:59:54.713163 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.713263 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 18:59:54.713339 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 18:59:54.713623 [DEBUG] raft-test: close: start - logger.go:44: 18:59:54.713885 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 18:59:54.714025 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 18:59:54.714165 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 18:59:54.714235 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 18:59:54.714305 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 18:59:54.714436 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 18:59:54.714588 [DEBUG] raft-test: close: done + logger.go:44: 19:02:51.220241 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 19:02:51.220542 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 19:02:51.220694 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 19:02:51.220820 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 19:02:51.220928 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 19:02:51.221110 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:51.221195 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 19:02:51.221274 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.221400 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:51.221480 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 19:02:51.221578 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.221701 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:51.221784 [DEBUG] raft-test: setup: done + logger.go:44: 19:02:51.221853 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 19:02:51.221920 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 19:02:51.222001 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 19:02:51.221878 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.237567 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 19:02:51.237774 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 19:02:51.238014 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.238170 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 19:02:51.238320 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 2 + logger.go:44: 19:02:51.238439 [INFO] raft: Election won. Tally: 2 + logger.go:44: 19:02:51.238600 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 19:02:51.239017 [INFO] raft: Added peer 1, starting replication + logger.go:44: 19:02:51.239101 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 19:02:51.239216 [INFO] raft: Added peer 2, starting replication + logger.go:44: 19:02:51.239492 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.239981 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] + logger.go:44: 19:02:51.240358 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 19:02:51.240017 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 19:02:51.240753 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Noop:term=2,index=2] + logger.go:44: 19:02:51.241785 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 19:02:51.241933 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.239530 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 19:02:51.243940 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 19:02:51.244069 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 19:02:51.244169 [DEBUG] raft-test: elect: done + logger.go:44: 19:02:51.244626 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] + logger.go:44: 19:02:51.244840 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] + logger.go:44: 19:02:51.245030 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 19:02:51.245127 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 19:02:51.245221 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 19:02:51.245481 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.245594 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:51.245643 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 19:02:51.245942 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 19:02:51.246000 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected + logger.go:44: 19:02:51.246072 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 19:02:51.246144 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 19:02:51.246547 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.246601 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:51.246673 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 19:02:51.246954 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 19:02:51.247009 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 19:02:51.247068 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 19:02:51.247131 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 19:02:51.247826 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.247882 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:51.247948 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 19:02:51.248830 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.248891 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:51.248946 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 19:02:51.257541 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.257636 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:51.257702 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 19:02:51.258583 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.258659 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:51.258723 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 19:02:51.260003 [WARN] raft: Failed to contact 1 in 14.58066ms + logger.go:44: 19:02:51.260098 [WARN] raft: Failed to contact 2 in 14.996275ms + logger.go:44: 19:02:51.260159 [WARN] raft: Failed to contact quorum of nodes, stepping down + logger.go:44: 19:02:51.260233 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.260301 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 19:02:51.260376 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 19:02:51.260015 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.260555 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:51.260629 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 19:02:51.260278 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.260894 [DEBUG] raft-test: close: start + logger.go:44: 19:02:51.260317 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:51.261084 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 19:02:51.261237 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 19:02:51.261524 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 19:02:51.261670 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 19:02:51.261747 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 19:02:51.261818 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 19:02:51.261937 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 19:02:51.262037 [DEBUG] raft-test: close: done === RUN TestControl_DeposeAfterCommandCommittedThenElect === RUN TestControl_DeposeAfterCommandCommittedThenElect/0 === RUN TestControl_DeposeAfterCommandCommittedThenElect/1 === RUN TestControl_DeposeAfterCommandCommittedThenElect/2 ---- PASS: TestControl_DeposeAfterCommandCommittedThenElect (0.26s) - --- PASS: TestControl_DeposeAfterCommandCommittedThenElect/0 (0.08s) - logger.go:44: 18:59:54.714991 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 18:59:54.715405 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 18:59:54.715580 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 18:59:54.715713 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 18:59:54.715835 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 18:59:54.716096 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:54.716180 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 18:59:54.716359 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:54.716432 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 18:59:54.716609 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:54.716684 [DEBUG] raft-test: setup: done - logger.go:44: 18:59:54.716754 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 18:59:54.716903 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 18:59:54.717058 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 18:59:54.717228 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.717387 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.717561 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.740353 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 18:59:54.740546 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 2 - logger.go:44: 18:59:54.740686 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.740728 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 18:59:54.740799 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 18:59:54.740758 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 1 - logger.go:44: 18:59:54.741228 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 18:59:54.741305 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 18:59:54.741409 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.741475 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 18:59:54.741564 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 2 - logger.go:44: 18:59:54.741625 [INFO] raft: Election won. Tally: 2 - logger.go:44: 18:59:54.741736 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 18:59:54.741825 [INFO] raft: Added peer 1, starting replication - logger.go:44: 18:59:54.741900 [INFO] raft: Added peer 2, starting replication - logger.go:44: 18:59:54.742274 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] - logger.go:44: 18:59:54.741597 [INFO] raft: Duplicate RequestVote for same term: 2 - logger.go:44: 18:59:54.742550 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "0") - logger.go:44: 18:59:54.742718 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 18:59:54.742828 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 18:59:54.743509 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] - logger.go:44: 18:59:54.743756 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 18:59:54.744061 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 18:59:54.744126 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 18:59:54.744175 [DEBUG] raft-test: elect: done - logger.go:44: 18:59:54.744428 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] - logger.go:44: 18:59:54.744070 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 18:59:54.744746 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] - logger.go:44: 18:59:54.744813 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.745022 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 18:59:54.745152 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 18:59:54.745280 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 18:59:54.745807 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.745983 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.746054 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 18:59:54.746098 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 18:59:54.746181 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected - logger.go:44: 18:59:54.746265 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 18:59:54.746457 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 18:59:54.746553 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 18:59:54.746864 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 18:59:54.747085 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 18:59:54.747190 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 18:59:54.747855 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.747960 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.748040 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 18:59:54.748135 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.748199 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.748257 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 18:59:54.748765 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.748876 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.748954 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 18:59:54.758552 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.758685 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.758750 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 18:59:54.759527 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.759624 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.759710 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 18:59:54.760283 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.760408 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.760473 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 18:59:54.762028 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.762163 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.762245 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 18:59:54.762555 [WARN] raft: Failed to contact 1 in 17.352256ms - logger.go:44: 18:59:54.762643 [WARN] raft: Failed to contact 2 in 17.496844ms - logger.go:44: 18:59:54.762699 [WARN] raft: Failed to contact quorum of nodes, stepping down - logger.go:44: 18:59:54.762836 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.762927 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 18:59:54.762992 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 18:59:54.763279 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 18:59:54.763359 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 18:59:54.763467 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 18:59:54.771287 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.771773 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.772017 [DEBUG] raft-test: fsm 2: applied 1 - logger.go:44: 18:59:54.773408 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.773993 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.774288 [DEBUG] raft-test: fsm 1: applied 1 - logger.go:44: 18:59:54.792438 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 18:59:54.792701 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 3 - logger.go:44: 18:59:54.792899 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.792982 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 1 - logger.go:44: 18:59:54.793231 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 2 - logger.go:44: 18:59:54.793286 [INFO] raft: Election won. Tally: 2 - logger.go:44: 18:59:54.793360 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 18:59:54.793446 [INFO] raft: Added peer 1, starting replication - logger.go:44: 18:59:54.793521 [INFO] raft: Added peer 2, starting replication - logger.go:44: 18:59:54.793858 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 18:59:54.794011 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 18:59:54.794073 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=3,index=4] - logger.go:44: 18:59:54.794303 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 18:59:54.794639 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=3,index=4] - logger.go:44: 18:59:54.794892 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 18:59:54.795279 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 18:59:54.795350 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 18:59:54.795402 [DEBUG] raft-test: elect: done - logger.go:44: 18:59:54.795607 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=3,index=5] - logger.go:44: 18:59:54.795639 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=3,index=5] - logger.go:44: 18:59:54.795945 [DEBUG] raft-test: fsm 0: applied 2 - logger.go:44: 18:59:54.796072 [DEBUG] raft-test: close: start - logger.go:44: 18:59:54.796217 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.796319 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 18:59:54.796351 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 18:59:54.796560 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.796807 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 18:59:54.796968 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 18:59:54.797165 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 18:59:54.797251 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 18:59:54.797390 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 18:59:54.797457 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 18:59:54.797576 [DEBUG] raft-test: close: done - --- PASS: TestControl_DeposeAfterCommandCommittedThenElect/1 (0.06s) - logger.go:44: 18:59:54.799019 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 18:59:54.800646 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 18:59:54.802757 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 18:59:54.802938 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 18:59:54.803370 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 18:59:54.803632 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:54.803800 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 18:59:54.804016 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:54.804096 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 18:59:54.804296 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:54.804368 [DEBUG] raft-test: setup: done - logger.go:44: 18:59:54.804439 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 18:59:54.804515 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 18:59:54.804605 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 18:59:54.804807 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.804972 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.805122 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.823673 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 18:59:54.823888 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 18:59:54.824031 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.824122 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 18:59:54.824376 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 2 - logger.go:44: 18:59:54.824432 [INFO] raft: Election won. Tally: 2 - logger.go:44: 18:59:54.824502 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 18:59:54.824595 [INFO] raft: Added peer 1, starting replication - logger.go:44: 18:59:54.824673 [INFO] raft: Added peer 2, starting replication - logger.go:44: 18:59:54.825041 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] - logger.go:44: 18:59:54.825383 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 18:59:54.825624 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 18:59:54.826816 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 18:59:54.826070 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] - logger.go:44: 18:59:54.827260 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 18:59:54.827650 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 18:59:54.828153 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 18:59:54.828230 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 18:59:54.828279 [DEBUG] raft-test: elect: done - logger.go:44: 18:59:54.828602 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] - logger.go:44: 18:59:54.828791 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] - logger.go:44: 18:59:54.829068 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 18:59:54.829212 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 18:59:54.829339 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 18:59:54.829847 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.830018 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.830258 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 18:59:54.829793 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.830455 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.830538 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 18:59:54.830468 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 18:59:54.830739 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 18:59:54.830820 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 18:59:54.830902 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected - logger.go:44: 18:59:54.830975 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 18:59:54.831111 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 18:59:54.830821 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 18:59:54.831303 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 18:59:54.832493 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.832613 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.832690 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 18:59:54.832953 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.833053 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.833127 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 18:59:54.843115 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.843337 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.843409 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 18:59:54.843659 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.843734 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.843796 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 18:59:54.844200 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.844279 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.844409 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 18:59:54.844690 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.844774 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.844848 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 18:59:54.845181 [WARN] raft: Failed to contact 1 in 16.145301ms - logger.go:44: 18:59:54.845271 [WARN] raft: Failed to contact 2 in 15.664245ms - logger.go:44: 18:59:54.845327 [WARN] raft: Failed to contact quorum of nodes, stepping down - logger.go:44: 18:59:54.845445 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.845537 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 18:59:54.845634 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 18:59:54.845893 [DEBUG] raft-test: elect: start (server 1) - logger.go:44: 18:59:54.845974 [DEBUG] raft-test: server 1: establish outbound connection to all other nodes - logger.go:44: 18:59:54.846066 [DEBUG] raft-test: elect: server 1: wait to become leader within 1.5s - logger.go:44: 18:59:54.847295 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 18:59:54.847412 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 - logger.go:44: 18:59:54.847533 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.847581 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 18:59:54.847603 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 - logger.go:44: 18:59:54.847679 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 18:59:54.851305 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 18:59:54.851469 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 3 - logger.go:44: 18:59:54.851653 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.851729 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 1 - logger.go:44: 18:59:54.851793 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 2 - logger.go:44: 18:59:54.851846 [INFO] raft: Election won. Tally: 2 - logger.go:44: 18:59:54.851914 [INFO] raft: Node at 1 [Leader] entering Leader state - logger.go:44: 18:59:54.852002 [INFO] raft: Added peer 0, starting replication - logger.go:44: 18:59:54.852077 [INFO] raft: Added peer 2, starting replication - logger.go:44: 18:59:54.852377 [DEBUG] raft-test: server 1: transport: append to 2: 1 entry [Noop:term=3,index=4] - logger.go:44: 18:59:54.851916 [INFO] raft: Duplicate RequestVote for same term: 3 - logger.go:44: 18:59:54.852749 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "1") - logger.go:44: 18:59:54.852919 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 18:59:54.853086 [DEBUG] raft-test: server 1: leadership: acquired - logger.go:44: 18:59:54.853254 [DEBUG] raft-test: elect: server 1: wait for other servers to become followers within 40ms - logger.go:44: 18:59:54.853477 [DEBUG] raft-test: server 1: transport: append to 0: 1 entry [Noop:term=3,index=4] - logger.go:44: 18:59:54.853877 [DEBUG] raft-test: fsm 1: applied 1 - logger.go:44: 18:59:54.853891 [DEBUG] raft-test: elect: server 0: became follower - logger.go:44: 18:59:54.854145 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 18:59:54.854197 [DEBUG] raft-test: elect: done - logger.go:44: 18:59:54.854389 [DEBUG] raft-test: server 1: pipeline: append to 2: 1 entry [Command:term=3,index=5] - logger.go:44: 18:59:54.854106 [INFO] raft: pipelining replication to peer {Voter 0 0} - logger.go:44: 18:59:54.854695 [DEBUG] raft-test: server 1: pipeline: append to 0: 1 entry [Command:term=3,index=5] - logger.go:44: 18:59:54.854976 [DEBUG] raft-test: fsm 1: applied 2 - logger.go:44: 18:59:54.855099 [DEBUG] raft-test: close: start - logger.go:44: 18:59:54.855322 [DEBUG] raft-test: fsm 2: applied 1 - logger.go:44: 18:59:54.855595 [INFO] raft: aborting pipeline replication to peer {Voter 0 0} - logger.go:44: 18:59:54.855707 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 18:59:54.855839 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 18:59:54.856001 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 18:59:54.856140 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 18:59:54.856213 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 18:59:54.856286 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 18:59:54.856838 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 18:59:54.857016 [DEBUG] raft-test: close: done - --- PASS: TestControl_DeposeAfterCommandCommittedThenElect/2 (0.11s) - logger.go:44: 18:59:54.857598 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 18:59:54.858186 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 18:59:54.858384 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 18:59:54.858509 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 18:59:54.858632 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 18:59:54.858916 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:54.859031 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 18:59:54.859151 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.859260 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:54.859341 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 18:59:54.859457 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.859549 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:54.859651 [DEBUG] raft-test: setup: done - logger.go:44: 18:59:54.859715 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.859722 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 18:59:54.859897 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 18:59:54.859987 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 18:59:54.882604 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 18:59:54.882829 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 2 - logger.go:44: 18:59:54.882966 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.883041 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 1 - logger.go:44: 18:59:54.883158 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 18:59:54.883272 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 18:59:54.885230 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 18:59:54.885385 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 2 - logger.go:44: 18:59:54.885538 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.885593 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 18:59:54.885613 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 1 - logger.go:44: 18:59:54.885772 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 18:59:54.889095 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 18:59:54.889259 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 18:59:54.889438 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.889516 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 18:59:54.889526 [INFO] raft: Duplicate RequestVote for same term: 2 - logger.go:44: 18:59:54.889628 [INFO] raft: Duplicate RequestVote for same term: 2 - logger.go:44: 18:59:54.901071 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 18:59:54.901238 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 3 - logger.go:44: 18:59:54.901393 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.901474 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 1 - logger.go:44: 18:59:54.901582 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 18:59:54.901729 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 18:59:54.908126 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 18:59:54.908303 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 - logger.go:44: 18:59:54.908459 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.908694 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 - logger.go:44: 18:59:54.908769 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 18:59:54.908849 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 18:59:54.908884 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 3 - logger.go:44: 18:59:54.909047 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.909117 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 1 - logger.go:44: 18:59:54.909125 [INFO] raft: Duplicate RequestVote for same term: 3 - logger.go:44: 18:59:54.908811 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down - logger.go:44: 18:59:54.909180 [INFO] raft: Duplicate RequestVote for same term: 3 - logger.go:44: 18:59:54.920010 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 18:59:54.920575 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 4 - logger.go:44: 18:59:54.920841 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.920940 [DEBUG] raft: Vote granted from 1 in term 4. Tally: 1 - logger.go:44: 18:59:54.921537 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 18:59:54.921812 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 18:59:54.926665 [WARN] raft: Election timeout reached, restarting election - logger.go:44: 18:59:54.926881 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 4 - logger.go:44: 18:59:54.927029 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.927111 [DEBUG] raft: Vote granted from 0 in term 4. Tally: 1 - logger.go:44: 18:59:54.927391 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.927530 [DEBUG] raft: Vote granted from 2 in term 4. Tally: 2 - logger.go:44: 18:59:54.927585 [INFO] raft: Election won. Tally: 2 - logger.go:44: 18:59:54.927653 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 18:59:54.927801 [INFO] raft: Added peer 1, starting replication - logger.go:44: 18:59:54.927887 [INFO] raft: Added peer 2, starting replication - logger.go:44: 18:59:54.928293 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=4,index=2] - logger.go:44: 18:59:54.929286 [INFO] raft: Duplicate RequestVote for same term: 4 - logger.go:44: 18:59:54.929472 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 18:59:54.929482 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 18:59:54.929966 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 18:59:54.929819 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=4,index=2] - logger.go:44: 18:59:54.930326 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "0") - logger.go:44: 18:59:54.930488 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 18:59:54.930915 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.931281 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 18:59:54.931393 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 18:59:54.931462 [DEBUG] raft-test: elect: done - logger.go:44: 18:59:54.931853 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=4,index=3] - logger.go:44: 18:59:54.932067 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=4,index=3] - logger.go:44: 18:59:54.933178 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.933716 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 18:59:54.933848 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.933802 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 18:59:54.934166 [DEBUG] raft-test: fsm 2: applied 1 - logger.go:44: 18:59:54.934173 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes - logger.go:44: 18:59:54.934416 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 18:59:54.934496 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 18:59:54.934583 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 18:59:54.934695 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 18:59:54.934438 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) - logger.go:44: 18:59:54.935954 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.936101 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.936231 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 18:59:54.936422 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 18:59:54.940151 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected - logger.go:44: 18:59:54.936701 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.942076 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.937083 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.942438 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.942084 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 18:59:54.942252 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 18:59:54.943197 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 18:59:54.943302 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 18:59:54.945671 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.945890 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.945987 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 18:59:54.948996 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.949340 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.949666 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 - logger.go:44: 18:59:54.951237 [WARN] raft: Failed to contact 1 in 17.543054ms - logger.go:44: 18:59:54.951409 [WARN] raft: Failed to contact 2 in 17.048288ms - logger.go:44: 18:59:54.951468 [WARN] raft: Failed to contact quorum of nodes, stepping down - logger.go:44: 18:59:54.951665 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.951763 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 18:59:54.951858 [DEBUG] raft-test: server 0: leadership lost - logger.go:44: 18:59:54.952104 [DEBUG] raft-test: elect: start (server 2) - logger.go:44: 18:59:54.952197 [DEBUG] raft-test: server 2: establish outbound connection to all other nodes - logger.go:44: 18:59:54.952293 [DEBUG] raft-test: elect: server 2: wait to become leader within 1.5s - logger.go:44: 18:59:54.954933 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 18:59:54.955154 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 5 - logger.go:44: 18:59:54.955289 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.955425 [DEBUG] raft: Vote granted from 1 in term 5. Tally: 1 - logger.go:44: 18:59:54.955542 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 18:59:54.955765 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 18:59:54.955917 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.955999 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:54.956070 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 18:59:54.958440 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.958578 [DEBUG] raft-test: server 0: transport: append to 2: not connected - logger.go:44: 18:59:54.958664 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 - logger.go:44: 18:59:54.965058 [WARN] raft: Heartbeat timeout from "0" reached, starting election - logger.go:44: 18:59:54.965224 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 5 - logger.go:44: 18:59:54.965343 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.965419 [DEBUG] raft: Vote granted from 2 in term 5. Tally: 1 - logger.go:44: 18:59:54.965559 [INFO] raft: Duplicate RequestVote for same term: 5 - logger.go:44: 18:59:54.965831 [DEBUG] raft: Vote granted from 0 in term 5. Tally: 2 - logger.go:44: 18:59:54.965887 [INFO] raft: Election won. Tally: 2 - logger.go:44: 18:59:54.965960 [INFO] raft: Node at 2 [Leader] entering Leader state - logger.go:44: 18:59:54.966059 [INFO] raft: Added peer 0, starting replication - logger.go:44: 18:59:54.966145 [INFO] raft: Added peer 1, starting replication - logger.go:44: 18:59:54.966479 [DEBUG] raft-test: server 2: transport: append to 1: 1 entry [Noop:term=5,index=4] - logger.go:44: 18:59:54.966663 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "2") - logger.go:44: 18:59:54.966819 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 18:59:54.966926 [DEBUG] raft-test: server 2: leadership: acquired - logger.go:44: 18:59:54.967073 [DEBUG] raft-test: elect: server 2: wait for other servers to become followers within 40ms - logger.go:44: 18:59:54.967273 [DEBUG] raft-test: server 2: transport: append to 0: 1 entry [Noop:term=5,index=4] - logger.go:44: 18:59:54.967469 [INFO] raft: pipelining replication to peer {Voter 0 0} - logger.go:44: 18:59:54.967598 [DEBUG] raft-test: fsm 1: applied 1 - logger.go:44: 18:59:54.967801 [DEBUG] raft-test: elect: server 0: became follower - logger.go:44: 18:59:54.967865 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 18:59:54.967916 [DEBUG] raft-test: elect: done - logger.go:44: 18:59:54.968133 [DEBUG] raft-test: server 2: pipeline: append to 1: 1 entry [Command:term=5,index=5] - logger.go:44: 18:59:54.968462 [DEBUG] raft-test: server 2: pipeline: append to 0: 1 entry [Command:term=5,index=5] - logger.go:44: 18:59:54.968862 [DEBUG] raft-test: fsm 2: applied 2 - logger.go:44: 18:59:54.969011 [DEBUG] raft-test: close: start - logger.go:44: 18:59:54.969221 [INFO] raft: aborting pipeline replication to peer {Voter 0 0} - logger.go:44: 18:59:54.969334 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 18:59:54.969474 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 18:59:54.969636 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 18:59:54.970261 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 18:59:54.970389 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 18:59:54.970476 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 18:59:54.970627 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 18:59:54.970695 [DEBUG] raft-test: close: done +--- PASS: TestControl_DeposeAfterCommandCommittedThenElect (0.34s) + --- PASS: TestControl_DeposeAfterCommandCommittedThenElect/0 (0.15s) + logger.go:44: 19:02:51.262392 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 19:02:51.262734 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 19:02:51.262888 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 19:02:51.263020 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 19:02:51.263145 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 19:02:51.263348 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:51.263442 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 19:02:51.263544 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.263650 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:51.263732 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 19:02:51.263809 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.263951 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:51.264037 [DEBUG] raft-test: setup: done + logger.go:44: 19:02:51.264111 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 19:02:51.264184 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 19:02:51.264112 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.264272 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 19:02:51.284552 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 19:02:51.284803 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 2 + logger.go:44: 19:02:51.285149 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.285262 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:51.285577 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 19:02:51.285647 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 1 + logger.go:44: 19:02:51.289976 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 19:02:51.290220 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 19:02:51.290516 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.290708 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 19:02:51.290756 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 19:02:51.291198 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 2 + logger.go:44: 19:02:51.291347 [INFO] raft: Election won. Tally: 2 + logger.go:44: 19:02:51.291545 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 19:02:51.291834 [INFO] raft: Added peer 1, starting replication + logger.go:44: 19:02:51.292007 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 19:02:51.292449 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.292457 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 19:02:51.293015 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 19:02:51.292102 [INFO] raft: Added peer 2, starting replication + logger.go:44: 19:02:51.292854 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "0") + logger.go:44: 19:02:51.293982 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Noop:term=2,index=2] + logger.go:44: 19:02:51.294911 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.294932 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 19:02:51.294975 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] + logger.go:44: 19:02:51.295262 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 19:02:51.295902 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.296214 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 19:02:51.296276 [DEBUG] raft-test: elect: done + logger.go:44: 19:02:51.296222 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 19:02:51.296466 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] + logger.go:44: 19:02:51.296470 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] + logger.go:44: 19:02:51.296699 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 19:02:51.296809 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 19:02:51.296907 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 19:02:51.297902 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 19:02:51.297966 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 19:02:51.298023 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 19:02:51.298101 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 19:02:51.298101 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 19:02:51.298233 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected + logger.go:44: 19:02:51.298293 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 19:02:51.298033 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.298395 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:51.298244 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.298547 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:51.298596 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 19:02:51.298356 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 19:02:51.298460 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 19:02:51.300115 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.300171 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:51.300226 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 19:02:51.300511 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.300566 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:51.300618 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 19:02:51.310402 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.310480 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:51.310538 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 19:02:51.311730 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.311793 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:51.311857 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 19:02:51.312187 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.312258 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:51.312320 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 19:02:51.312471 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.312533 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:51.312603 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 19:02:51.314279 [WARN] raft: Failed to contact 1 in 17.560258ms + logger.go:44: 19:02:51.314399 [WARN] raft: Failed to contact 2 in 17.605465ms + logger.go:44: 19:02:51.314487 [WARN] raft: Failed to contact quorum of nodes, stepping down + logger.go:44: 19:02:51.314648 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.314848 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 19:02:51.315163 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 19:02:51.315598 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 19:02:51.315723 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 19:02:51.315861 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 19:02:51.317193 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 19:02:51.317382 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 3 + logger.go:44: 19:02:51.317555 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.317683 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 1 + logger.go:44: 19:02:51.317716 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 19:02:51.317602 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:51.318372 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 19:02:51.318498 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 + logger.go:44: 19:02:51.318650 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.318763 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 + logger.go:44: 19:02:51.318695 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:51.318819 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 19:02:51.323201 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.323408 [DEBUG] raft-test: server 0: transport: append to 2: newer term + logger.go:44: 19:02:51.324450 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.324645 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.324854 [DEBUG] raft-test: server 0: transport: append to 1: newer term + logger.go:44: 19:02:51.324993 [ERR] raft: peer {Voter 1 1} has newer term, stopping replication + logger.go:44: 19:02:51.324466 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.325285 [DEBUG] raft-test: server 0: transport: append to 2: newer term + logger.go:44: 19:02:51.325448 [ERR] raft: peer {Voter 2 2} has newer term, stopping replication + logger.go:44: 19:02:51.324675 [DEBUG] raft-test: server 0: transport: append to 1: newer term + logger.go:44: 19:02:51.330520 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 19:02:51.330606 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 3 + logger.go:44: 19:02:51.330696 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.330743 [INFO] raft: Duplicate RequestVote for same term: 3 + logger.go:44: 19:02:51.330813 [INFO] raft: Duplicate RequestVote for same term: 3 + logger.go:44: 19:02:51.330760 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 1 + logger.go:44: 19:02:51.338451 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 19:02:51.338640 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 4 + logger.go:44: 19:02:51.338824 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.338942 [DEBUG] raft: Vote granted from 1 in term 4. Tally: 1 + logger.go:44: 19:02:51.338962 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 19:02:51.338863 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:51.344135 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 19:02:51.344299 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 4 + logger.go:44: 19:02:51.344507 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.344639 [DEBUG] raft: Vote granted from 2 in term 4. Tally: 1 + logger.go:44: 19:02:51.344539 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:51.344670 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 19:02:51.354176 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 19:02:51.354315 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 4 + logger.go:44: 19:02:51.354515 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.354646 [DEBUG] raft: Vote granted from 0 in term 4. Tally: 1 + logger.go:44: 19:02:51.354257 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 19:02:51.354928 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 5 + logger.go:44: 19:02:51.355088 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.355270 [DEBUG] raft: Vote granted from 1 in term 5. Tally: 1 + logger.go:44: 19:02:51.354726 [INFO] raft: Duplicate RequestVote for same term: 4 + logger.go:44: 19:02:51.355353 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:51.355419 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 19:02:51.355454 [DEBUG] raft: Newer term discovered, fallback to follower + logger.go:44: 19:02:51.356125 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.365390 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 19:02:51.365547 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 5 + logger.go:44: 19:02:51.365754 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.365873 [DEBUG] raft: Vote granted from 2 in term 5. Tally: 1 + logger.go:44: 19:02:51.365893 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:51.366015 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 19:02:51.377039 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 19:02:51.377228 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 6 + logger.go:44: 19:02:51.377517 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.377660 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:51.377769 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 19:02:51.377676 [DEBUG] raft: Vote granted from 1 in term 6. Tally: 1 + logger.go:44: 19:02:51.381230 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 19:02:51.381540 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 6 + logger.go:44: 19:02:51.381751 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.381882 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:51.382011 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 19:02:51.381897 [DEBUG] raft: Vote granted from 2 in term 6. Tally: 1 + logger.go:44: 19:02:51.385994 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 19:02:51.386176 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 6 + logger.go:44: 19:02:51.386379 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.386477 [INFO] raft: Duplicate RequestVote for same term: 6 + logger.go:44: 19:02:51.386503 [DEBUG] raft: Vote granted from 0 in term 6. Tally: 1 + logger.go:44: 19:02:51.386583 [INFO] raft: Duplicate RequestVote for same term: 6 + logger.go:44: 19:02:51.397630 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 19:02:51.397767 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 7 + logger.go:44: 19:02:51.397975 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.398140 [DEBUG] raft: Vote granted from 1 in term 7. Tally: 1 + logger.go:44: 19:02:51.398167 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 19:02:51.398030 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:51.405009 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 19:02:51.405168 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 7 + logger.go:44: 19:02:51.405460 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.405580 [INFO] raft: Duplicate RequestVote for same term: 7 + logger.go:44: 19:02:51.405605 [DEBUG] raft: Vote granted from 0 in term 7. Tally: 1 + logger.go:44: 19:02:51.406131 [DEBUG] raft: Vote granted from 2 in term 7. Tally: 2 + logger.go:44: 19:02:51.405864 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.406240 [INFO] raft: Election won. Tally: 2 + logger.go:44: 19:02:51.406631 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 19:02:51.406819 [INFO] raft: Added peer 1, starting replication + logger.go:44: 19:02:51.407021 [INFO] raft: Added peer 2, starting replication + logger.go:44: 19:02:51.406875 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 19:02:51.407515 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 19:02:51.407410 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.407595 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=7,index=4] + logger.go:44: 19:02:51.408109 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "0") + logger.go:44: 19:02:51.408250 [DEBUG] raft-test: fsm 2: applied 1 + logger.go:44: 19:02:51.408170 [DEBUG] raft-test: fsm 1: applied 1 + logger.go:44: 19:02:51.408340 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 19:02:51.408392 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 19:02:51.409388 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.409618 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Noop:term=7,index=4] + logger.go:44: 19:02:51.409980 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.410019 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 19:02:51.410177 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 19:02:51.410225 [DEBUG] raft-test: elect: done + logger.go:44: 19:02:51.410384 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=7,index=5] + logger.go:44: 19:02:51.410386 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=7,index=5] + logger.go:44: 19:02:51.410639 [DEBUG] raft-test: fsm 0: applied 2 + logger.go:44: 19:02:51.410751 [DEBUG] raft-test: close: start + logger.go:44: 19:02:51.410882 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 19:02:51.410899 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 19:02:51.411078 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 19:02:51.411207 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 19:02:51.411337 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 19:02:51.411412 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 19:02:51.411494 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 19:02:51.411629 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 19:02:51.411736 [DEBUG] raft-test: close: done + --- PASS: TestControl_DeposeAfterCommandCommittedThenElect/1 (0.13s) + logger.go:44: 19:02:51.412278 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 19:02:51.412566 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 19:02:51.412715 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 19:02:51.412832 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 19:02:51.412940 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 19:02:51.413134 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:51.413223 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 19:02:51.413290 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.413456 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:51.413536 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 19:02:51.413602 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.413718 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:51.413794 [DEBUG] raft-test: setup: done + logger.go:44: 19:02:51.413860 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.413867 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 19:02:51.414031 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 19:02:51.414109 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 19:02:51.429758 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 19:02:51.429844 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 2 + logger.go:44: 19:02:51.430083 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.430298 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 1 + logger.go:44: 19:02:51.430376 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 19:02:51.430176 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:51.435547 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 19:02:51.435786 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 2 + logger.go:44: 19:02:51.436082 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.436231 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:51.436389 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 19:02:51.436283 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 1 + logger.go:44: 19:02:51.441059 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 19:02:51.441304 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 19:02:51.441644 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.441750 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 19:02:51.441755 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 19:02:51.441845 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 19:02:51.455985 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 19:02:51.456229 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 3 + logger.go:44: 19:02:51.456453 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.456603 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 1 + logger.go:44: 19:02:51.456675 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 19:02:51.456512 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:51.464616 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 19:02:51.464775 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 + logger.go:44: 19:02:51.464986 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.465103 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 + logger.go:44: 19:02:51.465160 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 19:02:51.465033 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:51.464991 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 19:02:51.465644 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 3 + logger.go:44: 19:02:51.465829 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.465909 [INFO] raft: Duplicate RequestVote for same term: 3 + logger.go:44: 19:02:51.466020 [INFO] raft: Duplicate RequestVote for same term: 3 + logger.go:44: 19:02:51.465937 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 1 + logger.go:44: 19:02:51.474433 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 19:02:51.474564 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 4 + logger.go:44: 19:02:51.474727 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.474839 [DEBUG] raft: Vote granted from 1 in term 4. Tally: 1 + logger.go:44: 19:02:51.474854 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 19:02:51.474786 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:51.482592 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 19:02:51.482709 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 4 + logger.go:44: 19:02:51.482864 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.482962 [INFO] raft: Duplicate RequestVote for same term: 4 + logger.go:44: 19:02:51.482995 [DEBUG] raft: Vote granted from 0 in term 4. Tally: 1 + logger.go:44: 19:02:51.483237 [DEBUG] raft: Vote granted from 2 in term 4. Tally: 2 + logger.go:44: 19:02:51.483327 [INFO] raft: Election won. Tally: 2 + logger.go:44: 19:02:51.483453 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 19:02:51.483130 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.483575 [INFO] raft: Added peer 1, starting replication + logger.go:44: 19:02:51.483832 [INFO] raft: Added peer 2, starting replication + logger.go:44: 19:02:51.483605 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 19:02:51.484171 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=4,index=2] + logger.go:44: 19:02:51.484201 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 19:02:51.484219 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=4,index=2] + logger.go:44: 19:02:51.484565 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 19:02:51.484407 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "0") + logger.go:44: 19:02:51.484783 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 19:02:51.484515 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 19:02:51.485863 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 19:02:51.485937 [DEBUG] raft-test: elect: done + logger.go:44: 19:02:51.485883 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.486199 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=4,index=3] + logger.go:44: 19:02:51.486241 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=4,index=3] + logger.go:44: 19:02:51.486528 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 19:02:51.486666 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 19:02:51.486791 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 19:02:51.486942 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.487060 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:51.487127 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 19:02:51.487620 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 19:02:51.487694 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected + logger.go:44: 19:02:51.487770 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 19:02:51.487862 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 19:02:51.487968 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.488048 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:51.488097 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 19:02:51.488191 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 19:02:51.488115 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 19:02:51.488271 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 19:02:51.488460 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 19:02:51.489189 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.489265 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:51.489366 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 19:02:51.490619 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.490693 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:51.490767 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 19:02:51.500023 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.500115 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:51.500200 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 19:02:51.500349 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.500429 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:51.500494 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 19:02:51.501003 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.501076 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:51.501149 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 19:02:51.502824 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.502897 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:51.502999 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 19:02:51.504359 [WARN] raft: Failed to contact 1 in 17.765586ms + logger.go:44: 19:02:51.504439 [WARN] raft: Failed to contact 2 in 17.881375ms + logger.go:44: 19:02:51.504499 [WARN] raft: Failed to contact quorum of nodes, stepping down + logger.go:44: 19:02:51.504602 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.504674 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 19:02:51.504619 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 19:02:51.504858 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 5 + logger.go:44: 19:02:51.504975 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.505055 [DEBUG] raft: Vote granted from 1 in term 5. Tally: 1 + logger.go:44: 19:02:51.504763 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 19:02:51.505060 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 19:02:51.505021 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:51.505116 [DEBUG] raft-test: elect: start (server 1) + logger.go:44: 19:02:51.505619 [DEBUG] raft-test: server 1: establish outbound connection to all other nodes + logger.go:44: 19:02:51.505750 [DEBUG] raft-test: elect: server 1: wait to become leader within 1.5s + logger.go:44: 19:02:51.512881 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 19:02:51.513034 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 5 + logger.go:44: 19:02:51.512898 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.513246 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:51.512908 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.513509 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.513629 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:51.513252 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:51.513543 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:51.514007 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 19:02:51.513199 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.514306 [DEBUG] raft: Vote granted from 2 in term 5. Tally: 1 + logger.go:44: 19:02:51.513257 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 19:02:51.513397 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 19:02:51.513751 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 19:02:51.514483 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.514961 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:51.515071 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 19:02:51.521833 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 19:02:51.521919 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 5 + logger.go:44: 19:02:51.522039 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.522119 [DEBUG] raft: Vote granted from 0 in term 5. Tally: 1 + logger.go:44: 19:02:51.522161 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 19:02:51.522063 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 19:02:51.532637 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 19:02:51.532909 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 6 + logger.go:44: 19:02:51.538402 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.538681 [DEBUG] raft: Vote granted from 1 in term 6. Tally: 1 + logger.go:44: 19:02:51.540872 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.541139 [DEBUG] raft: Vote granted from 2 in term 6. Tally: 2 + logger.go:44: 19:02:51.542517 [INFO] raft: Election won. Tally: 2 + logger.go:44: 19:02:51.542589 [INFO] raft: Node at 1 [Leader] entering Leader state + logger.go:44: 19:02:51.542662 [INFO] raft: Added peer 0, starting replication + logger.go:44: 19:02:51.542734 [INFO] raft: Added peer 2, starting replication + logger.go:44: 19:02:51.542893 [DEBUG] raft-test: server 1: leadership: acquired + logger.go:44: 19:02:51.543042 [DEBUG] raft-test: elect: server 1: wait for other servers to become followers within 40ms + logger.go:44: 19:02:51.542947 [DEBUG] raft-test: server 1: transport: append to 2: 1 entry [Noop:term=6,index=4] + logger.go:44: 19:02:51.543383 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 19:02:51.543055 [DEBUG] raft-test: server 1: transport: append to 0: 1 entry [Noop:term=6,index=4] + logger.go:44: 19:02:51.541574 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.543671 [INFO] raft: pipelining replication to peer {Voter 0 0} + logger.go:44: 19:02:51.543788 [DEBUG] raft-test: fsm 1: applied 1 + logger.go:44: 19:02:51.544302 [DEBUG] raft-test: elect: server 0: became follower + logger.go:44: 19:02:51.544365 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 19:02:51.544405 [DEBUG] raft-test: elect: done + logger.go:44: 19:02:51.544559 [DEBUG] raft-test: server 1: pipeline: append to 0: 1 entry [Command:term=6,index=5] + logger.go:44: 19:02:51.544560 [DEBUG] raft-test: server 1: pipeline: append to 2: 1 entry [Command:term=6,index=5] + logger.go:44: 19:02:51.544857 [DEBUG] raft-test: fsm 1: applied 2 + logger.go:44: 19:02:51.544958 [DEBUG] raft-test: close: start + logger.go:44: 19:02:51.544873 [DEBUG] raft-test: fsm 2: applied 1 + logger.go:44: 19:02:51.545207 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 19:02:51.545302 [INFO] raft: aborting pipeline replication to peer {Voter 0 0} + logger.go:44: 19:02:51.545434 [DEBUG] raft-test: server 1: transport: append to 0: 0 entries + logger.go:44: 19:02:51.545616 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 19:02:51.545737 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 19:02:51.545866 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 19:02:51.545930 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 19:02:51.546025 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 19:02:51.546084 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 19:02:51.546132 [DEBUG] raft-test: close: done + --- PASS: TestControl_DeposeAfterCommandCommittedThenElect/2 (0.06s) + logger.go:44: 19:02:51.546507 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 19:02:51.546769 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 19:02:51.546893 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 19:02:51.547003 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 19:02:51.547101 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 19:02:51.547281 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:51.547355 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 19:02:51.547444 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.547519 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:51.547600 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 19:02:51.547661 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.547753 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:51.547842 [DEBUG] raft-test: setup: done + logger.go:44: 19:02:51.547903 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 19:02:51.547912 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.547961 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 19:02:51.548116 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 19:02:51.567365 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 19:02:51.567633 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 2 + logger.go:44: 19:02:51.567959 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.568200 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 1 + logger.go:44: 19:02:51.568330 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 19:02:51.568069 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:51.570070 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 19:02:51.570199 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 19:02:51.570372 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.570450 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 19:02:51.570489 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 19:02:51.570721 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 2 + logger.go:44: 19:02:51.570805 [INFO] raft: Election won. Tally: 2 + logger.go:44: 19:02:51.570910 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 19:02:51.571042 [INFO] raft: Added peer 1, starting replication + logger.go:44: 19:02:51.571148 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 19:02:51.571391 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 19:02:51.571204 [INFO] raft: Added peer 2, starting replication + logger.go:44: 19:02:51.571407 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.572060 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] + logger.go:44: 19:02:51.572170 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 19:02:51.572358 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Noop:term=2,index=2] + logger.go:44: 19:02:51.572071 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "0") + logger.go:44: 19:02:51.572368 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 19:02:51.572957 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 19:02:51.573075 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 19:02:51.573157 [DEBUG] raft-test: elect: done + logger.go:44: 19:02:51.573572 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] + logger.go:44: 19:02:51.573618 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] + logger.go:44: 19:02:51.573778 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.573783 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.574001 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 19:02:51.574652 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 19:02:51.574860 [DEBUG] raft-test: node 0: state: wait leadership lost (timeout=40ms) + logger.go:44: 19:02:51.575234 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 19:02:51.575350 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 19:02:51.575476 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 19:02:51.575615 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 19:02:51.575961 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 19:02:51.576078 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected + logger.go:44: 19:02:51.576192 [ERR] raft: Failed to pipeline AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 19:02:51.576332 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 19:02:51.576847 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.576957 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:51.577053 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 19:02:51.577184 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.577284 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:51.577412 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 19:02:51.577658 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.577768 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:51.577878 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 19:02:51.578046 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.578209 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:51.578338 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 19:02:51.589414 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.589486 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:51.589538 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 19:02:51.589421 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.589657 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:51.589722 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 19:02:51.589593 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.589842 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:51.589890 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 19:02:51.590762 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.590826 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:51.590887 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 19:02:51.592220 [WARN] raft: Failed to contact 2 in 17.877ms + logger.go:44: 19:02:51.592376 [WARN] raft: Failed to contact 1 in 17.659714ms + logger.go:44: 19:02:51.592489 [WARN] raft: Failed to contact quorum of nodes, stepping down + logger.go:44: 19:02:51.592688 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.592786 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 19:02:51.592959 [DEBUG] raft-test: server 0: leadership lost + logger.go:44: 19:02:51.593435 [DEBUG] raft-test: elect: start (server 2) + logger.go:44: 19:02:51.593608 [DEBUG] raft-test: server 2: establish outbound connection to all other nodes + logger.go:44: 19:02:51.593782 [DEBUG] raft-test: elect: server 2: wait to become leader within 1.5s + logger.go:44: 19:02:51.601236 [WARN] raft: Heartbeat timeout from "0" reached, starting election + logger.go:44: 19:02:51.601454 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 3 + logger.go:44: 19:02:51.601626 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.601700 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 1 + logger.go:44: 19:02:51.601770 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 2 + logger.go:44: 19:02:51.601824 [INFO] raft: Election won. Tally: 2 + logger.go:44: 19:02:51.601913 [INFO] raft: Node at 2 [Leader] entering Leader state + logger.go:44: 19:02:51.601860 [WARN] raft: Rejecting vote request from 2 since we have a leader: 0 + logger.go:44: 19:02:51.602068 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.602158 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:51.602005 [INFO] raft: Added peer 0, starting replication + logger.go:44: 19:02:51.602337 [INFO] raft: Added peer 1, starting replication + logger.go:44: 19:02:51.602027 [DEBUG] raft-test: server 2: leadership: acquired + logger.go:44: 19:02:51.602547 [DEBUG] raft-test: elect: server 2: wait for other servers to become followers within 40ms + logger.go:44: 19:02:51.602602 [DEBUG] raft-test: server 2: transport: append to 1: 1 entry [Noop:term=3,index=4] + logger.go:44: 19:02:51.602556 [DEBUG] raft-test: server 2: transport: append to 0: 1 entry [Noop:term=3,index=4] + logger.go:44: 19:02:51.602238 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 19:02:51.602907 [DEBUG] raft-test: fsm 2: applied 1 + logger.go:44: 19:02:51.602037 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.603121 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:51.603190 [ERR] raft: Failed to heartbeat to 2: cannot reach server 2 + logger.go:44: 19:02:51.602846 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.603348 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:51.602866 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 19:02:51.603009 [INFO] raft: pipelining replication to peer {Voter 0 0} + logger.go:44: 19:02:51.603424 [ERR] raft: Failed to AppendEntries to {Voter 2 2}: cannot reach server 2 + logger.go:44: 19:02:51.603818 [DEBUG] raft-test: elect: server 0: became follower + logger.go:44: 19:02:51.603889 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 19:02:51.603944 [DEBUG] raft-test: elect: done + logger.go:44: 19:02:51.604134 [DEBUG] raft-test: server 2: pipeline: append to 1: 1 entry [Command:term=3,index=5] + logger.go:44: 19:02:51.604147 [DEBUG] raft-test: server 2: pipeline: append to 0: 1 entry [Command:term=3,index=5] + logger.go:44: 19:02:51.604412 [DEBUG] raft-test: fsm 1: applied 1 + logger.go:44: 19:02:51.604502 [DEBUG] raft-test: fsm 2: applied 2 + logger.go:44: 19:02:51.604669 [DEBUG] raft-test: close: start + logger.go:44: 19:02:51.604815 [INFO] raft: aborting pipeline replication to peer {Voter 0 0} + logger.go:44: 19:02:51.604842 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 19:02:51.604846 [DEBUG] raft-test: server 2: transport: append to 1: 0 entries + logger.go:44: 19:02:51.605363 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 19:02:51.605531 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 19:02:51.605699 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 19:02:51.605776 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 19:02:51.605851 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 19:02:51.605968 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 19:02:51.606029 [DEBUG] raft-test: close: done === RUN TestControl_SnapshotAfterCommandCommitted ---- PASS: TestControl_SnapshotAfterCommandCommitted (0.03s) - logger.go:44: 18:59:54.971769 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 18:59:54.972253 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 18:59:54.972477 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 18:59:54.972634 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 18:59:54.972786 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 18:59:54.973104 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:54.973250 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 18:59:54.973496 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:54.973597 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 18:59:54.973959 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.974195 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.974582 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:54.974762 [DEBUG] raft-test: setup: done - logger.go:44: 18:59:54.974843 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 18:59:54.974926 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 18:59:54.975462 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 18:59:54.975272 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:54.992531 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 18:59:54.992713 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 18:59:54.992836 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:54.992909 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 18:59:54.993096 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 2 - logger.go:44: 18:59:54.993148 [INFO] raft: Election won. Tally: 2 - logger.go:44: 18:59:54.993214 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 18:59:54.993371 [INFO] raft: Added peer 1, starting replication - logger.go:44: 18:59:54.993454 [INFO] raft: Added peer 2, starting replication - logger.go:44: 18:59:54.993806 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] - logger.go:44: 18:59:54.994060 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 18:59:54.993844 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 18:59:54.994312 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 18:59:54.994162 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] - logger.go:44: 18:59:54.994675 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 18:59:54.995747 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 18:59:54.995907 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 18:59:54.995967 [DEBUG] raft-test: elect: done - logger.go:44: 18:59:54.996256 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 18:59:54.996450 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] - logger.go:44: 18:59:54.996309 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] - logger.go:44: 18:59:54.996647 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.996907 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 18:59:54.997083 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.997183 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 18:59:54.997293 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=4] - logger.go:44: 18:59:54.997428 [DEBUG] raft-test: fsm 0: applied 2 - logger.go:44: 18:59:54.997492 [DEBUG] raft-test: server 0: control: take snapshot - logger.go:44: 18:59:54.997543 [DEBUG] raft-test: fsm 1: applied 1 - logger.go:44: 18:59:54.997591 [DEBUG] raft-test: fsm 2: applied 1 - logger.go:44: 18:59:54.997713 [INFO] raft: Starting snapshot up to 4 - logger.go:44: 18:59:54.998321 [INFO] raft: Compacting logs from 1 to 3 - logger.go:44: 18:59:54.998392 [INFO] raft: Snapshot to 4 complete - logger.go:44: 18:59:54.998546 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 18:59:54.998628 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Other:term=2,index=5] - logger.go:44: 18:59:54.998713 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Other:term=2,index=5] - logger.go:44: 18:59:54.998934 [DEBUG] raft-test: fsm 2: applied 2 - logger.go:44: 18:59:54.999106 [DEBUG] raft-test: fsm 1: applied 2 - logger.go:44: 18:59:54.999222 [DEBUG] raft-test: close: start - logger.go:44: 18:59:54.999265 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:54.999431 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 18:59:54.999434 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:54.999466 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 18:59:54.999815 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 18:59:54.999981 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 18:59:55.000119 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 18:59:55.000198 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 18:59:55.000266 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 18:59:55.000402 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 18:59:55.000502 [DEBUG] raft-test: close: done +--- PASS: TestControl_SnapshotAfterCommandCommitted (0.06s) + logger.go:44: 19:02:51.606641 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 19:02:51.607004 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 19:02:51.607200 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 19:02:51.607360 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 19:02:51.607494 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 19:02:51.607747 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:51.607835 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 19:02:51.608045 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:51.608127 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 19:02:51.608317 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:51.608394 [DEBUG] raft-test: setup: done + logger.go:44: 19:02:51.608467 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 19:02:51.608572 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 19:02:51.608682 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 19:02:51.608835 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.609004 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.609159 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.627954 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 19:02:51.628256 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 2 + logger.go:44: 19:02:51.628587 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.628801 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 1 + logger.go:44: 19:02:51.628881 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:51.629023 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 19:02:51.631931 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 19:02:51.632193 [INFO] raft: Node at 2 [Candidate] entering Candidate state in term 2 + logger.go:44: 19:02:51.632403 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.632451 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:51.632517 [ERR] raft: Failed to make RequestVote RPC to {Voter 1 1}: connectivity to server 1 is down + logger.go:44: 19:02:51.632477 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 1 + logger.go:44: 19:02:51.636375 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 19:02:51.636461 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 19:02:51.636591 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.636648 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 19:02:51.636733 [INFO] raft: Duplicate RequestVote for same term: 2 + logger.go:44: 19:02:51.636666 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 19:02:51.646029 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 19:02:51.646184 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 3 + logger.go:44: 19:02:51.646412 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.646501 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down + logger.go:44: 19:02:51.646558 [DEBUG] raft: Vote granted from 1 in term 3. Tally: 1 + logger.go:44: 19:02:51.646620 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down + logger.go:44: 19:02:51.653212 [WARN] raft: Election timeout reached, restarting election + logger.go:44: 19:02:51.653410 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 3 + logger.go:44: 19:02:51.653573 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.653670 [INFO] raft: Duplicate RequestVote for same term: 3 + logger.go:44: 19:02:51.653683 [DEBUG] raft: Vote granted from 0 in term 3. Tally: 1 + logger.go:44: 19:02:51.653996 [DEBUG] raft: Vote granted from 2 in term 3. Tally: 2 + logger.go:44: 19:02:51.654085 [INFO] raft: Election won. Tally: 2 + logger.go:44: 19:02:51.653871 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.654196 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 19:02:51.654437 [INFO] raft: Added peer 1, starting replication + logger.go:44: 19:02:51.654565 [INFO] raft: Added peer 2, starting replication + logger.go:44: 19:02:51.654939 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=3,index=2] + logger.go:44: 19:02:51.655458 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 19:02:51.655611 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=3,index=2] + logger.go:44: 19:02:51.655685 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 19:02:51.656001 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 19:02:51.656097 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 19:02:51.656208 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 19:02:51.656288 [DEBUG] raft-test: elect: done + logger.go:44: 19:02:51.655797 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 19:02:51.656879 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=3,index=3] + logger.go:44: 19:02:51.655886 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "0") + logger.go:44: 19:02:51.657192 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=3,index=3] + logger.go:44: 19:02:51.657519 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 19:02:51.657787 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.657973 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=3,index=4] + logger.go:44: 19:02:51.657988 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=3,index=4] + logger.go:44: 19:02:51.658398 [DEBUG] raft-test: fsm 0: applied 2 + logger.go:44: 19:02:51.658505 [DEBUG] raft-test: server 0: control: take snapshot + logger.go:44: 19:02:51.658542 [DEBUG] raft-test: fsm 2: applied 1 + logger.go:44: 19:02:51.658834 [INFO] raft: Starting snapshot up to 4 + logger.go:44: 19:02:51.659043 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.659207 [INFO] raft: Compacting logs from 1 to 3 + logger.go:44: 19:02:51.659832 [INFO] raft: Snapshot to 4 complete + logger.go:44: 19:02:51.660149 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Other:term=3,index=5] + logger.go:44: 19:02:51.658591 [DEBUG] raft-test: fsm 1: applied 1 + logger.go:44: 19:02:51.659531 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 19:02:51.660734 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.660762 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Other:term=3,index=5] + logger.go:44: 19:02:51.661096 [DEBUG] raft-test: fsm 1: applied 2 + logger.go:44: 19:02:51.661246 [DEBUG] raft-test: fsm 2: applied 2 + logger.go:44: 19:02:51.661473 [DEBUG] raft-test: close: start + logger.go:44: 19:02:51.661762 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 19:02:51.661877 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 19:02:51.661793 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.662297 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 19:02:51.662534 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 19:02:51.662770 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 19:02:51.662894 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 19:02:51.663012 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 19:02:51.663222 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 19:02:51.663390 [DEBUG] raft-test: close: done === RUN TestControl_RestoreAfterDisconnection ---- PASS: TestControl_RestoreAfterDisconnection (0.05s) - logger.go:44: 18:59:55.000825 [DEBUG] raft-test: setup: start (3 servers) - logger.go:44: 18:59:55.001199 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 18:59:55.001407 [DEBUG] raft-test: setup: server 1: bootstrap - logger.go:44: 18:59:55.001560 [DEBUG] raft-test: setup: server 2: bootstrap - logger.go:44: 18:59:55.002029 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 18:59:55.002542 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:55.002670 [DEBUG] raft-test: setup: server 1: start - logger.go:44: 18:59:55.002779 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:55.002914 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:55.002991 [DEBUG] raft-test: setup: server 2: start - logger.go:44: 18:59:55.003071 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:55.003174 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] - logger.go:44: 18:59:55.003304 [DEBUG] raft-test: setup: done - logger.go:44: 18:59:55.003381 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 18:59:55.003459 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 18:59:55.003550 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 18:59:55.003707 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:55.024890 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 18:59:55.025126 [INFO] raft: Node at 1 [Candidate] entering Candidate state in term 2 - logger.go:44: 18:59:55.025265 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:55.025339 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 1 - logger.go:44: 18:59:55.025401 [ERR] raft: Failed to make RequestVote RPC to {Voter 2 2}: connectivity to server 2 is down - logger.go:44: 18:59:55.025364 [ERR] raft: Failed to make RequestVote RPC to {Voter 0 0}: connectivity to server 0 is down - logger.go:44: 18:59:55.030027 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 18:59:55.030187 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 18:59:55.030302 [DEBUG] raft: Votes needed: 2 - logger.go:44: 18:59:55.030374 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 18:59:55.030560 [DEBUG] raft: Vote granted from 2 in term 2. Tally: 2 - logger.go:44: 18:59:55.030613 [INFO] raft: Election won. Tally: 2 - logger.go:44: 18:59:55.030679 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 18:59:55.030760 [INFO] raft: Added peer 1, starting replication - logger.go:44: 18:59:55.030832 [INFO] raft: Added peer 2, starting replication - logger.go:44: 18:59:55.031196 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] - logger.go:44: 18:59:55.031414 [INFO] raft: pipelining replication to peer {Voter 2 2} - logger.go:44: 18:59:55.031565 [INFO] raft: Duplicate RequestVote for same term: 2 - logger.go:44: 18:59:55.031660 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 18:59:55.031802 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 18:59:55.032000 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=2,index=2] - logger.go:44: 18:59:55.032140 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "0") - logger.go:44: 18:59:55.032281 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 18:59:55.033833 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 18:59:55.034199 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 18:59:55.034302 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 18:59:55.034354 [DEBUG] raft-test: elect: done - logger.go:44: 18:59:55.034686 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] - logger.go:44: 18:59:55.034687 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] - logger.go:44: 18:59:55.035011 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 18:59:55.035227 [DEBUG] raft-test: term: disconnect 1 - logger.go:44: 18:59:55.035295 [DEBUG] raft-test: server 0: disconnecting follower 1 - logger.go:44: 18:59:55.035343 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:55.035783 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:55.035873 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 - logger.go:44: 18:59:55.035472 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=4] - logger.go:44: 18:59:55.035519 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=4] - logger.go:44: 18:59:55.036287 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 18:59:55.036376 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 18:59:55.036444 [DEBUG] raft-test: fsm 2: applied 1 - logger.go:44: 18:59:55.036416 [DEBUG] raft-test: fsm 0: applied 2 - logger.go:44: 18:59:55.036478 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 18:59:55.036861 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=5] - logger.go:44: 18:59:55.036931 [DEBUG] raft-test: server 0: transport: append to 1: 2 entries [Command:term=2,index=4 Command:term=2,index=5] - logger.go:44: 18:59:55.037089 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:55.037176 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 - logger.go:44: 18:59:55.035554 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:55.037787 [DEBUG] raft-test: fsm 2: applied 2 - logger.go:44: 18:59:55.037813 [DEBUG] raft-test: fsm 0: applied 3 - logger.go:44: 18:59:55.038106 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=6] - logger.go:44: 18:59:55.038398 [DEBUG] raft-test: fsm 0: applied 4 - logger.go:44: 18:59:55.038465 [DEBUG] raft-test: server 0: control: take snapshot - logger.go:44: 18:59:55.038667 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=7] - logger.go:44: 18:59:55.038814 [DEBUG] raft-test: fsm 2: applied 3 - logger.go:44: 18:59:55.038949 [INFO] raft: Starting snapshot up to 6 - logger.go:44: 18:59:55.039176 [INFO] raft: Compacting logs from 1 to 6 - logger.go:44: 18:59:55.039241 [INFO] raft: Snapshot to 6 complete - logger.go:44: 18:59:55.039487 [DEBUG] raft-test: fsm 0: applied 5 - logger.go:44: 18:59:55.039606 [DEBUG] raft-test: server 0: reconnecting follower 1 - logger.go:44: 18:59:55.039708 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 18:59:55.039828 [DEBUG] raft-test: elect: server 1: became follower - logger.go:44: 18:59:55.039888 [DEBUG] raft-test: elect: server 2: became follower - logger.go:44: 18:59:55.040078 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=8] - logger.go:44: 18:59:55.040248 [DEBUG] raft-test: fsm 2: applied 4 - logger.go:44: 18:59:55.040491 [DEBUG] raft-test: fsm 0: applied 6 - logger.go:44: 18:59:55.040735 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Other:term=2,index=9] - logger.go:44: 18:59:55.040879 [DEBUG] raft-test: fsm 2: applied 5 - logger.go:44: 18:59:55.041211 [DEBUG] raft-test: fsm 2: applied 6 - logger.go:44: 18:59:55.041314 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:55.042860 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 18:59:55.043438 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:55.044274 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 18:59:55.045735 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:55.049986 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries - logger.go:44: 18:59:55.049987 [ERR] raft: Failed to get log at index 3: log not found - logger.go:44: 18:59:55.050792 [INFO] raft: Copied 8 bytes to local snapshot - logger.go:44: 18:59:55.050930 [INFO] raft: Compacting logs from 1 to 2 - logger.go:44: 18:59:55.050993 [INFO] raft: Installed remote snapshot - logger.go:44: 18:59:55.051207 [DEBUG] raft-test: server 0: transport: append to 1: 3 entries [Command:term=2,index=7 Command:term=2,index=8 Other:term=2,index=9] - logger.go:44: 18:59:55.051458 [INFO] raft: pipelining replication to peer {Voter 1 1} - logger.go:44: 18:59:55.050082 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries - logger.go:44: 18:59:55.051785 [DEBUG] raft-test: fsm 1: applied 5 - logger.go:44: 18:59:55.051932 [DEBUG] raft-test: fsm 1: applied 6 - logger.go:44: 18:59:55.052186 [DEBUG] raft-test: close: start - logger.go:44: 18:59:55.052508 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} - logger.go:44: 18:59:55.052612 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} - logger.go:44: 18:59:55.052739 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 18:59:55.052887 [DEBUG] raft-test: close: server 1: shutdown done - logger.go:44: 18:59:55.053018 [DEBUG] raft-test: close: server 2: shutdown done - logger.go:44: 18:59:55.053085 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 18:59:55.053154 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 18:59:55.053280 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 18:59:55.053378 [DEBUG] raft-test: close: done +--- PASS: TestControl_RestoreAfterDisconnection (0.04s) + logger.go:44: 19:02:51.664530 [DEBUG] raft-test: setup: start (3 servers) + logger.go:44: 19:02:51.665183 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 19:02:51.665261 [DEBUG] raft-test: setup: server 1: bootstrap + logger.go:44: 19:02:51.665345 [DEBUG] raft-test: setup: server 2: bootstrap + logger.go:44: 19:02:51.665403 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 19:02:51.665509 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:51.665553 [DEBUG] raft-test: setup: server 1: start + logger.go:44: 19:02:51.665598 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.665647 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:51.665687 [DEBUG] raft-test: setup: server 2: start + logger.go:44: 19:02:51.665722 [INFO] raft: Node at 1 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.665783 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0} {Suffrage:Voter ID:1 Address:1} {Suffrage:Voter ID:2 Address:2}] + logger.go:44: 19:02:51.665824 [DEBUG] raft-test: setup: done + logger.go:44: 19:02:51.665857 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 19:02:51.665860 [INFO] raft: Node at 2 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.665891 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 19:02:51.665974 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 19:02:51.685831 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 19:02:51.685999 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 19:02:51.686167 [DEBUG] raft: Votes needed: 2 + logger.go:44: 19:02:51.686277 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 19:02:51.686379 [DEBUG] raft: Vote granted from 1 in term 2. Tally: 2 + logger.go:44: 19:02:51.686457 [INFO] raft: Election won. Tally: 2 + logger.go:44: 19:02:51.686557 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 19:02:51.686724 [INFO] raft: Added peer 1, starting replication + logger.go:44: 19:02:51.686803 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 19:02:51.687062 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.687115 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 19:02:51.687331 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 19:02:51.686873 [INFO] raft: Added peer 2, starting replication + logger.go:44: 19:02:51.687768 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Noop:term=2,index=2] + logger.go:44: 19:02:51.687866 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=2,index=2] + logger.go:44: 19:02:51.688229 [INFO] raft: pipelining replication to peer {Voter 2 2} + logger.go:44: 19:02:51.688920 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 19:02:51.689039 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 19:02:51.689131 [DEBUG] raft-test: elect: done + logger.go:44: 19:02:51.689507 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.689600 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=3] + logger.go:44: 19:02:51.689631 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=3] + logger.go:44: 19:02:51.690216 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 19:02:51.690244 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 19:02:51.690553 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.690763 [DEBUG] raft-test: term: disconnect 1 + logger.go:44: 19:02:51.691095 [DEBUG] raft-test: server 0: disconnecting follower 1 + logger.go:44: 19:02:51.691420 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=4] + logger.go:44: 19:02:51.691534 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.691428 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=2,index=4] + logger.go:44: 19:02:51.691757 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 19:02:51.691889 [ERR] raft: Failed to pipeline AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 19:02:51.691647 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:51.692117 [ERR] raft: Failed to heartbeat to 1: cannot reach server 1 + logger.go:44: 19:02:51.691897 [DEBUG] raft-test: fsm 0: applied 2 + logger.go:44: 19:02:51.692545 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=5] + logger.go:44: 19:02:51.691929 [DEBUG] raft-test: fsm 2: applied 1 + logger.go:44: 19:02:51.692034 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 19:02:51.693077 [DEBUG] raft-test: server 0: transport: append to 1: 2 entries [Command:term=2,index=4 Command:term=2,index=5] + logger.go:44: 19:02:51.693270 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:51.693439 [ERR] raft: Failed to AppendEntries to {Voter 1 1}: cannot reach server 1 + logger.go:44: 19:02:51.693798 [DEBUG] raft-test: fsm 0: applied 3 + logger.go:44: 19:02:51.694092 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=6] + logger.go:44: 19:02:51.694297 [DEBUG] raft-test: fsm 2: applied 2 + logger.go:44: 19:02:51.694515 [DEBUG] raft-test: fsm 2: applied 3 + logger.go:44: 19:02:51.694615 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.694661 [DEBUG] raft-test: fsm 0: applied 4 + logger.go:44: 19:02:51.694961 [DEBUG] raft-test: server 0: control: take snapshot + logger.go:44: 19:02:51.695276 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=7] + logger.go:44: 19:02:51.695557 [INFO] raft: Starting snapshot up to 6 + logger.go:44: 19:02:51.695841 [INFO] raft: Compacting logs from 1 to 6 + logger.go:44: 19:02:51.695948 [INFO] raft: Snapshot to 6 complete + logger.go:44: 19:02:51.695856 [DEBUG] raft-test: fsm 2: applied 4 + logger.go:44: 19:02:51.695974 [DEBUG] raft-test: fsm 0: applied 5 + logger.go:44: 19:02:51.696342 [DEBUG] raft-test: server 0: reconnecting follower 1 + logger.go:44: 19:02:51.696582 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 19:02:51.696641 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 19:02:51.696767 [DEBUG] raft-test: elect: server 1: became follower + logger.go:44: 19:02:51.696993 [DEBUG] raft-test: elect: server 2: became follower + logger.go:44: 19:02:51.697278 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=2,index=8] + logger.go:44: 19:02:51.697783 [DEBUG] raft-test: fsm 0: applied 6 + logger.go:44: 19:02:51.698114 [DEBUG] raft-test: fsm 2: applied 5 + logger.go:44: 19:02:51.698004 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.698380 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Other:term=2,index=9] + logger.go:44: 19:02:51.698893 [DEBUG] raft-test: fsm 2: applied 6 + logger.go:44: 19:02:51.700844 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.700865 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 19:02:51.702745 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 19:02:51.702908 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.704427 [ERR] raft: Failed to get log at index 3: log not found + logger.go:44: 19:02:51.704634 [INFO] raft: Copied 8 bytes to local snapshot + logger.go:44: 19:02:51.704643 [DEBUG] raft-test: server 0: pipeline: append to 2: 0 entries + logger.go:44: 19:02:51.704704 [INFO] raft: Compacting logs from 1 to 2 + logger.go:44: 19:02:51.704795 [INFO] raft: Installed remote snapshot + logger.go:44: 19:02:51.704928 [DEBUG] raft-test: server 0: transport: append to 1: 3 entries [Command:term=2,index=7 Command:term=2,index=8 Other:term=2,index=9] + logger.go:44: 19:02:51.705047 [INFO] raft: pipelining replication to peer {Voter 1 1} + logger.go:44: 19:02:51.705145 [DEBUG] raft-test: fsm 1: applied 5 + logger.go:44: 19:02:51.705216 [DEBUG] raft-test: fsm 1: applied 6 + logger.go:44: 19:02:51.705245 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:51.705299 [DEBUG] raft-test: close: start + logger.go:44: 19:02:51.705488 [INFO] raft: aborting pipeline replication to peer {Voter 1 1} + logger.go:44: 19:02:51.705521 [INFO] raft: aborting pipeline replication to peer {Voter 2 2} + logger.go:44: 19:02:51.705502 [DEBUG] raft-test: server 0: transport: append to 2: 0 entries + logger.go:44: 19:02:51.705660 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 19:02:51.705746 [DEBUG] raft-test: close: server 1: shutdown done + logger.go:44: 19:02:51.705825 [DEBUG] raft-test: close: server 2: shutdown done + logger.go:44: 19:02:51.705865 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 19:02:51.705937 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 19:02:51.705995 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 19:02:51.706054 [DEBUG] raft-test: close: done === RUN TestFSM_Restore --- PASS: TestFSM_Restore (0.00s) === RUN TestServer_StartAndShutdown ---- PASS: TestServer_StartAndShutdown (0.03s) - logger.go:44: 18:59:55.053928 [DEBUG] raft-test: setup: start (1 servers) - logger.go:44: 18:59:55.054173 [DEBUG] raft-test: setup: server 0: bootstrap - logger.go:44: 18:59:55.054336 [DEBUG] raft-test: setup: server 0: start - logger.go:44: 18:59:55.054588 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0}] - logger.go:44: 18:59:55.054701 [DEBUG] raft-test: setup: done - logger.go:44: 18:59:55.054774 [DEBUG] raft-test: elect: start (server 0) - logger.go:44: 18:59:55.054888 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 18:59:55.054964 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s - logger.go:44: 18:59:55.055139 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") - logger.go:44: 18:59:55.082870 [WARN] raft: Heartbeat timeout from "" reached, starting election - logger.go:44: 18:59:55.083023 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 - logger.go:44: 18:59:55.083119 [DEBUG] raft: Votes needed: 1 - logger.go:44: 18:59:55.083191 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 - logger.go:44: 18:59:55.083244 [INFO] raft: Election won. Tally: 1 - logger.go:44: 18:59:55.083308 [INFO] raft: Node at 0 [Leader] entering Leader state - logger.go:44: 18:59:55.083518 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 18:59:55.083696 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms - logger.go:44: 18:59:55.083826 [DEBUG] raft-test: elect: done - logger.go:44: 18:59:55.084068 [DEBUG] raft-test: fsm 0: applied 1 - logger.go:44: 18:59:55.084238 [DEBUG] raft-test: close: start - logger.go:44: 18:59:55.084394 [DEBUG] raft-test: close: server 0: shutdown done - logger.go:44: 18:59:55.084550 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 18:59:55.084606 [DEBUG] raft-test: close: done +--- PASS: TestServer_StartAndShutdown (0.02s) + logger.go:44: 19:02:51.706295 [DEBUG] raft-test: setup: start (1 servers) + logger.go:44: 19:02:51.706408 [DEBUG] raft-test: setup: server 0: bootstrap + logger.go:44: 19:02:51.706482 [DEBUG] raft-test: setup: server 0: start + logger.go:44: 19:02:51.706589 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:0 Address:0}] + logger.go:44: 19:02:51.706638 [DEBUG] raft-test: setup: done + logger.go:44: 19:02:51.706679 [DEBUG] raft-test: elect: start (server 0) + logger.go:44: 19:02:51.706739 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 19:02:51.706780 [DEBUG] raft-test: elect: server 0: wait to become leader within 1.5s + logger.go:44: 19:02:51.706858 [INFO] raft: Node at 0 [Follower] entering Follower state (Leader: "") + logger.go:44: 19:02:51.725230 [WARN] raft: Heartbeat timeout from "" reached, starting election + logger.go:44: 19:02:51.725497 [INFO] raft: Node at 0 [Candidate] entering Candidate state in term 2 + logger.go:44: 19:02:51.725705 [DEBUG] raft: Votes needed: 1 + logger.go:44: 19:02:51.725845 [DEBUG] raft: Vote granted from 0 in term 2. Tally: 1 + logger.go:44: 19:02:51.725949 [INFO] raft: Election won. Tally: 1 + logger.go:44: 19:02:51.726083 [INFO] raft: Node at 0 [Leader] entering Leader state + logger.go:44: 19:02:51.726383 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 19:02:51.726651 [DEBUG] raft-test: elect: server 0: wait for other servers to become followers within 40ms + logger.go:44: 19:02:51.726871 [DEBUG] raft-test: elect: done + logger.go:44: 19:02:51.727295 [DEBUG] raft-test: fsm 0: applied 1 + logger.go:44: 19:02:51.727590 [DEBUG] raft-test: close: start + logger.go:44: 19:02:51.727988 [DEBUG] raft-test: close: server 0: shutdown done + logger.go:44: 19:02:51.728241 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 19:02:51.728403 [DEBUG] raft-test: close: done === RUN ExampleControl_Commands ---- PASS: ExampleControl_Commands (0.05s) +--- PASS: ExampleControl_Commands (0.36s) === RUN ExampleControl_Snapshots ---- PASS: ExampleControl_Snapshots (0.05s) +--- PASS: ExampleControl_Snapshots (0.04s) PASS -ok github.com/CanonicalLtd/raft-test 1.279s +ok github.com/CanonicalLtd/raft-test 1.575s === RUN TestNotifier_Acquired --- PASS: TestNotifier_Acquired (0.00s) - logger.go:44: 18:59:51.370597 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 18:59:51.371156 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 19:02:49.918103 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 19:02:49.918323 [DEBUG] raft-test: server 0: leadership: stop watching === RUN TestNotifier_AcquiredTimeout --- PASS: TestNotifier_AcquiredTimeout (0.00s) - logger.go:44: 18:59:51.374556 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 19:02:49.918637 [DEBUG] raft-test: server 0: leadership: stop watching === RUN TestNotifier_LeaderhsipLost --- PASS: TestNotifier_LeaderhsipLost (0.00s) - logger.go:44: 18:59:51.376054 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 18:59:51.376328 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 18:59:51.376461 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 19:02:49.918807 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 19:02:49.918892 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 19:02:49.918923 [DEBUG] raft-test: server 0: leadership: stop watching === RUN TestNotifier_LostTwice --- PASS: TestNotifier_LostTwice (0.00s) - logger.go:44: 18:59:51.376858 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 18:59:51.377040 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 19:02:49.919068 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 19:02:49.919145 [DEBUG] raft-test: server 0: leadership: stop watching === RUN TestNotifier_UnexpectedLeadershipChange --- PASS: TestNotifier_UnexpectedLeadershipChange (0.00s) === RUN TestNotifier_InconsistentNotifications --- PASS: TestNotifier_InconsistentNotifications (0.01s) - logger.go:44: 18:59:51.388244 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 19:02:49.929569 [DEBUG] raft-test: server 0: leadership: acquired === RUN TestNotifier_DoubleAcquiredRequest --- PASS: TestNotifier_DoubleAcquiredRequest (0.00s) === RUN TestNotifier_DoubleNotification --- PASS: TestNotifier_DoubleNotification (0.01s) - logger.go:44: 18:59:51.399232 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 19:02:49.940175 [DEBUG] raft-test: server 0: leadership: acquired === RUN TestNotifier_OneLeadershipAfterTheOther --- PASS: TestNotifier_OneLeadershipAfterTheOther (0.00s) - logger.go:44: 18:59:51.399924 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 18:59:51.400133 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 18:59:51.400258 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 18:59:51.400375 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 19:02:49.940508 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 19:02:49.940602 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 19:02:49.940690 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 19:02:49.940754 [DEBUG] raft-test: server 0: leadership: stop watching === RUN TestTracker_Acquired --- PASS: TestTracker_Acquired (0.00s) - logger.go:44: 18:59:51.400752 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 18:59:51.400908 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 18:59:51.400972 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 18:59:51.401081 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 19:02:49.940944 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 19:02:49.941032 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 19:02:49.941065 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 19:02:49.941123 [DEBUG] raft-test: server 2: leadership: stop watching === RUN TestTracker_AcquiredTimeout --- PASS: TestTracker_AcquiredTimeout (0.00s) - logger.go:44: 18:59:51.401608 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 19:02:49.942290 [DEBUG] raft-test: server 0: leadership: stop watching === RUN TestTracker_AcquiredTwice --- PASS: TestTracker_AcquiredTwice (0.00s) - logger.go:44: 18:59:51.402194 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 18:59:51.402404 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 18:59:51.402473 [DEBUG] raft-test: server 1: leadership: stop watching - logger.go:44: 18:59:51.402585 [DEBUG] raft-test: server 2: leadership: stop watching + logger.go:44: 19:02:49.942568 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 19:02:49.942671 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 19:02:49.942706 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 19:02:49.942770 [DEBUG] raft-test: server 2: leadership: stop watching === RUN TestTracker_AcquiredAfterLost --- PASS: TestTracker_AcquiredAfterLost (0.00s) - logger.go:44: 18:59:51.402987 [DEBUG] raft-test: server 0: leadership: acquired - logger.go:44: 18:59:51.403138 [DEBUG] raft-test: server 0: leadership: lost - logger.go:44: 18:59:51.403241 [DEBUG] raft-test: server 2: leadership: stop watching - logger.go:44: 18:59:51.403352 [DEBUG] raft-test: server 0: leadership: stop watching - logger.go:44: 18:59:51.403413 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 19:02:49.942953 [DEBUG] raft-test: server 0: leadership: acquired + logger.go:44: 19:02:49.943026 [DEBUG] raft-test: server 0: leadership: lost + logger.go:44: 19:02:49.943066 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 19:02:49.943094 [DEBUG] raft-test: server 1: leadership: stop watching + logger.go:44: 19:02:49.943151 [DEBUG] raft-test: server 2: leadership: stop watching === RUN TestTracker_AddSameServerID --- PASS: TestTracker_AddSameServerID (0.00s) - logger.go:44: 18:59:51.403839 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 19:02:49.943354 [DEBUG] raft-test: server 0: leadership: stop watching === RUN TestTracker_AddAfterObserving --- PASS: TestTracker_AddAfterObserving (0.00s) - logger.go:44: 18:59:51.404200 [DEBUG] raft-test: server 0: leadership: stop watching + logger.go:44: 19:02:49.943528 [DEBUG] raft-test: server 0: leadership: stop watching PASS -ok github.com/CanonicalLtd/raft-test/internal/election 0.069s +ok github.com/CanonicalLtd/raft-test/internal/election 0.052s === RUN TestEvent --- PASS: TestEvent (0.00s) PASS -ok github.com/CanonicalLtd/raft-test/internal/event 0.032s +ok github.com/CanonicalLtd/raft-test/internal/event 0.025s ? github.com/CanonicalLtd/raft-test/internal/fsms [no test files] === RUN TestNew ---- PASS: TestNew (1.54s) - logger.go:44: 18:59:39.676386 [TRACE] raft-test: hello +--- PASS: TestNew (0.01s) + logger.go:44: 19:02:46.016787 [TRACE] raft-test: hello PASS -ok github.com/CanonicalLtd/raft-test/internal/logging 1.555s +ok github.com/CanonicalLtd/raft-test/internal/logging 0.015s === RUN TestStringifyLogs === RUN TestStringifyLogs/0_entries === RUN TestStringifyLogs/1_entry_[Noop:term=1,index=1] @@ -2577,43 +2729,43 @@ --- PASS: TestPeer_CommandLogsCount (0.00s) === RUN TestFaultyTransport_AppendEntries_Default --- PASS: TestFaultyTransport_AppendEntries_Default (0.00s) - logger.go:44: 18:59:55.716394 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries - logger.go:44: 18:59:55.716756 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:50.311796 [DEBUG] raft-test: server 0: transport: append to 1: 0 entries + logger.go:44: 19:02:50.311964 [DEBUG] raft-test: server 0: transport: append to 1: not connected === RUN TestFaultyTransport_AppendEntries_Connected --- PASS: TestFaultyTransport_AppendEntries_Connected (0.00s) - logger.go:44: 18:59:55.717110 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=1,index=1] + logger.go:44: 19:02:50.312134 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=1,index=1] === RUN TestFaultyTransport_PipelineAppendEntries_Disconnected --- PASS: TestFaultyTransport_PipelineAppendEntries_Disconnected (0.00s) - logger.go:44: 18:59:55.717709 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries - logger.go:44: 18:59:55.717853 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 19:02:50.312477 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 19:02:50.312540 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected === RUN TestFaultyTransport_PipelineAppendEntries_Connected --- PASS: TestFaultyTransport_PipelineAppendEntries_Connected (0.00s) - logger.go:44: 18:59:55.718227 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries + logger.go:44: 19:02:50.312713 [DEBUG] raft-test: server 0: pipeline: append to 1: 0 entries === RUN TestFaultyTransport_Deposing --- PASS: TestFaultyTransport_Deposing (0.00s) - logger.go:44: 18:59:55.718743 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=1,index=1] - logger.go:44: 18:59:55.718965 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=1,index=1] - logger.go:44: 18:59:55.719139 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=1,index=3] - logger.go:44: 18:59:55.719361 [DEBUG] raft-test: server 0: transport: append to 2: syncing logs - logger.go:44: 18:59:55.719470 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=1,index=3] - logger.go:44: 18:59:55.719643 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=1,index=3] - logger.go:44: 18:59:55.719705 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected - logger.go:44: 18:59:55.719809 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=1,index=3] - logger.go:44: 18:59:55.719869 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected - logger.go:44: 18:59:55.719941 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=1,index=3] - logger.go:44: 18:59:55.720014 [DEBUG] raft-test: server 0: transport: append to 1: not connected - logger.go:44: 18:59:55.720116 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=1,index=3] - logger.go:44: 18:59:55.720177 [DEBUG] raft-test: server 0: transport: append to 2: not connected + logger.go:44: 19:02:50.312966 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=1,index=1] + logger.go:44: 19:02:50.313114 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Noop:term=1,index=1] + logger.go:44: 19:02:50.313189 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=1,index=3] + logger.go:44: 19:02:50.313278 [DEBUG] raft-test: server 0: transport: append to 2: syncing logs + logger.go:44: 19:02:50.313346 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=1,index=3] + logger.go:44: 19:02:50.313441 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=1,index=3] + logger.go:44: 19:02:50.313473 [DEBUG] raft-test: server 0: pipeline: append to 1: not connected + logger.go:44: 19:02:50.313517 [DEBUG] raft-test: server 0: pipeline: append to 2: 1 entry [Command:term=1,index=3] + logger.go:44: 19:02:50.313544 [DEBUG] raft-test: server 0: pipeline: append to 2: not connected + logger.go:44: 19:02:50.313573 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Command:term=1,index=3] + logger.go:44: 19:02:50.313599 [DEBUG] raft-test: server 0: transport: append to 1: not connected + logger.go:44: 19:02:50.313640 [DEBUG] raft-test: server 0: transport: append to 2: 1 entry [Command:term=1,index=3] + logger.go:44: 19:02:50.313665 [DEBUG] raft-test: server 0: transport: append to 2: not connected === RUN TestNetwork_FaultyEnqueue --- PASS: TestNetwork_FaultyEnqueue (0.00s) - logger.go:44: 18:59:55.720511 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes - logger.go:44: 18:59:55.720704 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=1,index=1] - logger.go:44: 18:59:55.720850 [DEBUG] raft-test: server 0: transport: append to 1: newer term - logger.go:44: 18:59:55.721108 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=0,index=0] - logger.go:44: 18:59:55.721170 [DEBUG] raft-test: server 0: pipeline: append to: 1: enqueue fault: command 1 - logger.go:44: 18:59:55.721273 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes + logger.go:44: 19:02:50.313830 [DEBUG] raft-test: server 0: establish outbound connection to all other nodes + logger.go:44: 19:02:50.313913 [DEBUG] raft-test: server 0: transport: append to 1: 1 entry [Noop:term=1,index=1] + logger.go:44: 19:02:50.313967 [DEBUG] raft-test: server 0: transport: append to 1: newer term + logger.go:44: 19:02:50.314055 [DEBUG] raft-test: server 0: pipeline: append to 1: 1 entry [Command:term=0,index=0] + logger.go:44: 19:02:50.314082 [DEBUG] raft-test: server 0: pipeline: append to: 1: enqueue fault: command 1 + logger.go:44: 19:02:50.314121 [DEBUG] raft-test: server 0: dropping outbound connection to all other nodes PASS -ok github.com/CanonicalLtd/raft-test/internal/network 0.037s +ok github.com/CanonicalLtd/raft-test/internal/network 0.021s create-stamp debian/debhelper-build-stamp fakeroot debian/rules binary dh binary --buildsystem=golang --with=golang @@ -2641,12 +2793,14 @@ dpkg-source --after-build . dpkg-buildpackage: info: binary-only upload (no source included) I: copying local configuration +I: user script /srv/workspace/pbuilder/6245/tmp/hooks/B01_cleanup starting +I: user script /srv/workspace/pbuilder/6245/tmp/hooks/B01_cleanup finished I: unmounting dev/ptmx filesystem I: unmounting dev/pts filesystem I: unmounting dev/shm filesystem I: unmounting proc filesystem I: unmounting sys filesystem I: cleaning the build env -I: removing directory /srv/workspace/pbuilder/3458 and its subdirectories -I: Current time: Mon Feb 15 07:00:24 -12 2021 -I: pbuilder-time-stamp: 1613415624 +I: removing directory /srv/workspace/pbuilder/6245 and its subdirectories +I: Current time: Tue Feb 16 09:03:00 +14 2021 +I: pbuilder-time-stamp: 1613415780